diff --git a/.circleci/config.yml b/.circleci/config.yml index 21a0407c8..9ffe8cd76 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,6 +21,21 @@ jobs: steps: - checkout + # https://support.circleci.com/hc/en-us/articles/16164465307931-Using-an-environment-variable-to-define-a-path-when-using-the-Docker-executor + - run: + name: "Fix CIRCLE_WORKING_DIRECTORY" + # replace ~ by $HOME + command: echo 'CIRCLE_WORKING_DIRECTORY="${CIRCLE_WORKING_DIRECTORY/#\~/$HOME}"' >> $BASH_ENV + + - run: + name: Test env + command: | + echo "id: `id`" + cd + cd repo + pwd + echo "$CIRCLE_WORKING_DIRECTORY" + - run: name: update package manager command: apt-get update @@ -49,6 +64,10 @@ jobs: name: Install Docker client command: apt-get -y install docker-ce docker-ce-cli containerd.io + - run: + name: Install zip + command: apt-get --yes install zip + - setup_remote_docker: version: 19.03.13 docker_layer_caching: true @@ -73,3 +92,90 @@ jobs: - store_artifacts: path: build/libs + + - run: + name: Build Armadillo and R CICD image + command: | + ./gradlew docker + ./docker/bin/prepare.bash ci + + - run: + name: Make sure all images declared in docker-compose.yml are available and ready + command: | + cd $CIRCLE_WORKING_DIRECTORY + cd build/docker/armadillo-compose + + docker images ls + docker compose up -d + docker images ls + + - run: + name: Start docker-compose and wait for `release-test.R` to finish + command: | + # FIXME: make this name not directory dependent + ARMADILLO="armadillo-compose-armadillo-1" + + cd $CIRCLE_WORKING_DIRECTORY + cd build/docker/armadillo-compose + + docker compose up -d + docker ps + + + # 1 CircleCI docker in docker workaround + docker cp ./armadillo/config $ARMADILLO:/config + docker cp ./armadillo/data $ARMADILLO:/data + docker cp ./armadillo/logs $ARMADILLO:/logs + # .1 CircleCI docker in docker workaround + + # Poll to see Armadillo is up + docker container run --network container:$ARMADILLO \ + docker.io/jwilder/dockerize \ + -wait http://localhost:8080/ \ + -wait-retry-interval 20s \ + -timeout 30s || echo "Timed out" + + cd $CIRCLE_WORKING_DIRECTORY + cd build/docker/cicd + + # Run release-test.R + docker container run \ + --network container:$ARMADILLO \ + --interactive --tty \ + --entrypoint /bin/bash molgenis/r-cicd -c "cd /cicd/scripts/release ; ./armadillo-ready.bash" + + cd $CIRCLE_WORKING_DIRECTORY + cd build/docker/armadillo-compose + + # 2 CircleCI docker in docker workaround + # See what changed within Armadillo + docker cp $ARMADILLO:/logs ./armadillo/logs + # .2 CircleCI docker in docker workaround + + docker container ls + docker compose down + docker images ls + docker compose rm + + - store_artifacts: + path: build/libs + + - run: + name: Zip armadillo-compose + command: | + cd $CIRCLE_WORKING_DIRECTORY + mkdir build/artifacts + + # Workaround for ~ expansion not working + K=`pwd` + cd build/docker/ + zip -r $K/build/artifacts/armadillo-compose.zip armadillo-compose + + - store_artifacts: + path: build/artifacts/armadillo-compose.zip + + +workflows: + build-deploy: + jobs: + - build diff --git a/.gitignore b/.gitignore index 26a9e9be9..9035d791a 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ build/ **/audit.log # during development you can safely create an application.yml but don't commit it! -application.yml -application.yaml -application.properties +/application.yml +/application.yaml +/application.properties +.Rproj.user diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index df29d9f96..9acf7a2f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,6 +48,32 @@ List messages to see usage of conventional commits from the past. git log --pretty=format:"%s" | cut -c -20 ``` +## Continuous integration + +- we test on each PR and merges on master +- we build docker compose set for CI testing and demo purposes. + - [CI testing](./docker/ci/README.md) + - Demo zip file is a delivery you as artifact + - Master build have a armadillo-compose.zip for download + +### Local CI build + +``` +./gradlew docker +./docker/bin/prepare.bash ci + +cd build/docker/armadillo-compose +# Follow README.md to see Armadillo and R images run in container +docker compose build +docker compose up +``` + +then run `./release-test.R` against this. + +### Local CI test of armadillo-compose + +Follow [docker CI README.md](./docker/ci/README.md) to run `release-test.R` using `molgenis/r-cicd` image + ## Profile xenon with resourcer whitelisted returns a host.docker.internal error When developing locally, it might be possible to come across the container error: `Could not resolve host: host.docker.internal`, especially when developing on a non-supported operating system when resourcer is whitelisted (such as xenon). diff --git a/Dockerfile b/Dockerfile index 807b33bee..b2297b0d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,9 @@ -FROM eclipse-temurin:17.0.9_9-jdk-focal +FROM --platform=linux/amd64 eclipse-temurin:17.0.9_9-jdk-focal VOLUME /data +VOLUME /config +VOLUME /logs + ARG JAR_FILE EXPOSE 8080 COPY ${JAR_FILE} /app.jar -ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom","-jar","/app.jar"] \ No newline at end of file +ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom","-DSPRING_CONFIG_ADDITIONAL_LOCATION=/config/application.yml", "-jar","/app.jar"] diff --git a/application.template.yml b/application.template.yml index 16eecf3d1..997659512 100644 --- a/application.template.yml +++ b/application.template.yml @@ -2,6 +2,14 @@ armadillo: # set this false if you DON'T want Armadillo to create/edit/delete profile docker images via the user interface docker-management-enabled: true + # set this true if you want Armadillo runs as a docker container + # NOTE: this needs "docker-management-enabled" == false + docker-run-in-container: false + + # when running the R containers from a docker-compose.yml they get prefixes based on the directory name of the + # docker-compose.yml file ie armadillo-dev-" + profileName + "-1". Same goes for Armadillo ie armadillo-dev-armadillo-1" + container-prefix: '' + # uncomment this to configure an oidc user as admin user # oidc-admin-user: user@yourdomain.org diff --git a/armadillo/src/main/java/org/molgenis/armadillo/controller/ProfilesDockerController.java b/armadillo/src/main/java/org/molgenis/armadillo/controller/ProfilesDockerController.java index 9d424bf69..cce0ed7cb 100644 --- a/armadillo/src/main/java/org/molgenis/armadillo/controller/ProfilesDockerController.java +++ b/armadillo/src/main/java/org/molgenis/armadillo/controller/ProfilesDockerController.java @@ -36,6 +36,7 @@ public class ProfilesDockerController { public static final String DOCKER_MANAGEMENT_ENABLED = "armadillo.docker-management-enabled"; + public static final String DOCKER_RUN_IN_CONTAINER = "armadillo.docker-run-in-container"; private final DockerService dockerService; private final AuditEventPublisher auditor; diff --git a/armadillo/src/main/java/org/molgenis/armadillo/profile/DockerService.java b/armadillo/src/main/java/org/molgenis/armadillo/profile/DockerService.java index f61e4857e..d9a103159 100644 --- a/armadillo/src/main/java/org/molgenis/armadillo/profile/DockerService.java +++ b/armadillo/src/main/java/org/molgenis/armadillo/profile/DockerService.java @@ -25,6 +25,7 @@ import org.molgenis.armadillo.metadata.ProfileStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Service; @@ -39,6 +40,12 @@ public class DockerService { private final DockerClient dockerClient; private final ProfileService profileService; + @Value("${armadillo.docker-run-in-container:true}") + private boolean inContainer; + + @Value("${armadillo.container-prefix:''}") + private String containerPrefix; + public DockerService(DockerClient dockerClient, ProfileService profileService) { this.dockerClient = dockerClient; this.profileService = profileService; @@ -74,12 +81,47 @@ public Map getAllProfileStatuses() { return statuses; } + // `docker container ps` show these name structure + + /** + * The container can run in its own network/compose. + * + *

Both the profiles and/or Armadillo can be part of a docker-compose.yml. You can check with + * `docker container ps` to see this name structure. + * + * @param profileName the profile name set by DataManager. + * @return adjusted container name if applicable. + */ + String asContainerName(String profileName) { + if (!inContainer) { + LOG.warn("NO ".repeat(100) + " " + profileName); + return profileName; + } + + if (containerPrefix.isEmpty()) { + LOG.error("Running in container without prefix: " + profileName); + return profileName; + } + + LOG.warn("YES ".repeat(100) + " " + profileName); + return containerPrefix + profileName + "-1"; + } + + String asProfileName(String containerName) { + if (inContainer) { + return containerName.replace("armadillo-docker-compose-", "").replace("-1", ""); + } + return containerName; + } + public ContainerInfo getProfileStatus(String profileName) { // check profile exists profileService.getByName(profileName); + String containerName = asContainerName(profileName); try { - InspectContainerResponse containerInfo = dockerClient.inspectContainerCmd(profileName).exec(); + InspectContainerResponse containerInfo = + dockerClient.inspectContainerCmd(containerName).exec(); var tags = getImageTags(containerInfo.getName()); return ContainerInfo.create(tags, ProfileStatus.of(containerInfo.getState())); } catch (ProcessingException e) { @@ -94,12 +136,15 @@ public ContainerInfo getProfileStatus(String profileName) { } public void startProfile(String profileName) { + String containerName = asContainerName(profileName); + LOG.info(profileName + " : " + containerName); + var profileConfig = profileService.getByName(profileName); pullImage(profileConfig); - stopContainer(profileName); - removeContainer(profileName); // for reinstall + stopContainer(containerName); + removeContainer(containerName); // for reinstall installImage(profileConfig); - startContainer(profileName); + startContainer(containerName); } private void installImage(ProfileConfig profileConfig) { @@ -122,21 +167,22 @@ private void installImage(ProfileConfig profileConfig) { } } - private void startContainer(String profileName) { + private void startContainer(String containerName) { try { - dockerClient.startContainerCmd(profileName).exec(); + dockerClient.startContainerCmd(containerName).exec(); } catch (DockerException e) { - throw new ImageStartFailedException(profileName, e); + throw new ImageStartFailedException(containerName, e); } } - private void stopContainer(String profileName) { + private void stopContainer(String containerName) { + String profileName = "stoppingContainer has not profileName: " + containerName; try { - dockerClient.stopContainerCmd(profileName).exec(); + dockerClient.stopContainerCmd(containerName).exec(); } catch (DockerException e) { try { InspectContainerResponse containerInfo = - dockerClient.inspectContainerCmd(profileName).exec(); + dockerClient.inspectContainerCmd(containerName).exec(); // should not be a problem if not running if (TRUE.equals(containerInfo.getState().getRunning())) { throw new ImageStopFailedException(profileName, e); @@ -180,14 +226,14 @@ public void removeProfile(String profileName) { removeContainer(profileName); } - private void removeContainer(String profileName) { + private void removeContainer(String containerName) { try { - dockerClient.removeContainerCmd(profileName).exec(); + dockerClient.removeContainerCmd(containerName).exec(); } catch (NotFoundException nfe) { - LOG.info("Couldn't remove container '{}' because it doesn't exist", profileName); + LOG.info("Couldn't remove container '{}' because it doesn't exist", containerName); // not a problem, wanted to remove anyway } catch (DockerException e) { - throw new ContainerRemoveFailedException(profileName, e); + throw new ContainerRemoveFailedException(containerName, e); } } diff --git a/armadillo/src/main/resources/application.yml b/armadillo/src/main/resources/application.yml index 8e1b2137d..4b290b540 100644 --- a/armadillo/src/main/resources/application.yml +++ b/armadillo/src/main/resources/application.yml @@ -1,8 +1,19 @@ armadillo: # set this to 'true' if you want to accept permissions from oidc provider oidc-permission-enabled: false + # set this true if you want Armadillo to create/edit/delete profile docker images docker-management-enabled: true + + # set this true if you want Armadillo runs as a docker container + # NOTE: this needs "docker-management-enabled" == false + docker-run-in-container: true + + # when running the R containers from a docker-compose.yml they get prefixes based on the directory name of the + # docker-compose.yml file ie armadillo-dev-" + profileName + "-1". Same goes for Armadillo ie armadillo-dev-armadillo-1" + container-prefix: '' + + # uncomment this to configure a default admin user # oidc-admin-user: user@yourdomain.org diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..0d4743420 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,59 @@ +# Local docker compose + +When developing locally and want to run all in docker containers we need a custom +image of Armadillo. + +We do have [CI testing](./ci/README.md) in place. One of its deliverables is `armadillo-compose.zip` which can be used elsewhere. + +# Hacking + +## Armadillo on host? + +Is IP member of host? `docker network inspect host` +Is IP member of other networks? + +## Which network + +``` +docker network ls +NETWORK ID NAME DRIVER SCOPE +b187c856cdb8 armadillo-docker-compose_default bridge local +... +``` + +How to get 'my' (armadillo) network name and is it a container or VM with docker for datashield images? + +## Get network members + +``` +docker network inspect armadillo-docker-compose_default + +... + "Containers": { + "5190dcfefb75e073364e55da821c625999fd653fe6f72281728ad331d42dad22": { + "Name": "armadillo-docker-compose-armadillo-1", + "EndpointID": "d4e144cd9079da27b04fe03b65bf3a1b46f33ec663eaaf8d9e328f18a8af1615", + "MacAddress": "02:42:c0:a8:20:04", + "IPv4Address": "192.168.32.4/20", + "IPv6Address": "" + }, +... + } +... +``` + +## List all containers + +``` +docker container ls +docker ps + +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +5190dcfefb75 molgenis/molgenis-armadillo:latest "java -Djava.securit…" 32 minutes ago Up 32 minutes 0.0.0.0:8080->8080/tcp armadillo-docker-compose-armadillo-1 +``` + +## Inspect container + +``` +docker container inspect 5190dcfefb75e +``` \ No newline at end of file diff --git a/docker/bin/prepare.bash b/docker/bin/prepare.bash new file mode 100755 index 000000000..c9d06b423 --- /dev/null +++ b/docker/bin/prepare.bash @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +function fail { + echo "Failed $1" + exit 1 +} + +CICD_VERSION="latest" +CICD_IMAGE="molgenis/r-cicd" + +TARGET_ENV="ci" + +# Store the current working directory +cwd=$(pwd) + +# Get the directory of the script +PROJECT_DIR="$(git rev-parse --show-toplevel)" +BUILD_ROOT="$PROJECT_DIR/build/docker" +ARMADILLO_COMPOSE_BUILD_DIR="$BUILD_ROOT/armadillo-compose" +DOCKER_DIR="$PROJECT_DIR/docker" +TARGET_DIR="$DOCKER_DIR/$TARGET_ENV" + +cd "$TARGET_DIR" || fail "Cannot jump into $TARGET_DIR" + +mkdir -p "$ARMADILLO_COMPOSE_BUILD_DIR" || exit 1 + +echo "Project dir : $PROJECT_DIR" +echo "Armadillo compose dir : $ARMADILLO_COMPOSE_BUILD_DIR" +echo "Target : $TARGET_DIR" + +cd "$PROJECT_DIR" || fail "$PROJECT_DIR does not exists" + +rm -rf "${ARMADILLO_COMPOSE_BUILD_DIR:?}/"* || exit 1 + +# Make sure Armadillo has needed directories and files +ARMADILLO="$ARMADILLO_COMPOSE_BUILD_DIR/armadillo/" +# System dir must exists +mkdir -p "$ARMADILLO/data/system" || exit 1 +# lifecycle data must exits +cp -r "$PROJECT_DIR/data/shared-lifecycle" "$ARMADILLO/data/" || exit 1 +mkdir -p "$ARMADILLO/logs" || exit 1 +mkdir -p "$ARMADILLO/config" || exit 1 +# Specific application.yml file is needed +cp -r "$TARGET_DIR/application.yml" "$ARMADILLO/config" || exit 1 + +cp -r "$TARGET_DIR/docker-compose.yml" "$ARMADILLO_COMPOSE_BUILD_DIR" || exit 1 +cp "$TARGET_DIR/armadillo-compose.md" "$ARMADILLO_COMPOSE_BUILD_DIR/README.md" || exit 1 +cp "$BUILD_ROOT"/*.jar "$ARMADILLO_COMPOSE_BUILD_DIR/" || exit 1 +cp "$BUILD_ROOT/Dockerfile" "$ARMADILLO_COMPOSE_BUILD_DIR/" || exit 1 + +CICD_DIR="$BUILD_ROOT/cicd" +rm -rf "${CICD_DIR:?}/"* || exit 1 + +# expected by `release-test.R` +ARMADILLO_ROOT="$CICD_DIR/armadillo" +mkdir -p "$ARMADILLO_ROOT" || exit 1 +cp -r "$ARMADILLO/data" "$ARMADILLO_ROOT/" || exit 1 + +BIN_DIR="$ARMADILLO_ROOT/scripts/release/" +mkdir -p "$BIN_DIR" || exit 1 + +LOG_DIR="$ARMADILLO_ROOT/log/" +mkdir -p "$LOG_DIR" || exit 1 + +cp "$PROJECT_DIR/scripts/release/release-test.R" "$BIN_DIR/" || exit 1 +cp "$PROJECT_DIR/scripts/release/install_release_script_dependencies.R" "$BIN_DIR/" || exit 1 +cp "$TARGET_DIR/armadillo-ready.bash" "$BIN_DIR/" || exit 1 +cp "$TARGET_DIR/ci.env" "$BIN_DIR/.env" || exit 1 + +cp "$TARGET_DIR/Dockerfile" "$CICD_DIR/" +cd "$CICD_DIR" || exit 1 +docker build . --platform linux/amd64 --tag "$CICD_IMAGE:$CICD_VERSION" || fail "Unable to build R CICD image" + +cd "$cwd" || exit 1 diff --git a/docker/ci/.gitignore b/docker/ci/.gitignore new file mode 100644 index 000000000..4195b9c57 --- /dev/null +++ b/docker/ci/.gitignore @@ -0,0 +1 @@ +fake-tree diff --git a/docker/ci/Dockerfile b/docker/ci/Dockerfile new file mode 100644 index 000000000..04ec44c93 --- /dev/null +++ b/docker/ci/Dockerfile @@ -0,0 +1,34 @@ +FROM --platform=linux/amd64 rocker/verse:4.1.1 + +LABEL maintainer="Clemens Tolboom " +LABEL version="0.0.1" +LABEL description="Image build to run release-test.R" + +RUN apt update +RUN apt --yes install git curl ssh libcurl4-openssl-dev libssl-dev build-essential ninja-build cmake jq + +RUN install2.r --skipinstalled --repo https://cloud.r-project.org devtools diffobj getPass + +RUN install2.r --skipinstalled --repo https://cloud.r-project.org arrow future RCurl cli + +RUN install2.r --skipinstalled --repo https://cloud.r-project.org DSI resourcer MolgenisArmadillo DSMolgenisArmadillo + +RUN mkdir -p /cicd/armadillo/scripts/release + +# Copy slow changing file only +COPY armadillo/scripts/release/install_release_script_dependencies.R /cicd/scripts/release/ + +RUN /cicd/scripts/release/install_release_script_dependencies.R + +# Copy rest from of scripts +COPY armadillo/scripts/release/* /cicd/scripts/release/ + +COPY armadillo /cicd/ + +#RUN adduser armadillo + +#RUN chown -R armadillo:armadillo /armadillo/ + +#USER armadillo + +CMD ["bash"] diff --git a/docker/ci/README.md b/docker/ci/README.md new file mode 100644 index 000000000..b4679ab0c --- /dev/null +++ b/docker/ci/README.md @@ -0,0 +1,26 @@ +# Continue integration + +In the `docker/ci` directory + +- Check `ci.env` for correct values which is used by `release-test.R` script. +- Check `application.yml` for the correct profiles + - Make those profiles matching those `docker-compose.yml` +- Run `../prepare.bash` (calls `gradle clean build docker`) +- Note [build/docker/cicd/](../../build/docker/cicd) which hold `armadillo/` for `release-test.R` +- Run `docker compose up` + +## Start the R CICD + +```sh +docker container run \ + --network container:armadillo-compose-armadillo-1 \ + --volume armadillo:/cicd:rw \ + --interactive --tty \ + --entrypoint /bin/bash molgenis/r-cicd -c "ls -l /cicd ; cd /cicd/scripts/release ; ./armadillo-ready.bash" +``` + +## Debugging + +```sh +docker run --interactive --tty --entrypoint /bin/bash molgenis/r-cicd +``` diff --git a/docker/ci/application.yml b/docker/ci/application.yml new file mode 100644 index 000000000..ca7eff30e --- /dev/null +++ b/docker/ci/application.yml @@ -0,0 +1,92 @@ +# NOTE: when changing this file you must rerun: +# - docker compose --file docker-compose-local.yml build + +armadillo: + # DO NOT CHANGE + docker-management-enabled: false + # DO NOT CHANGE + docker-run-in-container: true + + # when running the R containers from a docker-compose.yml they get prefixes based on the directory name of the + # docker-compose.yml file ie armadillo-dev-" + profileName + "-1". Same goes for Armadillo ie armadillo-dev-armadillo-1" + container-prefix: 'dev-' + + # uncomment this to configure an oidc user as admin user + # oidc-admin-user: user@yourdomain.org + profiles: + - name: default + image: datashield/rock-base:latest + port: 8085 + host: default + package-whitelist: + - dsBase + function-blacklist: [ ] + options: + datashield: + # the seed can only be 9 digits + seed: 342325352 + - name: xenon + image: datashield/rock-dolomite-xenon:latest + host: xenon + port: 8085 + package-whitelist: + - dsBase + - resourcer + function-blacklist: [ ] + options: + datashield: + # the seed can only be 9 digits + seed: 342325352 + - name: rock + image: datashield/rock-base:latest + host: rock + port: 8085 + package-whitelist: + - dsBase + - resourcer + function-blacklist: [ ] + options: + datashield: + # the seed can only be 9 digits + seed: 342325352 + +# required settings: +spring: + security: + user: + # please change this admin password! + password: admin + ## uncomment oauth2 settings below to enable oidcgit + # oauth2: + # client: + # provider: + # molgenis: + # issuer-uri: 'http://auth.molgenis.org' + # registration: + # molgenis: + # client-id: '...' + # client-secret: '...' + # resourceserver: + # jwt: + # issuer-uri: 'http://auth.molgenis.org' + # opaquetoken: + # client-id: '...' + + # optional settings (review spring handbook to find more): + servlet: + multipart: + ## change this if your files are bigger + max-file-size: 1000MB + max-request-size: 1000MB + +storage: + ## to change location of the data storage + root-dir: /data + +logging: + level: + root: INFO + ## change to DEBUG to have more details, typically when developing + org.molgenis: DEBUG + ## Don't log upload data + org.apache.coyote.http11.Http11InputBuffer: INFO diff --git a/docker/ci/armadillo-compose.md b/docker/ci/armadillo-compose.md new file mode 100644 index 000000000..70410697e --- /dev/null +++ b/docker/ci/armadillo-compose.md @@ -0,0 +1,54 @@ +# Using Armadillo compose + +To use this zip content you need to +- have Docker (desktop) installed +- being able to run `docker compose` from the terminal +- cd into the `docker-compose` directory + +## Start + +### On install and updates + +When for the first time with **current** zip file extraction. + +When having done changes in some locations like +- the `armadillo/` directory. If unsure remove `armadillo/data/system/profiles.json`. +- the `docker-compose.yml file` like adding/removing new services next to armadillo. + +Build or rebuild the Armadillo server as a docker image using: + +```sh +docker compose up build +``` + +### Every time + +Start the set of Armadillo and R images. + +```sh +docker compose up +``` + +This will show all messages from all running images. To stop you need to use `CTRL-C` or use the stop step below. + +If you don't want to see these `docker compose up --detach` + +## Stop + +Stop the set of Armadillo and R images. + +```sh +docker compose down +``` +or use `CTRL-C` + +## Misc + +To clean up you can: + +- list containers `docker container ls` +- remove a container `docker container rm ` +- list images `docker image ls` +- remove image `docker image rm ` + +Original file: git: docker/ci/armadillo-compose.md diff --git a/docker/ci/armadillo-ready.bash b/docker/ci/armadillo-ready.bash new file mode 100755 index 000000000..51a8b413c --- /dev/null +++ b/docker/ci/armadillo-ready.bash @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Default duration is 60 if not supplied +duration=${1:-60} +HOSTNAME=${2:-armadillo} + +for ((i=1; i<=duration; i++)) +do + echo "$i: is Armadillo up?" + if wget --spider "http://${HOSTNAME}:8080"; then + echo "Armadillo is up according to wget ..." + ./release-test.R || exit 1 + exit 0 + else + echo "Armadillo still down ..." + sleep 2 + fi +done +exit 1 + diff --git a/docker/ci/ci.env b/docker/ci/ci.env new file mode 100644 index 000000000..561e7ab39 --- /dev/null +++ b/docker/ci/ci.env @@ -0,0 +1,19 @@ +GIT_CLONE_PATH= + +TEST_FILE_PATH= + +INTERACTIVE=N + +AS_DOCKER_CONTAINER=Y + +SKIP_TESTS=Y +DO_TESTS= + +PROFILE=xenon + +ADMIN_PASSWORD=admin +ARMADILLO_URL=http://armadillo:8080 + +OIDC_EMAIL= + +TOKEN= diff --git a/docker/ci/docker-compose.yml b/docker/ci/docker-compose.yml new file mode 100644 index 000000000..10e9632a4 --- /dev/null +++ b/docker/ci/docker-compose.yml @@ -0,0 +1,41 @@ +version: "3.4" +services: + armadillo: + hostname: armadillo + ports: + - 8080:8080 + + build: + context: '.' + dockerfile: 'Dockerfile' + args: + JAR_FILE: '*.jar' + + image: molgenis/molgenis-armadillo:latest + environment: + LOGGING_CONFIG: 'classpath:logback-file.xml' + AUDIT_LOG_PATH: '/app/logs/audit.log' + SPRING_SECURITY_USER_PASSWORD: 'admin' + volumes: + - ${PWD}/armadillo/logs:/logs + - ${PWD}/armadillo/data:/data + - ${PWD}/armadillo/config:/config + - /var/run/docker.sock:/var/run/docker.sock + + default: + hostname: default + image: datashield/rock-base:latest + environment: + DEBUG: "TRUE" + + xenon: + hostname: xenon + image: datashield/rock-dolomite-xenon:latest + environment: + DEBUG: "TRUE" + + rock: + hostname: rock + image: datashield/rock-base:latest + environment: + DEBUG: "TRUE" diff --git a/scripts/release/dev.env.dist b/scripts/release/dev.env.dist index 318edeb13..9abd30d8a 100644 --- a/scripts/release/dev.env.dist +++ b/scripts/release/dev.env.dist @@ -4,6 +4,9 @@ # This is actually the parent of the checkout dir without /molgenis-service-armadillo GIT_CLONE_PATH= +# Is armadillo running in a docker container (default N) +AS_DOCKER_CONTAINER=N + # Path to test file directory (if not set and shared-lifecycle is not within the data directory, will create a "testing" folder in the data directory) # The RDA test file will also be placed in this path TEST_FILE_PATH= diff --git a/scripts/release/install_release_script_dependencies.R b/scripts/release/install_release_script_dependencies.R index 6369a0455..9b6f35cb5 100755 --- a/scripts/release/install_release_script_dependencies.R +++ b/scripts/release/install_release_script_dependencies.R @@ -25,7 +25,14 @@ if(!"cli" %in% installed.packages()) { library(cli) cli_alert_info("Installing packages") -packages <- c("getPass", "arrow", "jsonlite", "future", "MolgenisArmadillo", "DSI", "devtools", "resourcer", "DSMolgenisArmadillo", "RCurl") +packages <- c("diffobj", "getPass", "arrow", "jsonlite", "future", + "RCurl", + "devtools", + "DSI", + "resourcer", + "MolgenisArmadillo", + "DSMolgenisArmadillo" +) install_requirements_from_cran <- function(packages) { n_requirements <- length(packages) @@ -44,6 +51,8 @@ install_requirements_from_cran <- function(packages) { } install_requirements_from_cran(packages) +warnings() + library("devtools") install_github("datashield/dsBaseClient") #check if all packages are installed diff --git a/scripts/release/release-test.R b/scripts/release/release-test.R index 7d5a243de..509b28de6 100755 --- a/scripts/release/release-test.R +++ b/scripts/release/release-test.R @@ -491,6 +491,12 @@ if(url.exists(armadillo_url)) { exit_test(msg) } +as_docker_container <- FALSE +if("Y" == Sys.getenv("AS_DOCKER_CONTAINER", "N")) { + as_docker_container <- TRUE +} +cli_alert_info(sprintf("Running in docker container %d", as.integer(as_docker_container))) + service_location = remove_slash_if_added(Sys.getenv("GIT_CLONE_PATH")) if(service_location == ""){ cli_alert_warning("Git clone path not set, attempting to set git clone root through normalized path") @@ -607,10 +613,13 @@ if(profile == ""){ cli_alert_info("Checking if profile is prepared for all tests") -create_profile_if_not_available(profile, profiles$available, token, auth_type) +if (!as_docker_container) { + create_profile_if_not_available(profile, profiles$available, token, auth_type) +} profile_info <- get_from_api_with_header(paste0("ds-profiles/", profile), token, auth_type) -start_profile_if_not_running("default", token, auth_type) - +if (!as_docker_container) { + start_profile_if_not_running("default", token, auth_type) +} seed <- unlist(profile_info$options$datashield.seed) whitelist <- unlist(profile_info$packageWhitelist) if(is.null(seed)){ @@ -918,7 +927,9 @@ if(ADMIN_MODE) { armadillo.login(armadillo_url) } profile <- "rock" -create_profile_if_not_available(profile, profiles$available, token, auth_type) +if (!as_docker_container) { + create_profile_if_not_available(profile, profiles$available, token, auth_type) +} available_projects <- armadillo.list_projects() project3 <- generate_random_project_name(available_projects) available_projects <- c(available_projects, project3)