From d2044b86302efe1ee45d656381e1d4b70b70be34 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Mon, 16 Oct 2023 14:34:59 +0200 Subject: [PATCH 1/3] Remove build and publish base and base-with-services to registry f-m --- .github/workflows/docker.yml | 55 ++++++++---------------------------- 1 file changed, 12 insertions(+), 43 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1e710f5d..fda56b1c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -59,20 +59,6 @@ concurrency: cancel-in-progress: true jobs: - amd64-base: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - image: base - architecture: amd64 - runsOn: ubuntu-latest - - amd64-base-with-services: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - image: base-with-services - architecture: amd64 - runsOn: ubuntu-latest - needs: [amd64-base] amd64-lab: uses: ./.github/workflows/docker-build-test-upload.yml @@ -80,7 +66,6 @@ jobs: image: lab architecture: amd64 runsOn: ubuntu-latest - needs: [amd64-base] amd64-full-stack: uses: ./.github/workflows/docker-build-test-upload.yml @@ -88,14 +73,7 @@ jobs: image: full-stack architecture: amd64 runsOn: ubuntu-latest - needs: [amd64-base-with-services, amd64-lab] - - arm64-base: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - image: base - architecture: arm64 - runsOn: ARM64 + needs: [amd64-lab] arm64-lab: uses: ./.github/workflows/docker-build-test-upload.yml @@ -103,15 +81,6 @@ jobs: image: lab architecture: arm64 runsOn: ARM64 - needs: [arm64-base] - - arm64-base-with-services: - uses: ./.github/workflows/docker-build-test-upload.yml - with: - image: base-with-services - architecture: arm64 - runsOn: ARM64 - needs: [arm64-base] arm64-full-stack: uses: ./.github/workflows/docker-build-test-upload.yml @@ -119,13 +88,13 @@ jobs: image: full-stack architecture: arm64 runsOn: ARM64 - needs: [arm64-base-with-services, arm64-lab] + needs: [arm64-lab] amd64-push-ghcr: uses: ./.github/workflows/docker-push.yml strategy: matrix: - image: ["base", "base-with-services", "lab", "full-stack"] + image: ["lab", "full-stack"] with: architecture: amd64 image: ${{ matrix.image }} @@ -133,13 +102,13 @@ jobs: secrets: REGISTRY_USERNAME: ${{ github.actor }} REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} - needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack] + needs: [amd64-lab, amd64-full-stack] arm64-push-ghcr: uses: ./.github/workflows/docker-push.yml strategy: matrix: - image: ["base", "base-with-services", "lab", "full-stack"] + image: ["lab", "full-stack"] with: architecture: arm64 image: ${{ matrix.image }} @@ -147,13 +116,13 @@ jobs: secrets: REGISTRY_USERNAME: ${{ github.actor }} REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} - needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack] + needs: [arm64-lab, arm64-full-stack] merge-tags-ghcr: uses: ./.github/workflows/docker-merge-tags.yml strategy: matrix: - image: ["base", "base-with-services", "lab", "full-stack"] + image: ["lab", "full-stack"] with: image: ${{ matrix.image }} registry: ghcr.io @@ -167,7 +136,7 @@ jobs: uses: ./.github/workflows/docker-push.yml strategy: matrix: - image: ["base", "base-with-services", "lab", "full-stack"] + image: ["lab", "full-stack"] with: architecture: amd64 image: ${{ matrix.image }} @@ -175,14 +144,14 @@ jobs: secrets: REGISTRY_USERNAME: ${{ secrets.DOCKER_USERNAME }} REGISTRY_TOKEN: ${{ secrets.DOCKER_PASSWORD }} - needs: [amd64-base, amd64-base-with-services, amd64-lab, amd64-full-stack] + needs: [amd64-lab, amd64-full-stack] arm64-push-dockerhub: if: github.repository == 'aiidalab/aiidalab-docker-stack' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) uses: ./.github/workflows/docker-push.yml strategy: matrix: - image: ["base", "base-with-services", "lab", "full-stack"] + image: ["lab", "full-stack"] with: architecture: arm64 image: ${{ matrix.image }} @@ -190,14 +159,14 @@ jobs: secrets: REGISTRY_USERNAME: ${{ secrets.DOCKER_USERNAME }} REGISTRY_TOKEN: ${{ secrets.DOCKER_PASSWORD }} - needs: [arm64-base, arm64-base-with-services, arm64-lab, arm64-full-stack] + needs: [arm64-lab, arm64-full-stack] merge-tags-dockerhub: if: github.repository == 'aiidalab/aiidalab-docker-stack' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) uses: ./.github/workflows/docker-merge-tags.yml strategy: matrix: - image: ["base", "base-with-services", "lab", "full-stack"] + image: ["lab", "full-stack"] with: image: ${{ matrix.image }} registry: docker.io From b80c7fc3b4ebdefd1a87f34a8809a22a79d2b3fa Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Mon, 16 Oct 2023 14:40:54 +0200 Subject: [PATCH 2/3] README update --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f415df63..62b30b07 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,7 @@ This repository contains the Dockerfiles for the official AiiDAlab docker image All images are based on the [jupyter/minimal-notebook](https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html#jupyter-minimal-notebook). Image variants: -- `base` – A minimal image that comes with AiiDA pre-installed and an AiiDA profile set up. -- `base-with-services` – Like `base`, but AiiDA services (PostgreSQL and RabbitMQ) are installed on the container and automatically launched on startup. -- `lab` – Like `base`, but uses the AiiDAlab home app as the primary interface (the standard JupyterLab interface is also available). +- `lab` – A minimal image that comes with AiiDA pre-installed and an AiiDA profile setup, uses the AiiDAlab home app as the primary interface (the standard JupyterLab interface is also available). - `full-stack` – Our most comprehensive image, like `lab`, but also comes with services pre-installed and launched. Supported tags (released on [Docker Hub](https://hub.docker.com/r/aiidalab)): @@ -55,7 +53,7 @@ The build system will attempt to detect the local architecture and automatically All commands `build`, `tests`, and `up` will use the locally detected platform and use a version tag based on the state of the local git repository. However, you can also specify a custom platform or version with the `--platform` and `--version` parameters, example: `doit build --arch=arm64 --version=my-version`. -You can specify target stacks to build with `--target`, example: `doit build --target base --target full-stack`. +You can specify target stacks to build with `--target`, example: `doit build --target lab --target full-stack`. ### Run automated tests From fe44d7da075b672a05140d73d8c45a6d59f8969e Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Mon, 16 Oct 2023 14:47:01 +0200 Subject: [PATCH 3/3] Rearrange the unit tests --- stack/docker-compose.base-with-services.yml | 19 -------- stack/docker-compose.base.yml | 48 ------------------ tests/conftest.py | 2 +- tests/test-base-with-services.py | 21 -------- tests/test-base.py | 53 -------------------- tests/test-common.py | 54 ++++++++++++++++++++- tests/test-full-stack.py | 22 +++++++++ 7 files changed, 76 insertions(+), 143 deletions(-) delete mode 100644 stack/docker-compose.base-with-services.yml delete mode 100644 stack/docker-compose.base.yml delete mode 100644 tests/test-base-with-services.py delete mode 100644 tests/test-base.py diff --git a/stack/docker-compose.base-with-services.yml b/stack/docker-compose.base-with-services.yml deleted file mode 100644 index 9074a8e7..00000000 --- a/stack/docker-compose.base-with-services.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -version: '3.4' - -services: - - aiidalab: - image: ${REGISTRY:-}${BASE_WITH_SERVICES_IMAGE:-aiidalab/base-with-services}:${VERSION:-newly-build} - environment: - TZ: Europe/Zurich - DOCKER_STACKS_JUPYTER_CMD: notebook - SETUP_DEFAULT_AIIDA_PROFILE: 'true' - AIIDALAB_DEFAULT_APPS: '' - volumes: - - aiidalab-home-folder:/home/jovyan - ports: - - "0.0.0.0:${AIIDALAB_PORT:-}:8888" - -volumes: - aiidalab-home-folder: diff --git a/stack/docker-compose.base.yml b/stack/docker-compose.base.yml deleted file mode 100644 index 3888e526..00000000 --- a/stack/docker-compose.base.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -version: '3.4' - -services: - - database: - image: postgres:12.3 - environment: - POSTGRES_USER: pguser - POSTGRES_PASSWORD: password - volumes: - - aiida-postgres-db:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER}"] - interval: 5s - timeout: 5s - retries: 10 - - messaging: - image: rabbitmq:3.8.3-management - environment: - RABBITMQ_DEFAULT_USER: guest - RABBITMQ_DEFAULT_PASS: guest - volumes: - - aiida-rmq-data:/var/lib/rabbitmq/ - - aiidalab: - image: ${REGISTRY:-}${BASE_IMAGE:-aiidalab/base}:${VERSION:-newly-build} - environment: - RMQHOST: messaging - TZ: Europe/Zurich - DOCKER_STACKS_JUPYTER_CMD: notebook - SETUP_DEFAULT_AIIDA_PROFILE: 'true' - AIIDALAB_DEFAULT_APPS: '' - volumes: - - aiidalab-home-folder:/home/jovyan - depends_on: - database: - condition: service_healthy - messaging: - condition: service_started - ports: - - "0.0.0.0:${AIIDALAB_PORT:-}:8888" - -volumes: - aiida-postgres-db: - aiida-rmq-data: - aiidalab-home-folder: diff --git a/tests/conftest.py b/tests/conftest.py index e8cf4799..636a3829 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,7 +20,7 @@ def pytest_addoption(parser): parser.addoption( "--variant", action="store", - default="base", + default="lab", help="Variant (image name) of the docker-compose file to use.", ) diff --git a/tests/test-base-with-services.py b/tests/test-base-with-services.py deleted file mode 100644 index 38f6b18a..00000000 --- a/tests/test-base-with-services.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Services related tests.""" -import json -from packaging.version import parse - - -def test_correct_pgsql_version_installed(aiidalab_exec, pgsql_version): - info = json.loads( - aiidalab_exec( - "mamba list -n aiida-core-services --json --full-name postgresql" - ).decode() - )[0] - assert info["name"] == "postgresql" - assert parse(info["version"]).major == parse(pgsql_version).major - - -def test_rabbitmq_can_start(aiidalab_exec): - """Test the rabbitmq-server can start, the output should be empty if - the command is successful.""" - output = aiidalab_exec("mamba run -n aiida-core-services rabbitmq-server -detached") - - assert output == b"" diff --git a/tests/test-base.py b/tests/test-base.py deleted file mode 100644 index 963fadcf..00000000 --- a/tests/test-base.py +++ /dev/null @@ -1,53 +0,0 @@ -"""This module contains tests for the base image, which are AiiDA and package management related tests.""" -import pytest -import json -from packaging.version import parse - - -@pytest.mark.parametrize("incompatible_version", ["1.6.3"]) -def test_prevent_pip_install_of_incompatible_aiida_version( - aiidalab_exec, nb_user, aiida_version, incompatible_version -): - package_manager = "pip" - assert parse(aiida_version) != parse(incompatible_version) - # Expected to succeed: - aiidalab_exec( - f"{package_manager} install aiida-core=={aiida_version}", user=nb_user - ) - with pytest.raises(Exception): - aiidalab_exec( - f"{package_manager} install aiida-core=={incompatible_version}", - user=nb_user, - ) - - -def test_correct_python_version_installed(aiidalab_exec, python_version): - info = json.loads(aiidalab_exec("mamba list --json --full-name python").decode())[0] - assert info["name"] == "python" - assert parse(info["version"]) == parse(python_version) - - -def test_create_conda_environment(aiidalab_exec, nb_user): - output = aiidalab_exec("conda create -y -n tmp", user=nb_user).decode().strip() - assert "conda activate tmp" in output - # New conda environments should be created in ~/.conda/envs/ - output = aiidalab_exec("conda env list", user=nb_user).decode().strip() - assert f"/home/{nb_user}/.conda/envs/tmp" in output - - -def test_pip_check(aiidalab_exec): - aiidalab_exec("pip check") - - -def test_correct_aiida_version_installed(aiidalab_exec, aiida_version): - info = json.loads( - aiidalab_exec("mamba list --json --full-name aiida-core").decode() - )[0] - assert info["name"] == "aiida-core" - assert parse(info["version"]) == parse(aiida_version) - - -def test_path_local_pip(aiidalab_exec, nb_user): - """test that the pip local bin path ~/.local/bin is added to PATH""" - output = aiidalab_exec("bash -c 'echo \"${PATH}\"'", user=nb_user).decode() - assert f"/home/{nb_user}/.local/bin" in output diff --git a/tests/test-common.py b/tests/test-common.py index a5ce02b8..29daf6a8 100644 --- a/tests/test-common.py +++ b/tests/test-common.py @@ -1,5 +1,57 @@ -"""Tests for all images, which are docker/docker-compose related tests.""" +"""Tests for all images, which are docker/docker-compose related and AiiDA/package management related tests.""" import requests +import pytest +import json +from packaging.version import parse + + +@pytest.mark.parametrize("incompatible_version", ["1.6.3"]) +def test_prevent_pip_install_of_incompatible_aiida_version( + aiidalab_exec, nb_user, aiida_version, incompatible_version +): + package_manager = "pip" + assert parse(aiida_version) != parse(incompatible_version) + # Expected to succeed: + aiidalab_exec( + f"{package_manager} install aiida-core=={aiida_version}", user=nb_user + ) + with pytest.raises(Exception): + aiidalab_exec( + f"{package_manager} install aiida-core=={incompatible_version}", + user=nb_user, + ) + + +def test_correct_python_version_installed(aiidalab_exec, python_version): + info = json.loads(aiidalab_exec("mamba list --json --full-name python").decode())[0] + assert info["name"] == "python" + assert parse(info["version"]) == parse(python_version) + + +def test_create_conda_environment(aiidalab_exec, nb_user): + output = aiidalab_exec("conda create -y -n tmp", user=nb_user).decode().strip() + assert "conda activate tmp" in output + # New conda environments should be created in ~/.conda/envs/ + output = aiidalab_exec("conda env list", user=nb_user).decode().strip() + assert f"/home/{nb_user}/.conda/envs/tmp" in output + + +def test_pip_check(aiidalab_exec): + aiidalab_exec("pip check") + + +def test_correct_aiida_version_installed(aiidalab_exec, aiida_version): + info = json.loads( + aiidalab_exec("mamba list --json --full-name aiida-core").decode() + )[0] + assert info["name"] == "aiida-core" + assert parse(info["version"]) == parse(aiida_version) + + +def test_path_local_pip(aiidalab_exec, nb_user): + """test that the pip local bin path ~/.local/bin is added to PATH""" + output = aiidalab_exec("bash -c 'echo \"${PATH}\"'", user=nb_user).decode() + assert f"/home/{nb_user}/.local/bin" in output def test_notebook_service_available(notebook_service): diff --git a/tests/test-full-stack.py b/tests/test-full-stack.py index a0d7f828..933c55ef 100644 --- a/tests/test-full-stack.py +++ b/tests/test-full-stack.py @@ -1,5 +1,27 @@ +"""Services related test and package integration tests by installing apps from app store.""" import pytest +import json +from packaging.version import parse + + +def test_correct_pgsql_version_installed(aiidalab_exec, pgsql_version): + info = json.loads( + aiidalab_exec( + "mamba list -n aiida-core-services --json --full-name postgresql" + ).decode() + )[0] + assert info["name"] == "postgresql" + assert parse(info["version"]).major == parse(pgsql_version).major + + +def test_rabbitmq_can_start(aiidalab_exec): + """Test the rabbitmq-server can start, the output should be empty if + the command is successful.""" + output = aiidalab_exec("mamba run -n aiida-core-services rabbitmq-server -detached") + + assert output == b"" + @pytest.fixture(scope="function") def generate_aiidalab_install_output(aiidalab_exec, nb_user):