diff --git a/.github/workflows/build_and_push_all_images.yml b/.github/workflows/build_and_push_all_images.yml new file mode 100644 index 0000000..8ace010 --- /dev/null +++ b/.github/workflows/build_and_push_all_images.yml @@ -0,0 +1,147 @@ +name: Build and push all images + +on: + workflow_call: + secrets: + GH_TOKEN: + required: true + inputs: + force_build: + description: If the build should be forced even if there are no changes + type: boolean + required: false + default: false + push: + description: Push the image to the registry + type: boolean + required: false + default: false + registry: + description: Container registry to use + type: string + required: false + default: ghcr.io + tag: + description: Tag to use for the images (e.g. latest, dev) + type: string + required: false + default: latest + e2xgrader_installation_source: + description: Where to install e2xgrader from + type: string + required: false + default: pypi + e2xgrader_version: + description: Use a specific e2xgrader version from PyPi + type: string + required: false + default: "" + e2xgrader_branch: + description: Which e2xgrader branch or tag to install from. Only takes effect if e2xgrader_installation_source is "github" + type: string + required: false + default: "main" + +jobs: + + minimal_notebook: + uses: ./.github/workflows/build_image.yml + with: + force_build: ${{ inputs.force_build }} + image_name: minimal-notebook + registry: ${{ inputs.registry }} + base_image_name: jupyter/minimal-notebook + base_image_tag: 4d70cf8da953 + image_tag: ${{ inputs.tag }} + push: ${{ inputs.push }} + secrets: inherit + + datascience_notebook: + needs: [minimal_notebook] + uses: ./.github/workflows/build_image.yml + with: + force_build: ${{ inputs.force_build || needs.minimal_notebook.outputs.did_build_image == 'true' }} + image_name: datascience-notebook + image_tag: ${{ inputs.tag }} + registry: ${{ inputs.registry }} + base_image_name: ${{ inputs.registry }}/digiklausur/docker-stacks/minimal-notebook + base_image_tag: ${{ inputs.tag }} + push: ${{ inputs.push }} + secrets: inherit + + e2x_datascience_notebook: + needs: [datascience_notebook] + uses: ./.github/workflows/build_e2xgrader_images.yml + with: + force_build: ${{ inputs.force_build || needs.datascience_notebook.outputs.did_build_image == 'true' }} + registry: ${{ inputs.registry }} + image_name: datascience-notebook + image_tag: ${{ inputs.tag }} + base_image_name: ${{ inputs.registry }}/digiklausur/docker-stacks/datascience-notebook + base_image_tag: ${{ inputs.tag }} + push: ${{ inputs.push }} + e2xgrader_installation_source: ${{ inputs.e2xgrader_installation_source }} + e2xgrader_version: ${{ inputs.e2xgrader_version }} + e2xgrader_branch: ${{ inputs.e2xgrader_branch }} + secrets: inherit + + ml_notebook: + needs: [datascience_notebook] + uses: ./.github/workflows/build_image.yml + with: + force_build: ${{ inputs.force_build || needs.datascience_notebook.outputs.did_build_image == 'true' }} + image_name: ml-notebook + image_tag: ${{ inputs.tag }} + registry: ${{ inputs.registry }} + base_image_name: ${{ inputs.registry }}/digiklausur/docker-stacks/datascience-notebook + base_image_tag: ${{ inputs.tag }} + push: ${{ inputs.push }} + secrets: inherit + + e2x_ml_notebook: + needs: [ml_notebook] + uses: ./.github/workflows/build_e2xgrader_images.yml + with: + force_build: ${{ inputs.force_build || needs.ml_notebook.outputs.did_build_image == 'true' }} + registry: ${{ inputs.registry }} + image_name: ml-notebook + image_tag: ${{ inputs.tag }} + base_image_name: ${{ inputs.registry }}/digiklausur/docker-stacks/ml-notebook + base_image_tag: ${{ inputs.tag }} + push: ${{ inputs.push }} + e2xgrader_installation_source: ${{ inputs.e2xgrader_installation_source }} + e2xgrader_version: ${{ inputs.e2xgrader_version }} + e2xgrader_branch: ${{ inputs.e2xgrader_branch }} + secrets: inherit + + nlp_notebook: + needs: [ml_notebook] + uses: ./.github/workflows/build_image.yml + with: + force_build: ${{ inputs.force_build || needs.ml_notebook.outputs.did_build_image == 'true' }} + image_name: nlp-notebook + image_tag: ${{ inputs.tag }} + registry: ${{ inputs.registry }} + base_image_name: ${{ inputs.registry }}/digiklausur/docker-stacks/ml-notebook + base_image_tag: ${{ inputs.tag }} + push: ${{ inputs.push }} + secrets: inherit + + e2x_nlp_notebook: + needs: [nlp_notebook] + uses: ./.github/workflows/build_e2xgrader_images.yml + with: + force_build: ${{ inputs.force_build || needs.nlp_notebook.outputs.did_build_image == 'true' }} + registry: ${{ inputs.registry }} + image_name: nlp-notebook + image_tag: ${{ inputs.tag }} + base_image_name: ${{ inputs.registry }}/digiklausur/docker-stacks/nlp-notebook + base_image_tag: ${{ inputs.tag }} + push: ${{ inputs.push }} + e2xgrader_installation_source: ${{ inputs.e2xgrader_installation_source }} + e2xgrader_version: ${{ inputs.e2xgrader_version }} + e2xgrader_branch: ${{ inputs.e2xgrader_branch }} + secrets: inherit + + + diff --git a/.github/workflows/build_and_push_to_registry.yml b/.github/workflows/build_and_push_to_registry.yml new file mode 100644 index 0000000..7108b24 --- /dev/null +++ b/.github/workflows/build_and_push_to_registry.yml @@ -0,0 +1,37 @@ +name: Test build and push to ghcr.io and quay.io + +on: + push: + branches: + - refactor_workflows_and_images + paths: + - .github/workflows/build_and_push_all_images.yml + - .github/workflows/build_and_push_to_registry.yml + - .github/workflows/build_e2xgrader_images.yml + - .github/workflows/build_image.yml + - images/** + +jobs: + + build_and_push_to_ghcr: + name: ghcr.io + uses: ./.github/workflows/build_and_push_all_images.yml + with: + force_build: false + push: true + registry: ghcr.io + tag: test + e2xgrader_installation_source: pypi + secrets: inherit + + build_and_push_to_quay: + name: quay.io + uses: ./.github/workflows/build_and_push_all_images.yml + with: + force_build: false + push: true + registry: quay.io + tag: test + e2xgrader_installation_source: pypi + secrets: inherit + diff --git a/.github/workflows/build_e2xgrader_images.yml b/.github/workflows/build_e2xgrader_images.yml new file mode 100644 index 0000000..115bf11 --- /dev/null +++ b/.github/workflows/build_e2xgrader_images.yml @@ -0,0 +1,129 @@ +name: Build e2xgrader image for all modes + +on: + workflow_call: + secrets: + GH_TOKEN: + required: true + QUAY_USERNAME: + required: true + QUAY_SECRET: + required: true + inputs: + force_build: + description: Force the build even if there are no changes + type: boolean + required: false + default: false + base_image_name: + description: The name of the base image to build from + type: string + required: true + base_image_tag: + description: The tag of the base image to build from + type: string + required: false + default: latest + image_name: + description: The name of the image we build. + type: string + required: true + image_tag: + description: The tag of the image we build. E.g. latest + type: string + required: false + default: latest + e2xgrader_installation_source: + description: Where to install e2xgrader from + type: string + required: false + default: pypi + e2xgrader_version: + description: Use a specific e2xgrader version from PyPi + type: string + required: false + default: "" + e2xgrader_branch: + description: Which e2xgrader branch or tag to install from. Only takes effect if e2xgrader_installation_source is "github" + type: string + required: false + default: "main" + push: + description: Push the image to the registry + type: boolean + required: false + default: false + registry: + description: Container registry to use + type: string + required: false + default: ghcr.io + +jobs: + build: + name: ${{ inputs.image_name }}:${{ inputs.image_tag }} (${{ matrix.e2xgrader_mode }}) + strategy: + matrix: + e2xgrader_mode: [student, student_exam, teacher] + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Set image tag + id: set_image_tag + run: | + if [ "${{ matrix.e2xgrader_mode }}" == "student_exam" ]; then + echo "image_tag=${{ inputs.image_name }}-exam:${{ inputs.image_tag }}" >> $GITHUB_OUTPUT + else + echo "image_tag=${{ inputs.image_name }}-${{ matrix.e2xgrader_mode }}:${{ inputs.image_tag }}" >> $GITHUB_OUTPUT + fi + - name: Print image tag + id: print_image_tag + run: | + echo ${{ steps.set_image_tag.outputs.image_tag }} + - name: Check if files changed + if: ${{ ! inputs.force_build }} + uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + image_changed: + images/e2xgrader-notebook/** + - name: Login to ghcr.io + if: ${{ inputs.registry == 'ghcr.io' }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GH_TOKEN }} + - name: Login to Quay Container Registry + if: ${{ inputs.registry == 'quay.io' }} + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_SECRET }} + - name: Build Docker image + if: ${{ inputs.force_build || steps.changes.outputs.image_changed == 'true' }} + id: build_image + uses: docker/build-push-action@v5 + with: + context: images/e2xgrader-notebook + push: ${{ inputs.push }} + tags: ${{ inputs.registry }}/digiklausur/docker-stacks/${{ steps.set_image_tag.outputs.image_tag }} + build-args: | + IMAGE_SOURCE=${{ inputs.base_image_name }}:${{ inputs.base_image_tag }} + E2XGRADER_MODE=${{ matrix.e2xgrader_mode }} + E2XGRADER_VERSION=${{ inputs.e2xgrader_version }} + FROM_REPO=${{ inputs.e2xgrader_installation_source == 'github' }} + E2XGRADER_BRANCH=${{ inputs.e2xgrader_branch }} + - name: List images + if: ${{ inputs.force_build || steps.changes.outputs.image_changed == 'true' }} + run: | + docker images + + + + + + diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml new file mode 100644 index 0000000..f769ec8 --- /dev/null +++ b/.github/workflows/build_image.yml @@ -0,0 +1,100 @@ +run-name: Build ${{ inputs.image_name }}:${{ inputs.image_tag }} + +on: + workflow_call: + secrets: + GH_TOKEN: + required: true + QUAY_USERNAME: + required: true + QUAY_SECRET: + required: true + inputs: + force_build: + description: If the build should be forced even if there are no changes + type: boolean + required: false + default: false + push: + description: Push the image to the registry + type: boolean + required: false + default: false + registry: + description: Container registry to use + type: string + required: false + default: ghcr.io + base_image_name: + description: The name of the base image to build from + type: string + required: true + base_image_tag: + description: The tag of the base image to build from + type: string + required: false + default: latest + image_name: + description: The name of the image to build. + type: string + required: true + image_tag: + description: The tag of the image to build. E.g. latest + type: string + required: false + default: latest + outputs: + did_build_image: + description: If the image was built + value: ${{ jobs.build_and_push_image.outputs.did_build_image }} + +jobs: + build_and_push_image: + name: ${{ inputs.image_name }}:${{ inputs.image_tag }} + runs-on: ubuntu-latest + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Print the base image + run: | + echo "The base image to build from is ${{ inputs.base_image_name }}:${{ inputs.base_image_tag }}" + - name: Check if files changed + if: ${{ ! inputs.force_build }} + uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + image_changed: + images/${{ inputs.image_name }}/** + - name: Login to ghcr.io + if: ${{ inputs.registry == 'ghcr.io' }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GH_TOKEN }} + - name: Login to Quay Container Registry + if: ${{ inputs.registry == 'quay.io' }} + uses: docker/login-action@v3 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_SECRET }} + - name: Build Docker image + if: ${{ inputs.force_build || steps.changes.outputs.image_changed == 'true' }} + id: build_image + uses: docker/build-push-action@v5 + with: + context: images/${{ inputs.image_name }} + push: ${{ inputs.push }} + tags: ${{ inputs.registry }}/digiklausur/docker-stacks/${{ inputs.image_name }}:${{ inputs.image_tag }} + build-args: | + IMAGE_SOURCE=${{ inputs.base_image_name }}:${{ inputs.base_image_tag }} + - name: List images + if: ${{ inputs.force_build || steps.changes.outputs.image_changed == 'true' }} + run: | + docker images + - name: Set output (image was built) + if: ${{ inputs.force_build || steps.changes.outputs.image_changed == 'true' }} + run: | + echo "image_built=true" >> $GITHUB_OUTPUT diff --git a/.github/workflows/docker_dev.yml b/.github/workflows/docker_dev.yml new file mode 100644 index 0000000..17552f5 --- /dev/null +++ b/.github/workflows/docker_dev.yml @@ -0,0 +1,37 @@ +name: Build and push dev images to ghcr.io and quay.io + +on: + push: + branches: + - dev + paths: + - .github/workflows/docker_dev.yml + - .github/workflows/build_and_push_all_images.yml + - .github/workflows/build_e2xgrader_images.yml + - .github/workflows/build_image.yml + - images/** + +jobs: + + build_and_push_to_ghcr: + name: ghcr.io + uses: ./.github/workflows/build_and_push_all_images.yml + with: + force_build: false + push: true + registry: ghcr.io + tag: dev + e2xgrader_installation_source: pypi + secrets: inherit + + build_and_push_to_quay: + name: quay.io + uses: ./.github/workflows/build_and_push_all_images.yml + with: + force_build: false + push: true + registry: quay.io + tag: dev + e2xgrader_installation_source: pypi + secrets: inherit + diff --git a/.github/workflows/docker_main.yml b/.github/workflows/docker_main.yml new file mode 100644 index 0000000..b205a47 --- /dev/null +++ b/.github/workflows/docker_main.yml @@ -0,0 +1,37 @@ +name: Build and push latest images to ghcr.io and quay.io + +on: + push: + branches: + - master + paths: + - .github/workflows/docker_main.yml + - .github/workflows/build_and_push_all_images.yml + - .github/workflows/build_e2xgrader_images.yml + - .github/workflows/build_image.yml + - images/** + +jobs: + + build_and_push_to_ghcr: + name: ghcr.io + uses: ./.github/workflows/build_and_push_all_images.yml + with: + force_build: false + push: true + registry: ghcr.io + tag: latest + e2xgrader_installation_source: pypi + secrets: inherit + + build_and_push_to_quay: + name: quay.io + uses: ./.github/workflows/build_and_push_all_images.yml + with: + force_build: false + push: true + registry: quay.io + tag: latest + e2xgrader_installation_source: pypi + secrets: inherit + diff --git a/.github/workflows/manual-workflow.yml b/.github/workflows/manual-workflow.yml deleted file mode 100644 index 3eaad52..0000000 --- a/.github/workflows/manual-workflow.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Manual workflow -on: - workflow_dispatch: - inputs: - branch: - description: 'Branch to build' - required: true - default: 'dev' - image: - descriptiion: 'Image to build' - required: true - default: 'all' - -jobs: - deploy: - name: Deploy - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GH_TOKEN }} - - - name: Build and publish to ghcr.io - id: build_and_deploy - run: | - BRANCH=${{ needs.event.inputs.branch }} - IMAGE=${{ needs.event.inputs.image }} - - if [ "$BRANCH" == "dev" ]; then - bash ci/build-and-deploy.sh --deployment dev --registry ghcr.io --image $IMAGE --publish latest - elif [ "$BRANCH" == "master" ]; then - bash ci/build-and-deploy.sh --deployment prod --registry ghcr.io --image $IMAGE --publish latest - else - echo "Branch $BRANCH unknown" - exit 1 - fi - - - name: Login to Quay Container Registry - uses: docker/login-action@v1 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_SECRET }} - - - name: Build and publish to quay.io - id: build_and_deploy_to_quay - run: | - BRANCH=${{ needs.event.inputs.branch }} - IMAGE=${{ needs.event.inputs.image }} - - if [ "$BRANCH" == "dev" ]; then - bash ci/build-and-deploy.sh --deployment dev --registry ghcr.io --image $IMAGE --publish latest - elif [ "$BRANCH" == "master" ]; then - bash ci/build-and-deploy.sh --deployment prod --registry ghcr.io --image $IMAGE --publish latest - else - echo "Branch $BRANCH unknown" - exit 1 - fi diff --git a/.github/workflows/ngshare.yml b/.github/workflows/ngshare.yml deleted file mode 100644 index 9833005..0000000 --- a/.github/workflows/ngshare.yml +++ /dev/null @@ -1,82 +0,0 @@ ---- -name: Ngshare -on: - push: - branches: - - master - - dev - paths: - - .github/** - - hub/** - - ci/** - -jobs: - build-and-publish-dev: - name: Deploy Dev - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/dev' - steps: - - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GH_TOKEN }} - - - name: Build and publish to ghcr.io - id: build_and_deploy_to_ghcr - run: | - bash ci/build-and-deploy.sh --deployment dev --registry ghcr.io --image ngshare --publish latest - - - name: Login to Quay Container Registry - uses: docker/login-action@v1 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_SECRET }} - - - name: Build and publish to quay.io - id: build_and_deploy_to_quay - run: | - bash ci/build-and-deploy.sh --deployment dev --registry quay.io --image ngshare --publish latest - - build-and-publish-prod: - name: Deploy Prod - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - steps: - - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GH_TOKEN }} - - - name: Build and publish to ghcr.io - id: build_and_deploy_to_ghcr - run: | - bash ci/build-and-deploy.sh --deployment prod --registry ghcr.io --image ngshare --publish latest - - - name: Login to Quay Container Registry - uses: docker/login-action@v1 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_SECRET }} - - - name: Build and publish to ghcr.io - id: build_and_deploy_to_quay - run: | - bash ci/build-and-deploy.sh --deployment prod --registry quay.io --image ngshare --publish latest diff --git a/.github/workflows/notebook-dev.yml b/.github/workflows/notebook-dev.yml deleted file mode 100644 index 6e8e90c..0000000 --- a/.github/workflows/notebook-dev.yml +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: Notebook Dev -on: - push: - branches: - - dev - paths: - - .github/** - - ci/** - - minimal-notebook/** - - datascience-notebook/** - - notebook/** - - exam-notebook/** - -jobs: - notebook: - name: Notebook - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/dev' - steps: - - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GH_TOKEN }} - - - name: Build and publish to ghcr.io - id: build_and_publish_to_ghcr - run: | - bash ci/build-and-deploy.sh --deployment dev --registry ghcr.io --image all-notebook --publish latest - - - name: Login to Quay Container Registry - uses: docker/login-action@v1 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_SECRET }} - - - name: Build and publish to quay.io - id: build_and_publish_to_quay - run: | - bash ci/build-and-deploy.sh --deployment dev --registry quay.io --image all-notebook --publish latest - diff --git a/.github/workflows/notebook-stable.yml b/.github/workflows/notebook-stable.yml deleted file mode 100644 index 5b9534a..0000000 --- a/.github/workflows/notebook-stable.yml +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: Notebook Stable -on: - push: - branches: - - master - paths: - - .github/** - - ci/** - - minimal-notebook/** - - datascience-notebook/** - - notebook/** - - exam-notebook/** - -jobs: - notebook: - name: Notebook - runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - steps: - - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GH_TOKEN }} - - - name: Build and publish to ghcr.io - id: build_and_publish_to_ghcr - run: | - bash ci/build-and-deploy.sh --deployment prod --registry ghcr.io --image all-notebook --publish latest - - - name: Login to Quay Container Registry - uses: docker/login-action@v1 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_SECRET }} - - - name: Build and publish to quay.io - id: build_and_publish_to_quay - run: | - bash ci/build-and-deploy.sh --deployment prod --registry quay.io --image all-notebook --publish latest - diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml deleted file mode 100644 index 18e1d2d..0000000 --- a/.github/workflows/pull-request.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -name: Pull Request -on: - pull_request: - branches: - - master - - dev - -jobs: - build: - name: Build - runs-on: ubuntu-latest - if: github.event_name == 'pull_request' - - steps: - - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - - name: Build image - id: build_image - run: | - bash ci/build-and-deploy.sh --deployment dev --registry ghcr.io --image all --publish none diff --git a/.github/workflows/tags.yml b/.github/workflows/tags.yml deleted file mode 100644 index 915f1b8..0000000 --- a/.github/workflows/tags.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -name: Deploy tags -on: - push: - tags: - - 'ws*' - - 'ss*' - - 'release-*' - -jobs: - build: - name: Build - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GH_TOKEN }} - - - name: Get the version - id: get_version - run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} - - - name: Build and publis to ghcr.io - id: docker_build_publish_ghcr - run: | - bash ci/build-and-deploy.sh --deployment prod --registry ghcr.io --image all --version ${{ steps.get_version.outputs.VERSION }} --publish version - - - name: Login to Quay Container Registry - uses: docker/login-action@v1 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_SECRET }} - - - name: Build and publis to quay.io - id: docker_build_publish_quay - run: | - bash ci/build-and-deploy.sh --deployment prod --registry quay.io --image all --version ${{ steps.get_version.outputs.VERSION }} --publish version diff --git a/.github/workflows/trigger_build_and_push.yml b/.github/workflows/trigger_build_and_push.yml new file mode 100644 index 0000000..a8b89c1 --- /dev/null +++ b/.github/workflows/trigger_build_and_push.yml @@ -0,0 +1,60 @@ +name: Manually build and push all images + +on: + workflow_dispatch: + inputs: + force_build: + description: If the build should be forced even if there are no changes + type: boolean + required: false + default: false + push: + description: Push the image to the registry + type: boolean + required: false + default: false + registry: + description: Container registry to use + type: choice + required: false + default: ghcr.io + options: + - ghcr.io + - quay.io + tag: + description: Tag to use for the images (e.g. latest, dev) + type: string + required: false + default: latest + e2xgrader_installation_source: + description: Where to install e2xgrader from + type: choice + required: false + default: pypi + options: + - pypi + - github + e2xgrader_version: + description: Use a specific e2xgrader version from PyPi + type: string + required: false + default: "" + e2xgrader_branch: + description: Which e2xgrader branch or tag to install from. Only takes effect if e2xgrader_installation_source is "github" + type: string + required: false + default: "main" + +jobs: + + manually_build_all_images: + uses: ./.github/workflows/build_and_push_all_images.yml + with: + force_build: ${{ inputs.force_build }} + push: ${{ inputs.push }} + registry: ${{ inputs.registry }} + tag: ${{ inputs.tag }} + e2xgrader_installation_source: ${{ inputs.e2xgrader_installation_source }} + e2xgrader_version: ${{ inputs.e2xgrader_version }} + e2xgrader_branch: ${{ inputs.e2xgrader_branch }} + diff --git a/Makefile b/Makefile deleted file mode 100644 index 4df3d9e..0000000 --- a/Makefile +++ /dev/null @@ -1,27 +0,0 @@ -ifeq ($(CONTAINER_REG_OWNER),) -$(info ERROR! container regisrty and owner is not set) -$(info example for docker hub registry/owner) -$(info CONTAINER_REG_OWNER=registry.hub.docker.com/digiklausur) -$(error CONTAINER_REG_OWNER is not set) -endif - -ifeq ($(IMAGE),) -$(info ERROR!) -$(info Specify image, e.g. notebook, minimal-notebook) -$(error IMAGE is not set) -endif - -ifeq ($(VERSION),) -$(info VERSION is not set) -VERSION=$(shell git rev-parse --short HEAD) -$(info using verison --> $(VERSION)) -else -$(info using version --> $(VERSION)) -endif - -build: - docker build -t $(CONTAINER_REG_OWNER)/$(IMAGE):$(VERSION) $(IMAGE) - -push: - docker push $(CONTAINER_REG_OWNER)/$(IMAGE):$(VERSION) $(IMAGE) - diff --git a/datascience-notebook/requirements.txt b/datascience-notebook/requirements.txt deleted file mode 100644 index d21e30c..0000000 --- a/datascience-notebook/requirements.txt +++ /dev/null @@ -1,40 +0,0 @@ -bokeh -colorama -easydict==1.9 -gensim==4.3.1 -graphviz -gymnasium==0.29.1 -imageio==2.31.1 -ipycanvas -ipysheet -ipywidgets -itables -matplotlib==3.7.2 -nltk==3.8 -numpy -nx -opencv-contrib-python==4.8.0.76 -pandas==2.0.3 -pdfkit -Pillow==10.0.0 -pyagrum -pydantic -pydantic_yaml -pydot -pyparsing>=3.0.0 -pypdf2 -python-sat[aiger,approxmc,pblib] -pyyaml -scikit-image==0.21.0 -scikit-learn==1.3.0 -scipy==1.11.2 -seaborn==0.12.2 -spacy==3.6.1 -stable-baselines3 -statsmodels==0.14.0 -sympy==1.12 -tensorflow==2.12.0 -transformers -weasyprint -wikipedia-api -xlrd diff --git a/exam-notebook/Dockerfile b/exam-notebook/Dockerfile deleted file mode 100644 index a80c38b..0000000 --- a/exam-notebook/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -#ARG IMAGE_SOURCE=ghcr.io/digiklausur/docker-stacks/minimal-notebook:latest -FROM ghcr.io/digiklausur/docker-stacks/minimal-notebook-dev:latest - -LABEL maintainer="e2x project H-BRS " - -USER root -# Copy nbgrader base config -COPY configs/nbgrader_config.py /etc/jupyter/nbgrader_config.py -RUN chown root:$NB_GID /etc/jupyter/nbgrader_config.py &&\ - chmod g+rwX /etc/jupyter/nbgrader_config.py - -USER $NB_USER - -# Install grading requirements -COPY requirements.txt /tmp/requirements.txt -RUN pip install --no-cache-dir -r /tmp/requirements.txt && \ - pip install --no-cache-dir e2xgrader=="0.3.0-dev2" - -# Set up kernels -RUN touch /etc/ipython/ipython_config.py && \ - python -m exam_kernel.install --sys-prefix && \ - python -m java_syntax_kernel.install --sys-prefix - -USER root -RUN rm /tmp/requirements.txt - -USER $NB_USER - -# Activate student exam mode by default -RUN e2xgrader activate student_exam --sys-prefix - -# Remove RISE -RUN pip uninstall -y RISE - -# Jupyter extension list -RUN jupyter nbextension list diff --git a/exam-notebook/requirements.txt b/exam-notebook/requirements.txt deleted file mode 100644 index 65f53cb..0000000 --- a/exam-notebook/requirements.txt +++ /dev/null @@ -1,37 +0,0 @@ -bokeh -colorama -easydict==1.9 -graphviz -imageio==2.31.1 -ipycanvas -ipysheet -ipywidgets -itables -matplotlib==3.7.2 -numpy -nx -pandas==2.0.3 -pdfkit -Pillow==10.0.0 -pyagrum -pydantic -pydantic_yaml -pydot -pyparsing>=3.0.0 -pypdf2 -python-sat[aiger,approxmc,pblib] -pyyaml -scikit-image==0.21.0 -scikit-learn==1.3.0 -scipy==1.11.2 -seaborn==0.12.2 -statsmodels==0.14.0 -sympy==1.12 -exam_kernel -e2xgradingtools -ngshare_exchange>=0.5.2 -assignmenttest @ git+https://github.com/Digiklausur/assignmenttest@master -e2xtest @ git+https://github.com/DigiKlausur/e2xtest@main -java_syntax_kernel @ git+https://github.com/DigiKlausur/java_syntax_kernel@main -nbgitpuller -nbresuse diff --git a/datascience-notebook/Dockerfile b/images/datascience-notebook/Dockerfile similarity index 65% rename from datascience-notebook/Dockerfile rename to images/datascience-notebook/Dockerfile index e607b9b..3624de2 100644 --- a/datascience-notebook/Dockerfile +++ b/images/datascience-notebook/Dockerfile @@ -2,13 +2,13 @@ ARG IMAGE_SOURCE=ghcr.io/digiklausur/docker-stacks/minimal-notebook:latest FROM $IMAGE_SOURCE LABEL maintainer="e2x project H-BRS " +LABEL description="e2x datascience notebook" USER $NB_USER # Install requirements COPY requirements.txt /tmp/requirements.txt -RUN pip install --no-cache-dir -r /tmp/requirements.txt && \ - pip install torch==2.0.0+cpu torchvision==0.15.1+cpu torchaudio==2.0.1 --index-url https://download.pytorch.org/whl/cpu +RUN pip install --no-cache-dir -r /tmp/requirements.txt # Jupyter extension list RUN jupyter nbextension list diff --git a/datascience-notebook/README.md b/images/datascience-notebook/README.md similarity index 100% rename from datascience-notebook/README.md rename to images/datascience-notebook/README.md diff --git a/images/datascience-notebook/requirements.txt b/images/datascience-notebook/requirements.txt new file mode 100644 index 0000000..ee36c81 --- /dev/null +++ b/images/datascience-notebook/requirements.txt @@ -0,0 +1,31 @@ +bokeh +colorama +easydict==1.13 +graphviz +imageio==2.34.0 +ipycanvas +ipysheet +ipywidgets +itables +matplotlib==3.8.3 +nbconvert>=7.16.3 +numpy +nx +pandas==2.2.1 +pdfkit +Pillow==10.3.0 +pyagrum +pydantic +pydantic_yaml +pydot +pyparsing>=3.0.0 +pypdf2 +python-sat[aiger,approxmc,pblib] +pyyaml +scikit-image==0.22.0 +scikit-learn==1.3.2 +scipy==1.13.0 +seaborn==0.13.2 +statsmodels==0.14.1 +sympy==1.12 +xlrd diff --git a/images/e2xgrader-notebook/Dockerfile b/images/e2xgrader-notebook/Dockerfile new file mode 100644 index 0000000..7113efc --- /dev/null +++ b/images/e2xgrader-notebook/Dockerfile @@ -0,0 +1,44 @@ +ARG IMAGE_SOURCE=ghcr.io/digiklausur/docker-stacks/minimal-notebook:latest +FROM $IMAGE_SOURCE + +ARG FROM_REPO=false +ARG E2XGRADER_VERSION="" +ARG E2XGRADER_BRANCH="" +ARG E2XGRADER_MODE=student + +# Inherit from any image, then just install e2xgrader and activate a mode +LABEL maintainer="e2x project H-BRS " + +USER root +# Copy nbgrader base config +COPY configs/nbgrader_config.py /etc/jupyter/nbgrader_config.py +RUN chown root:$NB_GID /etc/jupyter/nbgrader_config.py &&\ + chmod g+rwX /etc/jupyter/nbgrader_config.py + +USER $NB_USER + +# Install requirements shared between all modes +RUN echo "Installing common requirements" +COPY common_requirements.txt /tmp/common_requirements.txt +RUN pip install --no-cache-dir -r /tmp/common_requirements.txt + +# Install mode specific requirements +RUN echo "Installing requirements for ${E2XGRADER_MODE}" +COPY ${E2XGRADER_MODE}_requirements.txt /tmp/mode_requirements.txt +RUN pip install --no-cache-dir -r /tmp/mode_requirements.txt + +# Install e2xgrader +RUN if [ "$FROM_REPO" = "false" ] && [ -z "$E2XGRADER_VERSION" ]; then \ + echo "Installing e2xgrader from default source..."; \ + pip install e2xgrader; \ + elif [ "$FROM_REPO" = "false" ] && [ -n "$E2XGRADER_VERSION" ]; then \ + echo "Installing e2xgrader version: $E2XGRADER_VERSION from PyPI..."; \ + pip install e2xgrader=="$E2XGRADER_VERSION"; \ + elif [ "$FROM_REPO" = "true" ]; then \ + echo "Installing e2xgrader from GitHub repository: https://github.com/DigiKlausur/e2xgrader.git@$E2XGRADER_BRANCH"; \ + pip install --no-cache-dir git+https://github.com/DigiKlausur/e2xgrader.git@"${E2XGRADER_BRANCH}"; \ + fi + +# Activate a mode +RUN e2xgrader activate $E2XGRADER_MODE --sys-prefix + diff --git a/notebook/requirements.txt b/images/e2xgrader-notebook/common_requirements.txt similarity index 61% rename from notebook/requirements.txt rename to images/e2xgrader-notebook/common_requirements.txt index 8cda871..bc0a4a4 100644 --- a/notebook/requirements.txt +++ b/images/e2xgrader-notebook/common_requirements.txt @@ -1,10 +1,5 @@ -exam_kernel -e2xgradingtools -ngshare_exchange>=0.5.2 assignmenttest @ git+https://github.com/Digiklausur/assignmenttest@master +e2xgradingtools e2xtest @ git+https://github.com/DigiKlausur/e2xtest@main -pdf_feedback_exporter @ git+https://github.com/DigiKlausur/pdf_feedback_exporter@main +exam_kernel java_syntax_kernel @ git+https://github.com/DigiKlausur/java_syntax_kernel@main -nbgitpuller -nbresuse -nbconvert>=7.16.3 diff --git a/exam-notebook/configs/nbgrader_config.py b/images/e2xgrader-notebook/configs/nbgrader_config.py similarity index 100% rename from exam-notebook/configs/nbgrader_config.py rename to images/e2xgrader-notebook/configs/nbgrader_config.py diff --git a/images/e2xgrader-notebook/student_exam_requirements.txt b/images/e2xgrader-notebook/student_exam_requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/images/e2xgrader-notebook/student_requirements.txt b/images/e2xgrader-notebook/student_requirements.txt new file mode 100644 index 0000000..4890837 --- /dev/null +++ b/images/e2xgrader-notebook/student_requirements.txt @@ -0,0 +1 @@ +RISE==5.7.1 \ No newline at end of file diff --git a/images/e2xgrader-notebook/teacher_requirements.txt b/images/e2xgrader-notebook/teacher_requirements.txt new file mode 100644 index 0000000..7de7cfb --- /dev/null +++ b/images/e2xgrader-notebook/teacher_requirements.txt @@ -0,0 +1,3 @@ +RISE==5.7.1 +weasyprint +pdf_feedback_exporter @ git+https://github.com/DigiKlausur/pdf_feedback_exporter@main \ No newline at end of file diff --git a/minimal-notebook/Dockerfile b/images/minimal-notebook/Dockerfile similarity index 75% rename from minimal-notebook/Dockerfile rename to images/minimal-notebook/Dockerfile index b3f87f1..0d93779 100644 --- a/minimal-notebook/Dockerfile +++ b/images/minimal-notebook/Dockerfile @@ -1,9 +1,11 @@ -FROM jupyter/minimal-notebook:4d70cf8da953 +ARG IMAGE_SOURCE=jupyter/minimal-notebook:4d70cf8da953 +FROM $IMAGE_SOURCE # Built from... https://hub.docker.com/r/jupyter/minimal-notebook/ # https://github.com/jupyter/docker-stacks/blob/main/images/minimal-notebook/Dockerfile # Built from... Ubuntu 22.04 LABEL maintainer="e2x project H-BRS " +LABEL description="e2x minimal notebook" USER root @@ -29,10 +31,6 @@ RUN rm -rf $HOME/work RUN apt-get update -y \ && apt-get install --yes \ pandoc \ - texlive-xetex \ - texlive-fonts-recommended \ - texlive-latex-recommended \ - texlive-latex-extra \ vim \ libgl1-mesa-dev zip wkhtmltopdf \ build-essential \ @@ -45,20 +43,18 @@ USER $NB_USER RUN mamba install -y gh --channel conda-forge && \ mamba install -y graphviz -RUN pip install --upgrade pip && \ - pip install --no-cache-dir RISE==5.7.1 +RUN pip install --upgrade pip + +# Install requirements +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt # Enable extenstions and Jupyter contrib extensions RUN cd /tmp && \ git clone --depth 1 -b master https://github.com/ipython-contrib/jupyter_contrib_nbextensions.git && \ cd /tmp/jupyter_contrib_nbextensions/src/jupyter_contrib_nbextensions/nbextensions && \ - jupyter nbextension install --sys-prefix codefolding && \ - jupyter nbextension enable --sys-prefix codefolding/main && \ - jupyter nbextension install --sys-prefix collapsible_headings && \ - jupyter nbextension enable --sys-prefix collapsible_headings/main && \ jupyter nbextension install --sys-prefix execute_time && \ jupyter nbextension enable --sys-prefix execute_time/ExecuteTime && \ - jupyter-nbextension install rise --py --sys-prefix && \ jupyter nbextension list && \ rm -rf /tmp/jupyter_contrib_nbextensions diff --git a/minimal-notebook/README.md b/images/minimal-notebook/README.md similarity index 100% rename from minimal-notebook/README.md rename to images/minimal-notebook/README.md diff --git a/images/minimal-notebook/requirements.txt b/images/minimal-notebook/requirements.txt new file mode 100644 index 0000000..093e4d2 --- /dev/null +++ b/images/minimal-notebook/requirements.txt @@ -0,0 +1 @@ +nbresuse \ No newline at end of file diff --git a/images/ml-notebook/Dockerfile b/images/ml-notebook/Dockerfile new file mode 100644 index 0000000..c9c1f45 --- /dev/null +++ b/images/ml-notebook/Dockerfile @@ -0,0 +1,22 @@ +ARG IMAGE_SOURCE=ghcr.io/digiklausur/docker-stacks/datascience-notebook:latest +FROM $IMAGE_SOURCE + +LABEL maintainer="e2x project H-BRS " +LABEL description="e2x machine learning notebook" + +USER $NB_USER + +# Install pytorch cpu +RUN pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + +# Install requirements +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +# Jupyter extension list +RUN jupyter nbextension list + +USER root +RUN rm /tmp/requirements.txt + +USER $NB_USER diff --git a/images/ml-notebook/requirements.txt b/images/ml-notebook/requirements.txt new file mode 100644 index 0000000..8787e5d --- /dev/null +++ b/images/ml-notebook/requirements.txt @@ -0,0 +1,3 @@ +gymnasium==0.29.1 +opencv-contrib-python==4.9.0.80 +stable-baselines3 \ No newline at end of file diff --git a/images/nlp-notebook/Dockerfile b/images/nlp-notebook/Dockerfile new file mode 100644 index 0000000..e7bfa46 --- /dev/null +++ b/images/nlp-notebook/Dockerfile @@ -0,0 +1,19 @@ +ARG IMAGE_SOURCE=ghcr.io/digiklausur/docker-stacks/ml-notebook:latest +FROM $IMAGE_SOURCE + +LABEL maintainer="e2x project H-BRS " +LABEL description="e2x natural language processing notebook" + +USER $NB_USER + +# Install requirements +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt + +# Jupyter extension list +RUN jupyter nbextension list + +USER root +RUN rm /tmp/requirements.txt + +USER $NB_USER diff --git a/images/nlp-notebook/requirements.txt b/images/nlp-notebook/requirements.txt new file mode 100644 index 0000000..8b858cc --- /dev/null +++ b/images/nlp-notebook/requirements.txt @@ -0,0 +1,8 @@ +spacy==3.7.4 +gensim==4.3.2 +wikipedia +nltk==3.8.1 +transformers==4.39.3 +tokenizers +huggingface_hub +wordcloud \ No newline at end of file diff --git a/ngshare/Dockerfile b/ngshare/Dockerfile deleted file mode 100644 index 0a439a9..0000000 --- a/ngshare/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM jupyterhub/jupyterhub:latest -LABEL maintainer="DigiKlausur project HBRS " - -#RUN apt-get install git-all -RUN apt-get update && \ - apt-get install -y git &&\ - apt-get clean &&\ - rm -rf /var/lib/apt/lists/* -# Change time -ENV TZ=Europe/Berlin -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -# Install exam kernel -RUN pip install --no-cache-dir git+https://github.com/LibreTexts/ngshare.git@89685b12580990f983d4523e0ec45fd249afb702 - -USER 65535:65535 -ENTRYPOINT ["python3", "-m", "ngshare"] diff --git a/notebook/Dockerfile b/notebook/Dockerfile deleted file mode 100644 index 6ea6d33..0000000 --- a/notebook/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -ARG IMAGE_SOURCE=ghcr.io/digiklausur/docker-stacks/datascience-notebook:latest -FROM $IMAGE_SOURCE - -LABEL maintainer="e2x project H-BRS " - -ARG E2XGRADER=e2xgrader - -USER root -# Copy nbgrader base config -COPY configs/nbgrader_config.py /etc/jupyter/nbgrader_config.py -RUN chown root:$NB_GID /etc/jupyter/nbgrader_config.py &&\ - chmod g+rwX /etc/jupyter/nbgrader_config.py - -USER $NB_USER - -# Install grading requirements -COPY requirements.txt /tmp/requirements.txt -RUN pip install --no-cache-dir -r /tmp/requirements.txt && \ - pip install --no-cache-dir e2xgrader=="0.3.0-dev2" - -# Set up kernels -RUN touch /etc/ipython/ipython_config.py && \ - python -m exam_kernel.install --sys-prefix && \ - python -m java_syntax_kernel.install --sys-prefix - -# Activate student mode by default -RUN e2xgrader activate student --sys-prefix - -# Jupyter extension list -RUN jupyter nbextension list - -USER root -RUN rm /tmp/requirements.txt - -USER $NB_USER diff --git a/notebook/README.md b/notebook/README.md deleted file mode 100644 index 4a962c8..0000000 --- a/notebook/README.md +++ /dev/null @@ -1,4 +0,0 @@ -[![docker pulls](https://img.shields.io/docker/pulls/digiklausur/notebook.svg)](https://hub.docker.com/r/digiklausur/notebook) -[![image metadata](https://images.microbadger.com/badges/image/digiklausur/notebook.svg)](https://microbadger.com/images/digiklausur/notebook "digiklausur/notebook image metadata") - -### Jupyter Notebook image for e-assessment diff --git a/notebook/configs/nbgrader_config.py b/notebook/configs/nbgrader_config.py deleted file mode 100644 index 733be1d..0000000 --- a/notebook/configs/nbgrader_config.py +++ /dev/null @@ -1,5 +0,0 @@ -from e2xgrader.config import configure_base, configure_exchange - -c = get_config() # noqa: F821 -configure_base(c) -configure_exchange(c)