From de394fe04e084d0572fe56180601b0d4a5b6256e Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Tue, 12 Dec 2023 18:49:29 +0000 Subject: [PATCH] build kotsadm-migrations and use rqlite 7.21.4 --- .../action.yml | 62 +++++++++++++++++++ .github/workflows/alpha.yaml | 29 ++++----- .github/workflows/build-test.yaml | 18 ++---- .github/workflows/release.yaml | 15 ++--- .image.env | 2 +- Makefile | 2 +- cmd/imagedeps/README.md | 4 +- cmd/imagedeps/main_test.go | 2 +- cmd/imagedeps/testdata/rqlite/.image.env | 2 +- cmd/imagedeps/testdata/rqlite/constants.go | 2 +- deploy/rqlite/apko.yaml | 2 +- migrations/Makefile | 9 --- migrations/deploy/apko.yaml.tmpl | 36 +++++++++++ migrations/deploy/melange.yaml.tmpl | 32 ++++++++++ pkg/image/constants.go | 2 +- 15 files changed, 159 insertions(+), 60 deletions(-) create mode 100644 .github/actions/build-push-kotsadm-migrations-image/action.yml create mode 100644 migrations/deploy/apko.yaml.tmpl create mode 100644 migrations/deploy/melange.yaml.tmpl diff --git a/.github/actions/build-push-kotsadm-migrations-image/action.yml b/.github/actions/build-push-kotsadm-migrations-image/action.yml new file mode 100644 index 0000000000..858ac584db --- /dev/null +++ b/.github/actions/build-push-kotsadm-migrations-image/action.yml @@ -0,0 +1,62 @@ +name: 'Build and push kotsadm-migrations image' +description: 'Composite action for building and pushing kotsadm-migrations image' +inputs: + image-name: + description: 'Full destination kotsadm-migrations image name' + required: true + + git-tag: + description: 'Git tag' + required: true + + registry-username: + description: 'Username to login to registry' + default: '' + required: false + + registry-password: + description: 'Password to login to registry' + default: '' + required: false + +runs: + using: "composite" + steps: + - name: load environment variables from .image.env + uses: falti/dotenv-action@v1 + id: dotenv + with: + path: .image.env + + - uses: shrink/actions-docker-extract@v3 + with: + image: schemahero/schemahero:${{ steps.dotenv.outputs.SCHEMAHERO_TAG }} + path: /schemahero + destination: migrations + + - name: template melange and apko configs + shell: bash + run: | + export GIT_TAG=${{ inputs.git-tag }} + envsubst '${GIT_TAG}' < migrations/deploy/melange.yaml.tmpl > migrations/deploy/melange.yaml + envsubst '${GIT_TAG}' < migrations/deploy/apko.yaml.tmpl > migrations/deploy/apko.yaml + + - id: cache-dir + shell: bash + run: echo "cache_dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" + + - uses: chainguard-dev/actions/melange-build@main + with: + config: migrations/deploy/melange.yaml + archs: x86_64 + sign-with-temporary-key: true + cache-dir: ${{ steps.cache-dir.outputs.cache_dir }} + + - uses: chainguard-images/actions/apko-publish@main + with: + config: migrations/deploy/apko.yaml + archs: x86_64 + tag: ${{ inputs.image-name }} + vcs-url: true + generic-user: ${{ inputs.registry-username }} + generic-pass: ${{ inputs.registry-password }} diff --git a/.github/workflows/alpha.yaml b/.github/workflows/alpha.yaml index 8ac5131e43..d06d5149c5 100644 --- a/.github/workflows/alpha.yaml +++ b/.github/workflows/alpha.yaml @@ -19,24 +19,17 @@ jobs: uses: ./.github/actions/version-tag - build-schema-migrations: + build-kotsadm-migrations: runs-on: ubuntu-20.04 + needs: [generate-tag] steps: - - name: Checkout - uses: actions/checkout@v4 - - - uses: azure/docker-login@v1 - env: - DOCKER_CONFIG: ./migrations/.docker - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build - env: - DOCKER_CONFIG: ./.docker - run: | - mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make -C migrations schema-alpha + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-push-kotsadm-migrations-image + with: + image-name: index.docker.io/kotsadm/kotsadm-migrations:alpha + git-tag: ${{ needs.generate-tag.outputs.tag }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} build-rqlite: @@ -261,9 +254,9 @@ jobs: sarif_file: kotsadm-scan-output.sarif - scan_migrations: + scan_kotsadm_migrations: runs-on: ubuntu-20.04 - needs: [build-schema-migrations] + needs: [build-kotsadm-migrations] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 869c668c72..80adb0f0f8 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -383,7 +383,7 @@ jobs: build-migrations: runs-on: ubuntu-20.04 - needs: [ can-run-ci ] + needs: [ can-run-ci, generate-tag ] steps: - name: setup env run: | @@ -399,20 +399,10 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: load environment variables from .image.env - uses: falti/dotenv-action@v1 - id: dotenv + - uses: ./.github/actions/build-push-kotsadm-migrations-image with: - path: .image.env - - - name: build and push migrations for e2e - uses: docker/build-push-action@v5 - with: - tags: ttl.sh/automated-${{ github.run_id }}/kotsadm-migrations:24h - context: ./migrations - file: ./migrations/deploy/Dockerfile - push: true - build-args: SCHEMAHERO_TAG=${{ steps.dotenv.outputs.schemahero_tag }} + image-name: ttl.sh/automated-${{ github.run_id }}/kotsadm-migrations:24h + git-tag: ${{ needs.generate-tag.outputs.tag }} push-minio: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 999c4c097e..daabb41492 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -68,17 +68,12 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - uses: azure/docker-login@v1 - env: - DOCKER_CONFIG: ./migrations/.docker + - uses: ./.github/actions/build-push-kotsadm-migrations-image with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: "Release schema migrations on tag" - env: - GIT_TAG: ${{ needs.generate-tag.outputs.tag }} - DOCKER_CONFIG: ./.docker - run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make -C migrations schema-release + image-name: index.docker.io/kotsadm/kotsadm-migrations:${{ needs.generate-tag.outputs.tag }} + git-tag: ${{ needs.generate-tag.outputs.tag }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} build-web: runs-on: ubuntu-20.04 diff --git a/.image.env b/.image.env index f6d5f1c69f..18d2f5d4e9 100644 --- a/.image.env +++ b/.image.env @@ -2,7 +2,7 @@ # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. MINIO_TAG='0.20231101.183725' -RQLITE_TAG='8.0.1' +RQLITE_TAG='7.21.4' DEX_TAG='2.37.0' SCHEMAHERO_TAG='0.17.0' LVP_TAG='v0.5.5' \ No newline at end of file diff --git a/Makefile b/Makefile index 49dbbf09cd..7642f5ba30 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ include Makefile.build.mk CURRENT_USER := $(shell id -u -n) MINIO_TAG ?= 0.20231101.183725 -RQLITE_TAG ?= 8.0.1 +RQLITE_TAG ?= 7.21.4 DEX_TAG ?= 2.37.0 LVP_TAG ?= v0.5.5 diff --git a/cmd/imagedeps/README.md b/cmd/imagedeps/README.md index 50b797671f..732d036ee6 100644 --- a/cmd/imagedeps/README.md +++ b/cmd/imagedeps/README.md @@ -31,7 +31,7 @@ dex kotsadm/dex The preceding image spec will produce the following environment and Go files. ```shell MINIO_TAG='0.20231101.183725' -RQLITE_TAG='8.0.1' +RQLITE_TAG='7.21.4' DEX_TAG='2.37.0' ``` ```go @@ -39,7 +39,7 @@ package image const ( Minio = "kotsadm/minio:0.20231101.183725" - Rqlite = "kotsadm/rqlite:8.0.1" + Rqlite = "kotsadm/rqlite:7.21.4" Dex = "kotsadm/dex:2.37.0" ) ``` diff --git a/cmd/imagedeps/main_test.go b/cmd/imagedeps/main_test.go index 26dea136a6..5354fee1a8 100644 --- a/cmd/imagedeps/main_test.go +++ b/cmd/imagedeps/main_test.go @@ -14,7 +14,7 @@ import ( var ( minioTag = "0.20231101.183725" - rqliteTag = "8.0.1" + rqliteTag = "7.21.4" dexTag = "2.37.0" schemaheroTags = []string{ diff --git a/cmd/imagedeps/testdata/rqlite/.image.env b/cmd/imagedeps/testdata/rqlite/.image.env index 73a2d7b8ac..d6f44088a3 100644 --- a/cmd/imagedeps/testdata/rqlite/.image.env +++ b/cmd/imagedeps/testdata/rqlite/.image.env @@ -1,4 +1,4 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -RQLITE_TAG='8.0.1' \ No newline at end of file +RQLITE_TAG='7.21.4' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/rqlite/constants.go b/cmd/imagedeps/testdata/rqlite/constants.go index decfb545f0..455a24f3a8 100644 --- a/cmd/imagedeps/testdata/rqlite/constants.go +++ b/cmd/imagedeps/testdata/rqlite/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Rqlite = "kotsadm/rqlite:8.0.1" + Rqlite = "kotsadm/rqlite:7.21.4" ) diff --git a/deploy/rqlite/apko.yaml b/deploy/rqlite/apko.yaml index 7097434175..d57a18497c 100644 --- a/deploy/rqlite/apko.yaml +++ b/deploy/rqlite/apko.yaml @@ -4,7 +4,7 @@ contents: keyring: - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub packages: - - rqlite~8.0.1 + - rqlite~7.21.4 - rqlite-oci-entrypoint - bash - busybox diff --git a/migrations/Makefile b/migrations/Makefile index 2954166162..f28e7bce5e 100644 --- a/migrations/Makefile +++ b/migrations/Makefile @@ -1,15 +1,6 @@ SHELL:=/bin/bash -PROJECT_NAME ?= kotsadm-migrations SCHEMAHERO_TAG ?= 0.17.0 -.PHONY: schema-alpha -schema-alpha: IMAGE = kotsadm/${PROJECT_NAME}:alpha -schema-alpha: build_schema - -.PHONY: schema-release -schema-release: IMAGE = kotsadm/${PROJECT_NAME}:${GIT_TAG} -schema-release: build_schema - build_schema: docker build --pull --build-arg SCHEMAHERO_TAG=${SCHEMAHERO_TAG} -f deploy/Dockerfile -t ${IMAGE} . docker push ${IMAGE} diff --git a/migrations/deploy/apko.yaml.tmpl b/migrations/deploy/apko.yaml.tmpl new file mode 100644 index 0000000000..fb1796b3b3 --- /dev/null +++ b/migrations/deploy/apko.yaml.tmpl @@ -0,0 +1,36 @@ +contents: + repositories: + - https://packages.wolfi.dev/os + - ./packages/ + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + - ./melange.rsa.pub + packages: + - kotsadm-migrations-head # This is expected to be built locally by `melange`. + - bash + - busybox + - curl + - git + - wolfi-baselayout + +accounts: + groups: + - groupname: schemahero + gid: 1001 + users: + - username: schemahero + uid: 1001 + gid: 1001 + run-as: schemahero + +environment: + VERSION: ${GIT_TAG} + +entrypoint: + command: /schemahero + +cmd: apply + +archs: + - x86_64 + - aarch64 diff --git a/migrations/deploy/melange.yaml.tmpl b/migrations/deploy/melange.yaml.tmpl new file mode 100644 index 0000000000..88365e8866 --- /dev/null +++ b/migrations/deploy/melange.yaml.tmpl @@ -0,0 +1,32 @@ +package: + name: kotsadm-migrations-head + version: ${GIT_TAG} + epoch: 0 + description: kotsadm-migrations package + copyright: + - license: Apache-2.0 + +environment: + contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - ca-certificates-bundle + - busybox + - git + - go + environment: + GOMODCACHE: '/var/cache/melange' + +pipeline: + - runs: | + set -x + export DESTDIR="${{targets.destdir}}" + mkdir -p "${DESTDIR}" + + mv migrations/tables "${DESTDIR}/tables" + + # expected to have already been extracted from the schemahero image + mv migrations/schemahero "${DESTDIR}/schemahero" diff --git a/pkg/image/constants.go b/pkg/image/constants.go index fae7b78d0a..009a983805 100644 --- a/pkg/image/constants.go +++ b/pkg/image/constants.go @@ -6,7 +6,7 @@ package image const ( Minio = "kotsadm/minio:0.20231101.183725" - Rqlite = "kotsadm/rqlite:8.0.1" + Rqlite = "kotsadm/rqlite:7.21.4" Dex = "kotsadm/dex:2.37.0" Schemahero = "schemahero/schemahero:0.17.0" Lvp = "replicated/local-volume-provider:v0.5.5"