From 19bacc70bf7dda03b7ac58990b489c96fb3ae663 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Fri, 8 Dec 2023 17:05:47 +0000 Subject: [PATCH 01/22] build rqlite with apko --- .../build-push-rqlite-image/action.yml | 28 ++++++++++++++++++ .github/workflows/alpha.yaml | 11 +++++++ .github/workflows/build-test.yaml | 10 +++---- deploy/rqlite/apko.yaml | 29 +++++++++++++++++++ pkg/kotsadm/objects/rqlite_objects.go | 4 +++ 5 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 .github/actions/build-push-rqlite-image/action.yml create mode 100644 deploy/rqlite/apko.yaml diff --git a/.github/actions/build-push-rqlite-image/action.yml b/.github/actions/build-push-rqlite-image/action.yml new file mode 100644 index 0000000000..323ae19c15 --- /dev/null +++ b/.github/actions/build-push-rqlite-image/action.yml @@ -0,0 +1,28 @@ +name: 'Build and push rqlite image' +description: 'Composite action for building and pushing rqlite image' +inputs: + image-name: + description: 'Full destination rqlite image name' + required: true + + registry-username: + description: 'Username to login to registry' + default: '' + required: false + + registry-password: + description: 'Password to login to registry' + default: '' + required: false + +runs: + using: "composite" + steps: + - uses: chainguard-images/actions/apko-publish@main + with: + config: deploy/rqlite/apko.yaml + archs: x86_64 + tag: ${{ inputs.image-name }} + vcs-url: true + generic-user: ${{ inputs.registry-username }} + generic-pass: ${{ inputs.registry-password }} diff --git a/.github/workflows/alpha.yaml b/.github/workflows/alpha.yaml index 19d2d5c248..f0aba88527 100644 --- a/.github/workflows/alpha.yaml +++ b/.github/workflows/alpha.yaml @@ -39,6 +39,17 @@ jobs: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make -C migrations schema-alpha + build-rqlite: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-push-rqlite-image + with: + image-name: index.docker.io/kotsadm/rqlite:alpha + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + + build-kotsadm: runs-on: ubuntu-20.04 needs: [generate-tag] diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 17074b7728..b2720d3fd0 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -446,11 +446,11 @@ jobs: with: path: .image.env - - name: push rqlite for CI - run: | - docker pull rqlite/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} - docker tag rqlite/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} - docker push ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} + - uses: ./.github/actions/build-push-rqlite-image + with: + image-name: ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} push-dex: diff --git a/deploy/rqlite/apko.yaml b/deploy/rqlite/apko.yaml new file mode 100644 index 0000000000..3ecf7c19cd --- /dev/null +++ b/deploy/rqlite/apko.yaml @@ -0,0 +1,29 @@ +contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - rqlite # TODO NOW: pin to a version + - bash + - busybox + - wolfi-baselayout + +accounts: + groups: + - groupname: rqlite + gid: 1001 + users: + - username: rqlite + uid: 1001 + gid: 1001 + run-as: rqlite + +entrypoint: + command: docker-entrypoint.sh + +cmd: /rqlite + +archs: + - x86_64 + - aarch64 diff --git a/pkg/kotsadm/objects/rqlite_objects.go b/pkg/kotsadm/objects/rqlite_objects.go index 0b750c6e17..b8c56c7a99 100644 --- a/pkg/kotsadm/objects/rqlite_objects.go +++ b/pkg/kotsadm/objects/rqlite_objects.go @@ -128,6 +128,10 @@ func RqliteStatefulset(deployOptions types.DeployOptions, size resource.Quantity Name: "rqlite", ContainerPort: 4001, }, + { + Name: "raft", + ContainerPort: 4002, + }, }, VolumeMounts: volumeMounts, Env: getRqliteEnvs(), From fbd16ab0214a53ade284289a70a34451729f9a38 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Fri, 8 Dec 2023 17:33:51 +0000 Subject: [PATCH 02/22] fix cmd --- deploy/rqlite/apko.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/deploy/rqlite/apko.yaml b/deploy/rqlite/apko.yaml index 3ecf7c19cd..ae5b75b29b 100644 --- a/deploy/rqlite/apko.yaml +++ b/deploy/rqlite/apko.yaml @@ -4,7 +4,7 @@ contents: keyring: - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub packages: - - rqlite # TODO NOW: pin to a version + - rqlite-7.21.4 - bash - busybox - wolfi-baselayout @@ -19,10 +19,7 @@ accounts: gid: 1001 run-as: rqlite -entrypoint: - command: docker-entrypoint.sh - -cmd: /rqlite +cmd: rqlite archs: - x86_64 From cb90682d624f1bd0cd6440315c6bac2bf7e8f5b3 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Fri, 8 Dec 2023 17:42:26 +0000 Subject: [PATCH 03/22] fix syntax --- deploy/rqlite/apko.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/rqlite/apko.yaml b/deploy/rqlite/apko.yaml index ae5b75b29b..495facb8ed 100644 --- a/deploy/rqlite/apko.yaml +++ b/deploy/rqlite/apko.yaml @@ -4,7 +4,7 @@ contents: keyring: - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub packages: - - rqlite-7.21.4 + - rqlite~7.21.4 - bash - busybox - wolfi-baselayout From 50584eea46cc46e8a372fe7fdb1eb4e289772e7a Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Fri, 8 Dec 2023 18:02:04 +0000 Subject: [PATCH 04/22] one more try --- deploy/rqlite/apko.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deploy/rqlite/apko.yaml b/deploy/rqlite/apko.yaml index 495facb8ed..6345585026 100644 --- a/deploy/rqlite/apko.yaml +++ b/deploy/rqlite/apko.yaml @@ -5,6 +5,7 @@ contents: - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub packages: - rqlite~7.21.4 + - rqlite-oci-entrypoint - bash - busybox - wolfi-baselayout @@ -19,6 +20,9 @@ accounts: gid: 1001 run-as: rqlite +entrypoint: + command: /usr/bin/docker-entrypoint.sh + cmd: rqlite archs: From 97a7a772cdfcc66b9f76b90dde12047e3f838629 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Fri, 8 Dec 2023 18:08:59 +0000 Subject: [PATCH 05/22] updates --- .github/workflows/build-test.yaml | 2 -- deploy/rqlite/apko.yaml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index b2720d3fd0..1d6aa0eb25 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -449,8 +449,6 @@ jobs: - uses: ./.github/actions/build-push-rqlite-image with: image-name: ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} - registry-username: ${{ secrets.DOCKERHUB_USER }} - registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} push-dex: diff --git a/deploy/rqlite/apko.yaml b/deploy/rqlite/apko.yaml index 6345585026..d57a18497c 100644 --- a/deploy/rqlite/apko.yaml +++ b/deploy/rqlite/apko.yaml @@ -21,7 +21,7 @@ accounts: run-as: rqlite entrypoint: - command: /usr/bin/docker-entrypoint.sh + command: docker-entrypoint.sh cmd: rqlite From 00939d4f1d4619d3897ffdb16081945a6c43f6cd Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Fri, 8 Dec 2023 22:22:06 +0000 Subject: [PATCH 06/22] build minio with apko --- .../actions/build-push-minio-image/action.yml | 28 +++++++++++++++++++ .github/workflows/alpha.yaml | 27 ++++++++++++++++-- .github/workflows/build-test.yaml | 8 ++---- deploy/kurl/kotsadm/template/base/rqlite.yaml | 2 ++ deploy/minio/apko.yaml | 28 +++++++++++++++++++ 5 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 .github/actions/build-push-minio-image/action.yml create mode 100644 deploy/minio/apko.yaml diff --git a/.github/actions/build-push-minio-image/action.yml b/.github/actions/build-push-minio-image/action.yml new file mode 100644 index 0000000000..527a183918 --- /dev/null +++ b/.github/actions/build-push-minio-image/action.yml @@ -0,0 +1,28 @@ +name: 'Build and push minio image' +description: 'Composite action for building and pushing minio image' +inputs: + image-name: + description: 'Full destination minio image name' + required: true + + registry-username: + description: 'Username to login to registry' + default: '' + required: false + + registry-password: + description: 'Password to login to registry' + default: '' + required: false + +runs: + using: "composite" + steps: + - uses: chainguard-images/actions/apko-publish@main + with: + config: deploy/minio/apko.yaml + archs: x86_64 + tag: ${{ inputs.image-name }} + vcs-url: true + generic-user: ${{ inputs.registry-username }} + generic-pass: ${{ inputs.registry-password }} diff --git a/.github/workflows/alpha.yaml b/.github/workflows/alpha.yaml index f0aba88527..ab176c4da7 100644 --- a/.github/workflows/alpha.yaml +++ b/.github/workflows/alpha.yaml @@ -43,9 +43,30 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 + - name: Read image tags from env file + uses: falti/dotenv-action@v1 + id: dotenv + with: + path: .image.env - uses: ./.github/actions/build-push-rqlite-image with: - image-name: index.docker.io/kotsadm/rqlite:alpha + image-name: index.docker.io/kotsadm/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + + + build-minio: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - name: Read image tags from env file + uses: falti/dotenv-action@v1 + id: dotenv + with: + path: .image.env + - uses: ./.github/actions/build-push-minio-image + with: + image-name: index.docker.io/kotsadm/minio:${{ steps.dotenv.outputs.MINIO_TAG }} registry-username: ${{ secrets.DOCKERHUB_USER }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} @@ -95,7 +116,7 @@ jobs: id: scan uses: aquasecurity/trivy-action@master with: - image-ref: "rqlite/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}" + image-ref: "docker.io/kotsadm/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}" format: 'template' template: '@/contrib/sarif.tpl' output: 'rqlite-scan-output.sarif' @@ -122,7 +143,7 @@ jobs: id: scan uses: aquasecurity/trivy-action@master with: - image-ref: "minio/minio:${{ steps.dotenv.outputs.minio_tag }}" + image-ref: "docker.io/kotsadm/minio:${{ steps.dotenv.outputs.MINIO_TAG }}" format: 'template' template: '@/contrib/sarif.tpl' output: 'minio-scan-output.sarif' diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 1d6aa0eb25..63277babf4 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -427,11 +427,9 @@ jobs: with: path: .image.env - - name: push minio for e2e - run: | - docker pull minio/minio:${{ steps.dotenv.outputs.minio_tag }} - docker tag minio/minio:${{ steps.dotenv.outputs.minio_tag }} ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.minio_tag }} - docker push ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.minio_tag }} + - uses: ./.github/actions/build-push-minio-image + with: + image-name: ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.minio_tag }} push-rqlite: diff --git a/deploy/kurl/kotsadm/template/base/rqlite.yaml b/deploy/kurl/kotsadm/template/base/rqlite.yaml index 2a78dd37b1..5038d90722 100644 --- a/deploy/kurl/kotsadm/template/base/rqlite.yaml +++ b/deploy/kurl/kotsadm/template/base/rqlite.yaml @@ -61,6 +61,8 @@ spec: ports: - name: rqlite containerPort: 4001 + - name: raft + containerPort: 4002 volumeMounts: - name: kotsadm-rqlite mountPath: /rqlite/file diff --git a/deploy/minio/apko.yaml b/deploy/minio/apko.yaml new file mode 100644 index 0000000000..8ddf39807c --- /dev/null +++ b/deploy/minio/apko.yaml @@ -0,0 +1,28 @@ +contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - minio + - mc + - bash + - busybox + - wolfi-baselayout + +accounts: + groups: + - groupname: minio + gid: 1001 + users: + - username: minio + uid: 1001 + gid: 1001 + run-as: minio + +entrypoint: + command: minio + +archs: + - x86_64 + - aarch64 From eb0b6f60ecfb18d98b0f714ded5a92ebe955b0c5 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Fri, 8 Dec 2023 22:23:51 +0000 Subject: [PATCH 07/22] updates --- pkg/kotsadm/objects/minio_objects.go | 2 +- pkg/kotsadm/objects/scripts/import-minio-data.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/kotsadm/objects/minio_objects.go b/pkg/kotsadm/objects/minio_objects.go index 56c12594c7..4ae3acfc76 100644 --- a/pkg/kotsadm/objects/minio_objects.go +++ b/pkg/kotsadm/objects/minio_objects.go @@ -136,7 +136,7 @@ func MinioStatefulset(deployOptions types.DeployOptions, size resource.Quantity) Command: []string{ "/bin/sh", "-ce", - "/usr/bin/docker-entrypoint.sh minio -C /home/minio/.minio/ --quiet server /export", + "minio -C /home/minio/.minio/ --quiet server /export", }, Ports: []corev1.ContainerPort{ { diff --git a/pkg/kotsadm/objects/scripts/import-minio-data.sh b/pkg/kotsadm/objects/scripts/import-minio-data.sh index 65acbe11fd..2eb1aef9eb 100644 --- a/pkg/kotsadm/objects/scripts/import-minio-data.sh +++ b/pkg/kotsadm/objects/scripts/import-minio-data.sh @@ -33,7 +33,7 @@ shopt -s dotglob rm -rfv /export/* echo "starting new minio instance" -/bin/sh -ce "/usr/bin/docker-entrypoint.sh minio -C /home/minio/.minio/ server /export" & +/bin/sh -ce "minio -C /home/minio/.minio/ server /export" & MINIO_PID=$! # alias the minio instance From f47fa3e57811e1c2619b176b8683a864543af85e Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Fri, 8 Dec 2023 23:11:41 +0000 Subject: [PATCH 08/22] update command --- pkg/kotsadm/minio.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/kotsadm/minio.go b/pkg/kotsadm/minio.go index 7385b989e9..fb478a421f 100644 --- a/pkg/kotsadm/minio.go +++ b/pkg/kotsadm/minio.go @@ -127,6 +127,7 @@ func ensureMinioStatefulset(deployOptions types.DeployOptions, clientset kuberne existingMinio.Spec.Template.Spec.Volumes = desiredMinio.Spec.Template.Spec.DeepCopy().Volumes existingMinio.Spec.Template.Spec.Containers[0].Image = desiredMinio.Spec.Template.Spec.Containers[0].Image existingMinio.Spec.Template.Spec.Containers[0].VolumeMounts = desiredMinio.Spec.Template.Spec.Containers[0].DeepCopy().VolumeMounts + existingMinio.Spec.Template.Spec.Containers[0].Command = desiredMinio.Spec.Template.Spec.Containers[0].Command existingMinio.Spec.Template.Spec.InitContainers = desiredMinio.Spec.Template.Spec.DeepCopy().InitContainers _, err = clientset.AppsV1().StatefulSets(deployOptions.Namespace).Update(ctx, existingMinio, metav1.UpdateOptions{}) From b2d93a8075dd026569488754725acf172b3505d1 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Sun, 10 Dec 2023 00:49:25 +0000 Subject: [PATCH 09/22] updates --- .../actions/build-push-dex-image/action.yml | 28 +++++++++++ .github/workflows/alpha.yaml | 18 ++++++- .github/workflows/build-test.yaml | 9 ++-- .github/workflows/release.yaml | 32 +++++++++++-- .image.env | 8 ++-- Makefile | 26 +++++----- cmd/imagedeps/README.md | 20 ++++---- cmd/imagedeps/image-spec | 6 +-- cmd/imagedeps/main.go | 24 ++++++++-- cmd/imagedeps/tag-finder.go | 47 ++++++++++++++++--- deploy/apko_melange_build.md | 4 +- deploy/dex.Dockerfile | 2 - deploy/dex/apko.yaml | 29 ++++++++++++ deploy/kurl/kotsadm/template/base/Manifest | 2 +- deploy/kurl/kotsadm/template/base/rqlite.yaml | 2 +- deploy/kurl/kotsadm/template/generate.sh | 3 +- deploy/minio/apko.yaml | 2 +- deploy/rqlite/apko.yaml | 2 +- migrations/Makefile | 5 +- pkg/image/constants.go | 8 ++-- pkg/kotsadm/minio.go | 9 +++- pkg/kotsadm/minio_test.go | 7 +++ pkg/kotsadm/objects/images.go | 9 ++-- pkg/snapshot/filesystem_minio.go | 2 +- 24 files changed, 226 insertions(+), 78 deletions(-) create mode 100644 .github/actions/build-push-dex-image/action.yml delete mode 100644 deploy/dex.Dockerfile create mode 100644 deploy/dex/apko.yaml diff --git a/.github/actions/build-push-dex-image/action.yml b/.github/actions/build-push-dex-image/action.yml new file mode 100644 index 0000000000..ec70b426ae --- /dev/null +++ b/.github/actions/build-push-dex-image/action.yml @@ -0,0 +1,28 @@ +name: 'Build and push dex image' +description: 'Composite action for building and pushing dex image' +inputs: + image-name: + description: 'Full destination dex image name' + required: true + + registry-username: + description: 'Username to login to registry' + default: '' + required: false + + registry-password: + description: 'Password to login to registry' + default: '' + required: false + +runs: + using: "composite" + steps: + - uses: chainguard-images/actions/apko-publish@main + with: + config: deploy/dex/apko.yaml + archs: x86_64 + tag: ${{ inputs.image-name }} + vcs-url: true + generic-user: ${{ inputs.registry-username }} + generic-pass: ${{ inputs.registry-password }} diff --git a/.github/workflows/alpha.yaml b/.github/workflows/alpha.yaml index ab176c4da7..0a721884ec 100644 --- a/.github/workflows/alpha.yaml +++ b/.github/workflows/alpha.yaml @@ -71,6 +71,22 @@ jobs: registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + build-dex: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - name: Read image tags from env file + uses: falti/dotenv-action@v1 + id: dotenv + with: + path: .image.env + - uses: ./.github/actions/build-push-dex-image + with: + image-name: index.docker.io/kotsadm/dex:${{ steps.dotenv.outputs.DEX_TAG }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + + build-kotsadm: runs-on: ubuntu-20.04 needs: [generate-tag] @@ -171,7 +187,7 @@ jobs: id: scan uses: aquasecurity/trivy-action@master with: - image-ref: "ghcr.io/dexidp/dex:${{ steps.dotenv.outputs.dex_tag }}" + image-ref: "docker.io/kotsadm/dex:${{ steps.dotenv.outputs.dex_tag }}" format: 'template' template: '@/contrib/sarif.tpl' output: 'dex-scan-output.sarif' diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 63277babf4..187b34a0ba 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -429,7 +429,7 @@ jobs: - uses: ./.github/actions/build-push-minio-image with: - image-name: ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.minio_tag }} + image-name: ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.MINIO_TAG }} push-rqlite: @@ -461,10 +461,9 @@ jobs: with: path: .image.env - - name: push dex for CI - run: | - docker build --pull -f deploy/dex.Dockerfile -t ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }} --build-arg TAG=${{ steps.dotenv.outputs.DEX_TAG }} . - docker push ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }} + - uses: ./.github/actions/build-push-dex-image + with: + image-name: ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }} # only run validate-kurl-addon if changes to "deploy/kurl/kotsadm/template/**" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 86bcbcb582..29a9633010 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -6,6 +6,7 @@ on: - "v*.*.*" branches: - main + - build-rqlite-with-apko jobs: generate-tag: @@ -164,6 +165,13 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + + - name: Read image tags from env file + uses: falti/dotenv-action@v1 + id: dotenv + with: + path: .image.env + - uses: ./.github/actions/build-push-kotsadm-image with: chainguard-gcp-wif-pool: ${{ secrets.CHAINGUARD_GCP_WIF_POOL }} @@ -173,17 +181,31 @@ jobs: git-tag: ${{ needs.generate-tag.outputs.tag }} registry-username: ${{ secrets.DOCKERHUB_USER }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} - - uses: azure/docker-login@v1 - env: - DOCKER_CONFIG: ./.docker + + - uses: ./.github/actions/build-push-minio-image with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + image-name: index.docker.io/kotsadm/minio:${{ steps.dotenv.outputs.MINIO_TAG }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - uses: ./.github/actions/build-push-rqlite-image + with: + image-name: index.docker.io/kotsadm/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - uses: ./.github/actions/build-push-dex-image + with: + image-name: index.docker.io/kotsadm/dex:${{ steps.dotenv.outputs.DEX_TAG }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Build tagged release env: GIT_TAG: ${{ needs.generate-tag.outputs.tag }} DOCKER_CONFIG: ./.docker run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make build-release + - name: Upload airgap image uses: actions/upload-artifact@v3 with: diff --git a/.image.env b/.image.env index cb5fd3bde3..f6d5f1c69f 100644 --- a/.image.env +++ b/.image.env @@ -1,8 +1,8 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -MINIO_TAG='RELEASE.2023-11-11T08-14-41Z' -RQLITE_TAG='7.21.4' -DEX_TAG='v2.37.0' -SCHEMAHERO_TAG='0.16.0' +MINIO_TAG='0.20231101.183725' +RQLITE_TAG='8.0.1' +DEX_TAG='2.37.0' +SCHEMAHERO_TAG='0.17.0' LVP_TAG='v0.5.5' \ No newline at end of file diff --git a/Makefile b/Makefile index b2c6216431..d37bc28cb2 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ include Makefile.build.mk CURRENT_USER := $(shell id -u -n) -MINIO_TAG ?= RELEASE.2023-11-11T08-14-41Z -RQLITE_TAG ?= 7.21.4 -DEX_TAG ?= v2.37.0 +MINIO_TAG ?= 0.20231101.183725 +RQLITE_TAG ?= 8.0.1 +DEX_TAG ?= 2.37.0 LVP_TAG ?= v0.5.5 define sendMetrics @@ -118,31 +118,27 @@ build-ttl.sh: build all-ttl.sh: build-ttl.sh source .image.env && IMAGE=ttl.sh/${CURRENT_USER}/kotsadm-migrations:24h make -C migrations build_schema - docker pull minio/minio:${MINIO_TAG} - docker tag minio/minio:${MINIO_TAG} ttl.sh/${CURRENT_USER}/minio:${MINIO_TAG} + docker pull kotsadm/minio:${MINIO_TAG} + docker tag kotsadm/minio:${MINIO_TAG} ttl.sh/${CURRENT_USER}/minio:${MINIO_TAG} docker push ttl.sh/${CURRENT_USER}/minio:${MINIO_TAG} - docker pull rqlite/rqlite:${RQLITE_TAG} - docker tag rqlite/rqlite:${RQLITE_TAG} ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG} + docker pull kotsadm/rqlite:${RQLITE_TAG} + docker tag kotsadm/rqlite:${RQLITE_TAG} ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG} docker push ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG} -.PHONY: build-alpha -build-alpha: - docker build --pull -f deploy/Dockerfile --build-arg version=${GIT_TAG} -t kotsadm/kotsadm:alpha . - docker push kotsadm/kotsadm:alpha - .PHONY: build-release build-release: mkdir -p bin/docker-archive/kotsadm skopeo copy docker://kotsadm/kotsadm:${GIT_TAG} docker-archive:bin/docker-archive/kotsadm/${GIT_TAG} - docker build --pull -f deploy/dex.Dockerfile -t kotsadm/dex:${DEX_TAG} --build-arg TAG=${DEX_TAG} . - docker push kotsadm/dex:${DEX_TAG} mkdir -p bin/docker-archive/dex skopeo copy docker://kotsadm/dex:${DEX_TAG} docker-archive:bin/docker-archive/dex/${DEX_TAG} mkdir -p bin/docker-archive/minio - skopeo copy docker://minio/minio:${MINIO_TAG} docker-archive:bin/docker-archive/minio/${MINIO_TAG} + skopeo copy docker://kotsadm/minio:${MINIO_TAG} docker-archive:bin/docker-archive/minio/${MINIO_TAG} + + mkdir -p bin/docker-archive/rqlite + skopeo copy docker://kotsadm/rqlite:${RQLITE_TAG} docker-archive:bin/docker-archive/rqlite/${RQLITE_TAG} mkdir -p bin/docker-archive/local-volume-provider skopeo copy docker://replicated/local-volume-provider:${LVP_TAG} docker-archive:bin/docker-archive/local-volume-provider/${LVP_TAG} diff --git a/cmd/imagedeps/README.md b/cmd/imagedeps/README.md index 8962ea024d..50b797671f 100644 --- a/cmd/imagedeps/README.md +++ b/cmd/imagedeps/README.md @@ -20,27 +20,27 @@ is useful to restrict release tags to a major version, or to filter out garbage | Name | Image URI | Matcher Regexp (Optional) | |------|--------------------|----------| -| Name of the image for example **minio** | Untagged image reference **ghcr.io/dexidp/dex**| An optional regular expression, only matching tags will be included. | +| Name of the image for example **minio** | Untagged image reference **kotsadm/minio**| An optional regular expression, only matching tags will be included. | ### Sample image-spec ```text -minio minio/minio -rqlite rqlite/rqlite -dex ghcr.io/dexidp/dex +minio kotsadm/minio +rqlite kotsadm/rqlite +dex kotsadm/dex ``` The preceding image spec will produce the following environment and Go files. ```shell -MINIO_TAG='RELEASE.2021-09-15T04-54-25Z' -RQLITE_TAG='7.7.0' -DEX_TAG='v2.30.0' +MINIO_TAG='0.20231101.183725' +RQLITE_TAG='8.0.1' +DEX_TAG='2.37.0' ``` ```go package image const ( - Minio = "minio/minio:RELEASE.2021-09-15T04-54-25Z" - Rqlite = "rqlite/rqlite:7.7.0" - Dex = "ghcr.io/dexidp/dex:v2.30.0" + Minio = "kotsadm/minio:0.20231101.183725" + Rqlite = "kotsadm/rqlite:8.0.1" + Dex = "kotsadm/dex:2.37.0" ) ``` diff --git a/cmd/imagedeps/image-spec b/cmd/imagedeps/image-spec index 66a0ae252a..5bdafba0c5 100644 --- a/cmd/imagedeps/image-spec +++ b/cmd/imagedeps/image-spec @@ -1,5 +1,5 @@ -minio minio/minio -rqlite rqlite/rqlite ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ -dex ghcr.io/dexidp/dex +minio kotsadm/minio +rqlite kotsadm/rqlite +dex kotsadm/dex schemahero schemahero/schemahero ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ lvp replicated/local-volume-provider ^v([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ \ No newline at end of file diff --git a/cmd/imagedeps/main.go b/cmd/imagedeps/main.go index 97897a16fb..470116b44a 100644 --- a/cmd/imagedeps/main.go +++ b/cmd/imagedeps/main.go @@ -5,7 +5,6 @@ import ( "bytes" "fmt" "go/format" - "io/ioutil" "log" "os" "regexp" @@ -56,6 +55,9 @@ var ( replacers = []*replacer{ getMakefileReplacer("Makefile"), getMakefileReplacer("migrations/Makefile"), + getApkoFileReplacer("deploy/minio/apko.yaml", "minio"), + getApkoFileReplacer("deploy/rqlite/apko.yaml", "rqlite"), + getApkoFileReplacer("deploy/dex/apko.yaml", "dex"), } ) @@ -146,7 +148,7 @@ func generateOutput(filename, fileTemplate string, refs []*ImageRef, fn template return err } - if err := ioutil.WriteFile(filename, buff, 0644); err != nil { + if err := os.WriteFile(filename, buff, 0644); err != nil { return err } @@ -154,7 +156,7 @@ func generateOutput(filename, fileTemplate string, refs []*ImageRef, fn template } func (r *replacer) replace(refs []*ImageRef) error { - b, err := ioutil.ReadFile(r.path) + b, err := os.ReadFile(r.path) if err != nil { return errors.Wrap(err, "failed to read file") } @@ -168,7 +170,7 @@ func (r *replacer) replace(refs []*ImageRef) error { content = reg.ReplaceAllString(content, r.valueFn(ref)) } - if err := ioutil.WriteFile(r.path, []byte(content), 0644); err != nil { + if err := os.WriteFile(r.path, []byte(content), 0644); err != nil { return errors.Wrap(err, "failed to write file") } @@ -198,7 +200,7 @@ func getMakefileVarName(s string) string { return strings.ToUpper(strings.ReplaceAll(s, "-", "_")) + "_TAG" } -// converts a name from the input string into an a makefile variable name +// converts a name from the input string into an a dockerfile variable name // for example: foo_bar_baz -> FOO_BAR_BAZ func getDockerfileVarName(s string) string { return strings.ToUpper(strings.ReplaceAll(s, "-", "_")) + "_TAG" @@ -227,3 +229,15 @@ func getDockerfileReplacer(path string) *replacer { }, } } + +func getApkoFileReplacer(path string, pkg string) *replacer { + return &replacer{ + path: path, + regexFn: func(ir *ImageRef) string { + return fmt.Sprintf(`- %s~\d+\.\d+\.\d+`, ir.name) + }, + valueFn: func(ir *ImageRef) string { + return ir.GetApkoFileLine(ir.name) + }, + } +} diff --git a/cmd/imagedeps/tag-finder.go b/cmd/imagedeps/tag-finder.go index eac106a303..902e027436 100644 --- a/cmd/imagedeps/tag-finder.go +++ b/cmd/imagedeps/tag-finder.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "io" "net/http" "os" "path" @@ -14,6 +15,7 @@ import ( "github.com/google/go-github/v39/github" "github.com/heroku/docker-registry-client/registry" "golang.org/x/oauth2" + "gopkg.in/yaml.v2" ) type ImageRef struct { @@ -42,6 +44,11 @@ func (ir ImageRef) GetDockerfileLine() string { return fmt.Sprintf("ARG %s=%s", getDockerfileVarName(ir.name), ir.tag) } +// GetApkoFileLine generates a line of text intended for use in an Apko file. +func (ir ImageRef) GetApkoFileLine(pkg string) string { + return fmt.Sprintf("- %s~%s", pkg, ir.tag) +} + type getTagsFn func(string) ([]string, error) type getReleaseFn func(string, string) ([]*github.RepositoryRelease, error) type tagFinderFn func(inputLine string) (*ImageRef, error) @@ -119,19 +126,19 @@ func getTagFinder(opts ...func(c *configuration)) tagFinderFn { switch imageName { case minioReference: - latestReleaseTag, err = getLatestTagFromGithub(config.releaseFinder, "minio", "minio", matcherFn) + latestReleaseTag, err = getLatestTagFromWolfi("minio") if err != nil { - return nil, fmt.Errorf("failed to get release tag for minio/minio %w", err) + return nil, fmt.Errorf("failed to get latest minio tag from wolfi %w", err) } case dexReference: - latestReleaseTag, err = getLatestTagFromGithub(config.releaseFinder, "dexidp", "dex", matcherFn) + latestReleaseTag, err = getLatestTagFromWolfi("dex") if err != nil { - return nil, fmt.Errorf("failed to get release tag for dexidp/dex %w", err) + return nil, fmt.Errorf("failed to get latest dex tag from wolfi %w", err) } case rqliteReference: - latestReleaseTag, err = getLatestTagFromRegistry("rqlite/rqlite", config.repositoryTagsFinder, matcherFn) + latestReleaseTag, err = getLatestTagFromWolfi("rqlite") if err != nil { - return nil, fmt.Errorf("failed to get release tag for %s %w", imageName, err) + return nil, fmt.Errorf("failed to get latest rqlite tag from wolfi %w", err) } case schemaheroReference: latestReleaseTag, err = getLatestTagFromRegistry("schemahero/schemahero", config.repositoryTagsFinder, matcherFn) @@ -274,3 +281,31 @@ func getRegistryTags(untaggedRef string) ([]string, error) { } return tags, nil } + +func getLatestTagFromWolfi(pkg string) (string, error) { + resp, err := http.Get(fmt.Sprintf("https://raw.githubusercontent.com/wolfi-dev/os/main/%s.yaml", pkg)) + if err != nil { + return "", fmt.Errorf("failed to get %s.yaml from wolfi-dev/os: %w", pkg, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read body %w", err) + } + + var yamlData struct { + Package struct { + Version string `yaml:"version"` + } `yaml:"package"` + } + if err := yaml.Unmarshal(body, &yamlData); err != nil { + return "", fmt.Errorf("failed to unmarshal yaml %w", err) + } + + return yamlData.Package.Version, nil +} diff --git a/deploy/apko_melange_build.md b/deploy/apko_melange_build.md index c67583f632..4d682a598a 100644 --- a/deploy/apko_melange_build.md +++ b/deploy/apko_melange_build.md @@ -30,10 +30,10 @@ melange build melange.yaml --arch=x86_64 > 💡 Only building for your local platform makes builds faster, since it doesn't have to emulate with qemu. > If you're on an arm64 machine (e.g., Apple Silicon), use `--arch=aarch64` here and below. -Then, build the image from the newly built `kots` package, and the other packages needed by the image, using `apko`: +Then, build the image from the newly built `kotsadm` package, and the other packages needed by the image, using `apko`: ```sh -apko publish apko.yaml ttl.sh/kots --arch=x86_64 +apko publish apko.yaml ttl.sh/kotsadm --arch=x86_64 ``` This will print the image to stdout, so you can run it: diff --git a/deploy/dex.Dockerfile b/deploy/dex.Dockerfile deleted file mode 100644 index 7db7f0ff67..0000000000 --- a/deploy/dex.Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -ARG TAG=v2.32.0 -FROM ghcr.io/dexidp/dex:$TAG diff --git a/deploy/dex/apko.yaml b/deploy/dex/apko.yaml new file mode 100644 index 0000000000..e312449ce6 --- /dev/null +++ b/deploy/dex/apko.yaml @@ -0,0 +1,29 @@ +contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - dex~2.37.0 + - bash + - busybox + - wolfi-baselayout + +accounts: + groups: + - groupname: dex + gid: 1001 + users: + - username: dex + uid: 1001 + gid: 1001 + run-as: dex + +entrypoint: + command: docker-entrypoint + +cmd: dex serve /etc/dex/config.docker.yaml + +archs: + - x86_64 + - aarch64 diff --git a/deploy/kurl/kotsadm/template/base/Manifest b/deploy/kurl/kotsadm/template/base/Manifest index 0e57a590cf..f1fb302415 100644 --- a/deploy/kurl/kotsadm/template/base/Manifest +++ b/deploy/kurl/kotsadm/template/base/Manifest @@ -1,7 +1,7 @@ image kotsadm-migrations __KOTSADM_MIGRATIONS_IMAGE__ image kotsadm __KOTSADM_IMAGE__ image kurl-proxy __KURL_PROXY_IMAGE__ -image rqlite rqlite/rqlite:__RQLITE_TAG__ +image rqlite __RQLITE_IMAGE__ image dex __DEX_IMAGE__ asset kots.tar.gz __KOTSADM_BINARY__ diff --git a/deploy/kurl/kotsadm/template/base/rqlite.yaml b/deploy/kurl/kotsadm/template/base/rqlite.yaml index 5038d90722..a71e5d6ff2 100644 --- a/deploy/kurl/kotsadm/template/base/rqlite.yaml +++ b/deploy/kurl/kotsadm/template/base/rqlite.yaml @@ -50,7 +50,7 @@ spec: - kotsadm-rqlite topologyKey: "kubernetes.io/hostname" containers: - - image: rqlite/rqlite:__RQLITE_TAG__ + - image: __RQLITE_IMAGE__ name: rqlite args: - -disco-mode=dns diff --git a/deploy/kurl/kotsadm/template/generate.sh b/deploy/kurl/kotsadm/template/generate.sh index 234ea3cd60..a3f82b516a 100755 --- a/deploy/kurl/kotsadm/template/generate.sh +++ b/deploy/kurl/kotsadm/template/generate.sh @@ -28,7 +28,8 @@ function generate() { sed -i -e "s|__KOTSADM_BINARY__|$kotsadm_binary|g" "${dir}/Manifest" # The following environment variables will be exported by the .image.env file - find "$dir" -type f -exec sed -i -e "s|__RQLITE_TAG__|$RQLITE_TAG|g" {} \; + local rqlite_image="$kotsadm_image_registry/$kotsadm_image_namespace/rqlite:$RQLITE_TAG" + find "$dir" -type f -exec sed -i -e "s|__RQLITE_IMAGE__|$rqlite_image|g" {} \; local dex_image="$kotsadm_image_registry/$kotsadm_image_namespace/dex:$DEX_TAG" find "$dir" -type f -exec sed -i -e "s|__DEX_IMAGE__|$dex_image|g" {} \; } diff --git a/deploy/minio/apko.yaml b/deploy/minio/apko.yaml index 8ddf39807c..2937015a7b 100644 --- a/deploy/minio/apko.yaml +++ b/deploy/minio/apko.yaml @@ -4,7 +4,7 @@ contents: keyring: - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub packages: - - minio + - minio~0.20231101.183725 - mc - bash - busybox diff --git a/deploy/rqlite/apko.yaml b/deploy/rqlite/apko.yaml index d57a18497c..7097434175 100644 --- a/deploy/rqlite/apko.yaml +++ b/deploy/rqlite/apko.yaml @@ -4,7 +4,7 @@ contents: keyring: - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub packages: - - rqlite~7.21.4 + - rqlite~8.0.1 - rqlite-oci-entrypoint - bash - busybox diff --git a/migrations/Makefile b/migrations/Makefile index 011acb5480..3e38a4e8af 100644 --- a/migrations/Makefile +++ b/migrations/Makefile @@ -1,7 +1,6 @@ SHELL:=/bin/bash PROJECT_NAME ?= kotsadm-migrations -RQLITE_TAG ?= 7.21.4 -SCHEMAHERO_TAG ?= 0.16.0 +SCHEMAHERO_TAG ?= 0.17.0 .PHONY: schema-alpha schema-alpha: IMAGE = kotsadm/${PROJECT_NAME}:alpha @@ -12,8 +11,6 @@ schema-release: IMAGE = kotsadm/${PROJECT_NAME}:${GIT_TAG} schema-release: build_schema mkdir -p bin/docker-archive/${PROJECT_NAME} skopeo copy docker-daemon:kotsadm/${PROJECT_NAME}:${GIT_TAG} docker-archive:bin/docker-archive/${PROJECT_NAME}/${GIT_TAG} - mkdir -p bin/docker-archive/rqlite - skopeo copy docker://rqlite/rqlite:${RQLITE_TAG} docker-archive:bin/docker-archive/rqlite/${RQLITE_TAG} build_schema: docker build --pull --build-arg SCHEMAHERO_TAG=${SCHEMAHERO_TAG} -f deploy/Dockerfile -t ${IMAGE} . diff --git a/pkg/image/constants.go b/pkg/image/constants.go index f1b491f53d..fae7b78d0a 100644 --- a/pkg/image/constants.go +++ b/pkg/image/constants.go @@ -5,9 +5,9 @@ package image // image name. const ( - Minio = "minio/minio:RELEASE.2023-11-11T08-14-41Z" - Rqlite = "rqlite/rqlite:7.21.4" - Dex = "ghcr.io/dexidp/dex:v2.37.0" - Schemahero = "schemahero/schemahero:0.16.0" + Minio = "kotsadm/minio:0.20231101.183725" + Rqlite = "kotsadm/rqlite:8.0.1" + Dex = "kotsadm/dex:2.37.0" + Schemahero = "schemahero/schemahero:0.17.0" Lvp = "replicated/local-volume-provider:v0.5.5" ) diff --git a/pkg/kotsadm/minio.go b/pkg/kotsadm/minio.go index fb478a421f..f6e5a4e928 100644 --- a/pkg/kotsadm/minio.go +++ b/pkg/kotsadm/minio.go @@ -25,7 +25,8 @@ import ( ) var ( - MinioImageTagDateRegexp = regexp.MustCompile(`RELEASE\.(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z)`) + MinioChainguardImageTagRegexp = regexp.MustCompile(`:0\.\d+`) + MinioImageTagDateRegexp = regexp.MustCompile(`RELEASE\.(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z)`) // MigrateToMinioXlBeforeTime is the time that the minio version was released that removed the legacy backend // that we need to migrate from: https://github.com/minio/minio/releases/tag/RELEASE.2022-10-29T06-21-33Z MigrateToMinioXlBeforeTime = time.Date(2022, 10, 29, 6, 21, 33, 0, time.UTC) @@ -394,6 +395,12 @@ func IsMinioXlMigrationNeeded(clientset kubernetes.Interface, namespace string) // imageNeedsMinioXlMigration returns true if the minio image is older than the migrate before time (2022-10-29T06-21-33Z). func imageNeedsMinioXlMigration(minioImage string) (bool, error) { + isCGImage := len(MinioChainguardImageTagRegexp.FindStringSubmatch(minioImage)) > 0 + if isCGImage { + // minio images built with chainguard are all new and don't need to be migrated + return false, nil + } + existingImageTagDateMatch := MinioImageTagDateRegexp.FindStringSubmatch(minioImage) if len(existingImageTagDateMatch) != 2 { return false, errors.New("failed to parse existing image tag date") diff --git a/pkg/kotsadm/minio_test.go b/pkg/kotsadm/minio_test.go index f10fec56b1..6a65f5228b 100644 --- a/pkg/kotsadm/minio_test.go +++ b/pkg/kotsadm/minio_test.go @@ -91,6 +91,13 @@ func Test_IsMinioXlMigrationNeeded(t *testing.T) { wantMinioImage: "minio/minio:RELEASE.2023-02-10T18-48-39Z", wantErr: false, }, + { + name: "should not migrate image built with chainguard", + clientset: fake.NewSimpleClientset(minioStsWithImage("kotsadm/minio:0.20231101.183725")), + wantMigration: false, + wantMinioImage: "kotsadm/minio:0.20231101.183725", + wantErr: false, + }, { name: "should not migrate if no minio", clientset: fake.NewSimpleClientset(), diff --git a/pkg/kotsadm/objects/images.go b/pkg/kotsadm/objects/images.go index 1d097d2611..bfc799b3ce 100644 --- a/pkg/kotsadm/objects/images.go +++ b/pkg/kotsadm/objects/images.go @@ -18,9 +18,9 @@ func GetAdminConsoleImages(deployOptions types.DeployOptions) map[string]string rqliteTag, _ := image.GetTag(image.Rqlite) dexTag, _ := image.GetTag(image.Dex) - minioImage := fmt.Sprintf("minio/minio:%s", minioTag) - rqliteImage := fmt.Sprintf("rqlite/rqlite:%s", rqliteTag) - dexImage := fmt.Sprintf("kotsadm/dex:%s", dexTag) + minioImage := image.Minio + rqliteImage := image.Rqlite + dexImage := image.Dex if s := kotsadmversion.KotsadmPullSecret(deployOptions.Namespace, deployOptions.RegistryConfig); s != nil { minioImage = fmt.Sprintf("%s/minio:%s", kotsadmversion.KotsadmRegistry(deployOptions.RegistryConfig), minioTag) @@ -43,12 +43,11 @@ func GetAdminConsoleImages(deployOptions types.DeployOptions) map[string]string } func GetOriginalAdminConsoleImages(deployOptions types.DeployOptions) map[string]string { - dexTag, _ := image.GetTag(image.Dex) // dex image is special; we host a copy return map[string]string{ "kotsadm-migrations": fmt.Sprintf("kotsadm/kotsadm-migrations:%s", kotsadmversion.KotsadmTag(deployOptions.RegistryConfig)), "kotsadm": fmt.Sprintf("kotsadm/kotsadm:%s", kotsadmversion.KotsadmTag(deployOptions.RegistryConfig)), "minio": image.Minio, "rqlite": image.Rqlite, - "dex": fmt.Sprintf("kotsadm/dex:%s", dexTag), + "dex": image.Dex, } } diff --git a/pkg/snapshot/filesystem_minio.go b/pkg/snapshot/filesystem_minio.go index cfebe071bd..07e1048d2a 100644 --- a/pkg/snapshot/filesystem_minio.go +++ b/pkg/snapshot/filesystem_minio.go @@ -269,7 +269,7 @@ func fileSystemMinioDeploymentResource(clientset kubernetes.Interface, secretChe if err != nil { return nil, errors.Wrap(err, "failed to get minio image tag") } - minioImage := fmt.Sprintf("minio/minio:%s", minioTag) + minioImage := fmt.Sprintf("minio/minio:%s", minioTag) // TODO NOW: figure this out imagePullSecrets := []corev1.LocalObjectReference{} isKurl, err := kurl.IsKurl(clientset) From d5e93b14e28db40a4d6646171df9c0305b04460a Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Sun, 10 Dec 2023 00:53:22 +0000 Subject: [PATCH 10/22] add gomplate to dex --- deploy/dex/apko.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/dex/apko.yaml b/deploy/dex/apko.yaml index e312449ce6..8c92702b41 100644 --- a/deploy/dex/apko.yaml +++ b/deploy/dex/apko.yaml @@ -5,6 +5,7 @@ contents: - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub packages: - dex~2.37.0 + - gomplate - bash - busybox - wolfi-baselayout From 8f67c007eb3bcacf76bb993e6514508891a2ab08 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Sun, 10 Dec 2023 14:49:01 +0000 Subject: [PATCH 11/22] fix imagedeps tests --- cmd/imagedeps/main_test.go | 79 ++++++++----------- cmd/imagedeps/tag-finder.go | 16 +++- cmd/imagedeps/testdata/basic/input-spec | 1 - .../{filter-github => dex}/.image.env | 2 +- .../{filter-github => dex}/constants.go | 2 +- cmd/imagedeps/testdata/dex/input-spec | 1 + .../testdata/filter-github/input-spec | 1 - .../testdata/{basic => minio}/.image.env | 2 +- .../testdata/{basic => minio}/constants.go | 2 +- cmd/imagedeps/testdata/minio/input-spec | 1 + cmd/imagedeps/testdata/rqlite/.image.env | 2 +- cmd/imagedeps/testdata/rqlite/constants.go | 2 +- cmd/imagedeps/testdata/schemahero/.image.env | 2 +- .../testdata/schemahero/constants.go | 2 +- cmd/imagedeps/testdata/schemahero/input-spec | 2 +- .../replacers/expected/test.Dockerfile | 0 .../replacers/expected/test.mk | 0 .../replacers/input/test.Dockerfile | 0 .../replacers/input/test.mk | 0 .../testdata/with-overrides/.image.env | 5 -- .../testdata/with-overrides/constants.go | 10 --- .../testdata/with-overrides/input-spec | 2 - 22 files changed, 56 insertions(+), 78 deletions(-) delete mode 100644 cmd/imagedeps/testdata/basic/input-spec rename cmd/imagedeps/testdata/{filter-github => dex}/.image.env (83%) rename cmd/imagedeps/testdata/{filter-github => dex}/constants.go (81%) create mode 100644 cmd/imagedeps/testdata/dex/input-spec delete mode 100644 cmd/imagedeps/testdata/filter-github/input-spec rename cmd/imagedeps/testdata/{basic => minio}/.image.env (83%) rename cmd/imagedeps/testdata/{basic => minio}/constants.go (81%) create mode 100644 cmd/imagedeps/testdata/minio/input-spec rename cmd/imagedeps/testdata/{with-overrides => schemahero}/replacers/expected/test.Dockerfile (100%) rename cmd/imagedeps/testdata/{with-overrides => schemahero}/replacers/expected/test.mk (100%) rename cmd/imagedeps/testdata/{with-overrides => schemahero}/replacers/input/test.Dockerfile (100%) rename cmd/imagedeps/testdata/{with-overrides => schemahero}/replacers/input/test.mk (100%) delete mode 100644 cmd/imagedeps/testdata/with-overrides/.image.env delete mode 100644 cmd/imagedeps/testdata/with-overrides/constants.go delete mode 100644 cmd/imagedeps/testdata/with-overrides/input-spec diff --git a/cmd/imagedeps/main_test.go b/cmd/imagedeps/main_test.go index e1ac5eb665..26dea136a6 100644 --- a/cmd/imagedeps/main_test.go +++ b/cmd/imagedeps/main_test.go @@ -12,16 +12,24 @@ import ( "github.com/stretchr/testify/require" ) -var releaseTags = []string{ - "RELEASE.2022-06-11T19-55-32Z.fips", - "RELEASE.2021-09-09T21-37-06Z.xxx", - "RELEASE.2021-09-09T21-37-05Z", - "RELEASE.2021-09-09T21-37-04Z", -} -var semVerTags = []string{ - "0.12.7", "0.12.6", "0.12.5", - "0.12.4", "0.12.3", "0.12.2", -} +var ( + minioTag = "0.20231101.183725" + rqliteTag = "8.0.1" + dexTag = "2.37.0" + + schemaheroTags = []string{ + "0.13.2", + "0.13.1", + "0.12.7", + "0.12.2", + } + + lvpTags = []string{ + "v0.3.3", + "v0.3.2", + "v0.3.1", + } +) func makeReleases(tags []string) []*github.RepositoryRelease { var releases []*github.RepositoryRelease @@ -46,29 +54,21 @@ func TestFunctional(t *testing.T) { expectError bool }{ { - name: "basic", + name: "minio", fn: getTagFinder( - withGithubReleaseTagFinder( - func(_ string, _ string) ([]*github.RepositoryRelease, error) { - return makeReleases(releaseTags), nil + withWolfiGetTag( + func(_ string) (string, error) { + return minioTag, nil }, ), ), }, { - name: "with-overrides", + name: "schemahero", fn: getTagFinder( withRepoGetTags( func(_ string) ([]string, error) { - return []string{ - "0.13.2", "0.13.1", - "0.12.7", "0.12.2", - }, nil - }, - ), - withGithubReleaseTagFinder( - func(_ string, _ string) ([]*github.RepositoryRelease, error) { - return makeReleases(releaseTags), nil + return schemaheroTags, nil }, ), ), @@ -80,32 +80,19 @@ func TestFunctional(t *testing.T) { { name: "rqlite", fn: getTagFinder( - withRepoGetTags( - func(_ string) ([]string, error) { - return []string{ - "7.7.0", "7.6.1", "7.6.0", - "6.10.2", "6.10.1", "6.8.2", - }, nil - }, - ), - ), - }, - { - name: "filter-github", - fn: getTagFinder( - withGithubReleaseTagFinder( - func(_ string, _ string) ([]*github.RepositoryRelease, error) { - return makeReleases(releaseTags), nil + withWolfiGetTag( + func(_ string) (string, error) { + return rqliteTag, nil }, ), ), }, { - name: "schemahero", + name: "dex", fn: getTagFinder( - withRepoGetTags( - func(_ string) ([]string, error) { - return semVerTags, nil + withWolfiGetTag( + func(_ string) (string, error) { + return dexTag, nil }, ), ), @@ -115,9 +102,7 @@ func TestFunctional(t *testing.T) { fn: getTagFinder( withRepoGetTags( func(_ string) ([]string, error) { - return []string{ - "v0.3.3", - }, nil + return lvpTags, nil }, ), ), diff --git a/cmd/imagedeps/tag-finder.go b/cmd/imagedeps/tag-finder.go index 902e027436..85cad39fb1 100644 --- a/cmd/imagedeps/tag-finder.go +++ b/cmd/imagedeps/tag-finder.go @@ -49,6 +49,7 @@ func (ir ImageRef) GetApkoFileLine(pkg string) string { return fmt.Sprintf("- %s~%s", pkg, ir.tag) } +type getTagFn func(string) (string, error) type getTagsFn func(string) ([]string, error) type getReleaseFn func(string, string) ([]*github.RepositoryRelease, error) type tagFinderFn func(inputLine string) (*ImageRef, error) @@ -72,6 +73,7 @@ func getFilter(expression string) (filterFn, error) { type configuration struct { repositoryTagsFinder getTagsFn releaseFinder getReleaseFn + wolfiTagFinder getTagFn } // pass to getTagFinder to override the repository tag finder @@ -88,12 +90,20 @@ func withGithubReleaseTagFinder(fn getReleaseFn) func(c *configuration) { } } +// pass to getTagFinder to override the wolfi tag finder +func withWolfiGetTag(fn getTagFn) func(c *configuration) { + return func(c *configuration) { + c.wolfiTagFinder = fn + } +} + // returns a tag finder function that returns information about an image and it's latest tag. func getTagFinder(opts ...func(c *configuration)) tagFinderFn { // set defaults config := configuration{ repositoryTagsFinder: getRegistryTags, releaseFinder: getReleases, + wolfiTagFinder: getLatestTagFromWolfi, } // apply options for _, opt := range opts { @@ -126,17 +136,17 @@ func getTagFinder(opts ...func(c *configuration)) tagFinderFn { switch imageName { case minioReference: - latestReleaseTag, err = getLatestTagFromWolfi("minio") + latestReleaseTag, err = config.wolfiTagFinder("minio") if err != nil { return nil, fmt.Errorf("failed to get latest minio tag from wolfi %w", err) } case dexReference: - latestReleaseTag, err = getLatestTagFromWolfi("dex") + latestReleaseTag, err = config.wolfiTagFinder("dex") if err != nil { return nil, fmt.Errorf("failed to get latest dex tag from wolfi %w", err) } case rqliteReference: - latestReleaseTag, err = getLatestTagFromWolfi("rqlite") + latestReleaseTag, err = config.wolfiTagFinder("rqlite") if err != nil { return nil, fmt.Errorf("failed to get latest rqlite tag from wolfi %w", err) } diff --git a/cmd/imagedeps/testdata/basic/input-spec b/cmd/imagedeps/testdata/basic/input-spec deleted file mode 100644 index 7c0313fd1f..0000000000 --- a/cmd/imagedeps/testdata/basic/input-spec +++ /dev/null @@ -1 +0,0 @@ -minio minio/minio \ No newline at end of file diff --git a/cmd/imagedeps/testdata/filter-github/.image.env b/cmd/imagedeps/testdata/dex/.image.env similarity index 83% rename from cmd/imagedeps/testdata/filter-github/.image.env rename to cmd/imagedeps/testdata/dex/.image.env index 1055c8ca12..3077a28796 100644 --- a/cmd/imagedeps/testdata/filter-github/.image.env +++ b/cmd/imagedeps/testdata/dex/.image.env @@ -1,4 +1,4 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -MINIO_TAG='RELEASE.2021-09-09T21-37-06Z.xxx' \ No newline at end of file +DEX_TAG='2.37.0' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/filter-github/constants.go b/cmd/imagedeps/testdata/dex/constants.go similarity index 81% rename from cmd/imagedeps/testdata/filter-github/constants.go rename to cmd/imagedeps/testdata/dex/constants.go index 4c28919a57..0a50c62152 100644 --- a/cmd/imagedeps/testdata/filter-github/constants.go +++ b/cmd/imagedeps/testdata/dex/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Minio = "minio/minio:RELEASE.2021-09-09T21-37-06Z.xxx" + Dex = "kotsadm/dex:2.37.0" ) diff --git a/cmd/imagedeps/testdata/dex/input-spec b/cmd/imagedeps/testdata/dex/input-spec new file mode 100644 index 0000000000..a48afc0a87 --- /dev/null +++ b/cmd/imagedeps/testdata/dex/input-spec @@ -0,0 +1 @@ +dex kotsadm/dex \ No newline at end of file diff --git a/cmd/imagedeps/testdata/filter-github/input-spec b/cmd/imagedeps/testdata/filter-github/input-spec deleted file mode 100644 index 80c831f8a5..0000000000 --- a/cmd/imagedeps/testdata/filter-github/input-spec +++ /dev/null @@ -1 +0,0 @@ -minio minio/minio xxx$ \ No newline at end of file diff --git a/cmd/imagedeps/testdata/basic/.image.env b/cmd/imagedeps/testdata/minio/.image.env similarity index 83% rename from cmd/imagedeps/testdata/basic/.image.env rename to cmd/imagedeps/testdata/minio/.image.env index 2999afe7b5..d97660f84f 100644 --- a/cmd/imagedeps/testdata/basic/.image.env +++ b/cmd/imagedeps/testdata/minio/.image.env @@ -1,4 +1,4 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -MINIO_TAG='RELEASE.2022-06-11T19-55-32Z.fips' \ No newline at end of file +MINIO_TAG='0.20231101.183725' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/basic/constants.go b/cmd/imagedeps/testdata/minio/constants.go similarity index 81% rename from cmd/imagedeps/testdata/basic/constants.go rename to cmd/imagedeps/testdata/minio/constants.go index 240e8c0597..d68ed5eead 100644 --- a/cmd/imagedeps/testdata/basic/constants.go +++ b/cmd/imagedeps/testdata/minio/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Minio = "minio/minio:RELEASE.2022-06-11T19-55-32Z.fips" + Minio = "kotsadm/minio:0.20231101.183725" ) diff --git a/cmd/imagedeps/testdata/minio/input-spec b/cmd/imagedeps/testdata/minio/input-spec new file mode 100644 index 0000000000..128028b8c1 --- /dev/null +++ b/cmd/imagedeps/testdata/minio/input-spec @@ -0,0 +1 @@ +minio kotsadm/minio \ No newline at end of file diff --git a/cmd/imagedeps/testdata/rqlite/.image.env b/cmd/imagedeps/testdata/rqlite/.image.env index 992292b61d..73a2d7b8ac 100644 --- a/cmd/imagedeps/testdata/rqlite/.image.env +++ b/cmd/imagedeps/testdata/rqlite/.image.env @@ -1,4 +1,4 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -RQLITE_TAG='7.7.0' \ No newline at end of file +RQLITE_TAG='8.0.1' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/rqlite/constants.go b/cmd/imagedeps/testdata/rqlite/constants.go index ab99db1066..0318baee55 100644 --- a/cmd/imagedeps/testdata/rqlite/constants.go +++ b/cmd/imagedeps/testdata/rqlite/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Rqlite = "rqlite/rqlite:7.7.0" + Rqlite = "rqlite/rqlite:8.0.1" ) diff --git a/cmd/imagedeps/testdata/schemahero/.image.env b/cmd/imagedeps/testdata/schemahero/.image.env index 931c6000ca..4615bd4e65 100644 --- a/cmd/imagedeps/testdata/schemahero/.image.env +++ b/cmd/imagedeps/testdata/schemahero/.image.env @@ -1,4 +1,4 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -SCHEMAHERO_TAG='0.12.7' \ No newline at end of file +SCHEMAHERO_TAG='0.13.2' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/schemahero/constants.go b/cmd/imagedeps/testdata/schemahero/constants.go index 50a8eca1f2..3b8bc02ce8 100644 --- a/cmd/imagedeps/testdata/schemahero/constants.go +++ b/cmd/imagedeps/testdata/schemahero/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Schemahero = "schemahero/schemahero:0.12.7" + Schemahero = "schemahero/schemahero:0.13.2" ) diff --git a/cmd/imagedeps/testdata/schemahero/input-spec b/cmd/imagedeps/testdata/schemahero/input-spec index 84d47c6bd9..d614df9429 100644 --- a/cmd/imagedeps/testdata/schemahero/input-spec +++ b/cmd/imagedeps/testdata/schemahero/input-spec @@ -1 +1 @@ -schemahero schemahero/schemahero ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ \ No newline at end of file +schemahero schemahero/schemahero ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ diff --git a/cmd/imagedeps/testdata/with-overrides/replacers/expected/test.Dockerfile b/cmd/imagedeps/testdata/schemahero/replacers/expected/test.Dockerfile similarity index 100% rename from cmd/imagedeps/testdata/with-overrides/replacers/expected/test.Dockerfile rename to cmd/imagedeps/testdata/schemahero/replacers/expected/test.Dockerfile diff --git a/cmd/imagedeps/testdata/with-overrides/replacers/expected/test.mk b/cmd/imagedeps/testdata/schemahero/replacers/expected/test.mk similarity index 100% rename from cmd/imagedeps/testdata/with-overrides/replacers/expected/test.mk rename to cmd/imagedeps/testdata/schemahero/replacers/expected/test.mk diff --git a/cmd/imagedeps/testdata/with-overrides/replacers/input/test.Dockerfile b/cmd/imagedeps/testdata/schemahero/replacers/input/test.Dockerfile similarity index 100% rename from cmd/imagedeps/testdata/with-overrides/replacers/input/test.Dockerfile rename to cmd/imagedeps/testdata/schemahero/replacers/input/test.Dockerfile diff --git a/cmd/imagedeps/testdata/with-overrides/replacers/input/test.mk b/cmd/imagedeps/testdata/schemahero/replacers/input/test.mk similarity index 100% rename from cmd/imagedeps/testdata/with-overrides/replacers/input/test.mk rename to cmd/imagedeps/testdata/schemahero/replacers/input/test.mk diff --git a/cmd/imagedeps/testdata/with-overrides/.image.env b/cmd/imagedeps/testdata/with-overrides/.image.env deleted file mode 100644 index f8ab2a50cc..0000000000 --- a/cmd/imagedeps/testdata/with-overrides/.image.env +++ /dev/null @@ -1,5 +0,0 @@ -# Generated file, do not modify. This file is generated from a text file containing a list of images. The -# most recent tag is interpolated from the source repository and used to generate a fully qualified image -# name. -MINIO_TAG='RELEASE.2022-06-11T19-55-32Z.fips' -SCHEMAHERO_TAG='0.13.2' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/with-overrides/constants.go b/cmd/imagedeps/testdata/with-overrides/constants.go deleted file mode 100644 index 99df410f88..0000000000 --- a/cmd/imagedeps/testdata/with-overrides/constants.go +++ /dev/null @@ -1,10 +0,0 @@ -package image - -// Generated file, do not modify. This file is generated from a text file containing a list of images. The -// most recent tag is interpolated from the source repository and used to generate a fully qualified -// image name. - -const ( - Minio = "minio/minio:RELEASE.2022-06-11T19-55-32Z.fips" - Schemahero = "schemahero/schemahero:0.13.2" -) diff --git a/cmd/imagedeps/testdata/with-overrides/input-spec b/cmd/imagedeps/testdata/with-overrides/input-spec deleted file mode 100644 index b2a4505a31..0000000000 --- a/cmd/imagedeps/testdata/with-overrides/input-spec +++ /dev/null @@ -1,2 +0,0 @@ -minio minio/minio -schemahero schemahero/schemahero ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ From 6f83d5b54bd3e5a6653627554dbf71c9f52776ad Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Sun, 10 Dec 2023 15:49:47 +0000 Subject: [PATCH 12/22] fix minio image and dex command --- pkg/identity/deploy/deploy.go | 2 +- pkg/snapshot/filesystem_minio.go | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/identity/deploy/deploy.go b/pkg/identity/deploy/deploy.go index b7b050a2b8..90d09a795d 100644 --- a/pkg/identity/deploy/deploy.go +++ b/pkg/identity/deploy/deploy.go @@ -462,7 +462,7 @@ func deploymentResource(issuerURL, configChecksum string, options Options) (*app Image: image, ImagePullPolicy: corev1.PullIfNotPresent, Name: "dex", - Command: []string{"/usr/local/bin/dex", "serve", "/etc/dex/cfg/dexConfig.yaml"}, + Command: []string{"dex", "serve", "/etc/dex/cfg/dexConfig.yaml"}, Ports: []corev1.ContainerPort{ {Name: "http", ContainerPort: 5556}, }, diff --git a/pkg/snapshot/filesystem_minio.go b/pkg/snapshot/filesystem_minio.go index 07e1048d2a..c188d94bf9 100644 --- a/pkg/snapshot/filesystem_minio.go +++ b/pkg/snapshot/filesystem_minio.go @@ -269,7 +269,12 @@ func fileSystemMinioDeploymentResource(clientset kubernetes.Interface, secretChe if err != nil { return nil, errors.Wrap(err, "failed to get minio image tag") } - minioImage := fmt.Sprintf("minio/minio:%s", minioTag) // TODO NOW: figure this out + + minioImage := fmt.Sprintf("kotsadm/minio:%s", minioTag) + if strings.HasPrefix(minioTag, "RELEASE.") { + minioImage = fmt.Sprintf("minio/minio:%s", minioTag) + } + imagePullSecrets := []corev1.LocalObjectReference{} isKurl, err := kurl.IsKurl(clientset) From bc6ed9033962d3ad4003fbf2cf538142d7bde501 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Mon, 11 Dec 2023 15:25:09 +0000 Subject: [PATCH 13/22] refactor --- .../actions/build-push-dex-image/action.yml | 28 ------------ .../build-push-image-with-apko/action.yml | 43 +++++++++++++++++++ .../actions/build-push-minio-image/action.yml | 28 ------------ .../build-push-rqlite-image/action.yml | 28 ------------ .github/workflows/alpha.yaml | 38 ++++++---------- .github/workflows/build-test.yaml | 11 +++-- .github/workflows/release.yaml | 12 +++--- cmd/imagedeps/testdata/rqlite/constants.go | 2 +- cmd/imagedeps/testdata/rqlite/input-spec | 2 +- migrations/kustomize/overlays/dev/rqlite.yaml | 4 +- .../kustomize/overlays/okteto/rqlite.yaml | 4 +- 11 files changed, 76 insertions(+), 124 deletions(-) delete mode 100644 .github/actions/build-push-dex-image/action.yml create mode 100644 .github/actions/build-push-image-with-apko/action.yml delete mode 100644 .github/actions/build-push-minio-image/action.yml delete mode 100644 .github/actions/build-push-rqlite-image/action.yml diff --git a/.github/actions/build-push-dex-image/action.yml b/.github/actions/build-push-dex-image/action.yml deleted file mode 100644 index ec70b426ae..0000000000 --- a/.github/actions/build-push-dex-image/action.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: 'Build and push dex image' -description: 'Composite action for building and pushing dex image' -inputs: - image-name: - description: 'Full destination dex image name' - required: true - - registry-username: - description: 'Username to login to registry' - default: '' - required: false - - registry-password: - description: 'Password to login to registry' - default: '' - required: false - -runs: - using: "composite" - steps: - - uses: chainguard-images/actions/apko-publish@main - with: - config: deploy/dex/apko.yaml - archs: x86_64 - tag: ${{ inputs.image-name }} - vcs-url: true - generic-user: ${{ inputs.registry-username }} - generic-pass: ${{ inputs.registry-password }} diff --git a/.github/actions/build-push-image-with-apko/action.yml b/.github/actions/build-push-image-with-apko/action.yml new file mode 100644 index 0000000000..adfe9a5428 --- /dev/null +++ b/.github/actions/build-push-image-with-apko/action.yml @@ -0,0 +1,43 @@ +name: 'Build and push images with apko' +description: 'Composite action for building and pushing images with apko' +inputs: + apko-config: + description: 'Path to apko config' + required: true + + image-name: + description: 'Full destination image name' + required: true + + registry-username: + description: 'Username to login to registry' + default: '' + required: false + + registry-password: + description: 'Password to login to registry' + default: '' + required: false + +runs: + using: "composite" + steps: + - id: check-image-exists + shell: bash + run: | + set -euo pipefail + if docker manifest inspect ${{ inputs.image-name }} > /dev/null 2>&1; then + echo "image-exists=true" >> $GITHUB_ENV + else + echo "image-exists=false" >> $GITHUB_ENV + fi + + - uses: chainguard-images/actions/apko-publish@main + if: steps.check-image-exists.outputs.image-exists == 'false' + with: + config: ${{ inputs.apko-config }} + archs: x86_64 + tag: ${{ inputs.image-name }} + vcs-url: true + generic-user: ${{ inputs.registry-username }} + generic-pass: ${{ inputs.registry-password }} diff --git a/.github/actions/build-push-minio-image/action.yml b/.github/actions/build-push-minio-image/action.yml deleted file mode 100644 index 527a183918..0000000000 --- a/.github/actions/build-push-minio-image/action.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: 'Build and push minio image' -description: 'Composite action for building and pushing minio image' -inputs: - image-name: - description: 'Full destination minio image name' - required: true - - registry-username: - description: 'Username to login to registry' - default: '' - required: false - - registry-password: - description: 'Password to login to registry' - default: '' - required: false - -runs: - using: "composite" - steps: - - uses: chainguard-images/actions/apko-publish@main - with: - config: deploy/minio/apko.yaml - archs: x86_64 - tag: ${{ inputs.image-name }} - vcs-url: true - generic-user: ${{ inputs.registry-username }} - generic-pass: ${{ inputs.registry-password }} diff --git a/.github/actions/build-push-rqlite-image/action.yml b/.github/actions/build-push-rqlite-image/action.yml deleted file mode 100644 index 323ae19c15..0000000000 --- a/.github/actions/build-push-rqlite-image/action.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: 'Build and push rqlite image' -description: 'Composite action for building and pushing rqlite image' -inputs: - image-name: - description: 'Full destination rqlite image name' - required: true - - registry-username: - description: 'Username to login to registry' - default: '' - required: false - - registry-password: - description: 'Password to login to registry' - default: '' - required: false - -runs: - using: "composite" - steps: - - uses: chainguard-images/actions/apko-publish@main - with: - config: deploy/rqlite/apko.yaml - archs: x86_64 - tag: ${{ inputs.image-name }} - vcs-url: true - generic-user: ${{ inputs.registry-username }} - generic-pass: ${{ inputs.registry-password }} diff --git a/.github/workflows/alpha.yaml b/.github/workflows/alpha.yaml index 0a721884ec..8ac5131e43 100644 --- a/.github/workflows/alpha.yaml +++ b/.github/workflows/alpha.yaml @@ -43,14 +43,10 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 - - name: Read image tags from env file - uses: falti/dotenv-action@v1 - id: dotenv - with: - path: .image.env - - uses: ./.github/actions/build-push-rqlite-image + - uses: ./.github/actions/build-push-image-with-apko with: - image-name: index.docker.io/kotsadm/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} + apko-config: deploy/rqlite/apko.yaml + image-name: index.docker.io/kotsadm/rqlite:alpha registry-username: ${{ secrets.DOCKERHUB_USER }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} @@ -59,14 +55,10 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 - - name: Read image tags from env file - uses: falti/dotenv-action@v1 - id: dotenv + - uses: ./.github/actions/build-push-image-with-apko with: - path: .image.env - - uses: ./.github/actions/build-push-minio-image - with: - image-name: index.docker.io/kotsadm/minio:${{ steps.dotenv.outputs.MINIO_TAG }} + apko-config: deploy/minio/apko.yaml + image-name: index.docker.io/kotsadm/minio:alpha registry-username: ${{ secrets.DOCKERHUB_USER }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} @@ -75,14 +67,10 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 - - name: Read image tags from env file - uses: falti/dotenv-action@v1 - id: dotenv - with: - path: .image.env - - uses: ./.github/actions/build-push-dex-image + - uses: ./.github/actions/build-push-image-with-apko with: - image-name: index.docker.io/kotsadm/dex:${{ steps.dotenv.outputs.DEX_TAG }} + apko-config: deploy/dex/apko.yaml + image-name: index.docker.io/kotsadm/dex:alpha registry-username: ${{ secrets.DOCKERHUB_USER }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} @@ -91,7 +79,7 @@ jobs: runs-on: ubuntu-20.04 needs: [generate-tag] permissions: - id-token: write # required to be able to assume the GCP SA identity to pull Chainguard packages. + id-token: write # required to be able to assume the GCP SA identity to pull private Chainguard packages. steps: - uses: actions/checkout@v4 - uses: ./.github/actions/build-push-kotsadm-image @@ -132,7 +120,7 @@ jobs: id: scan uses: aquasecurity/trivy-action@master with: - image-ref: "docker.io/kotsadm/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}" + image-ref: "docker.io/kotsadm/rqlite:alpha" format: 'template' template: '@/contrib/sarif.tpl' output: 'rqlite-scan-output.sarif' @@ -159,7 +147,7 @@ jobs: id: scan uses: aquasecurity/trivy-action@master with: - image-ref: "docker.io/kotsadm/minio:${{ steps.dotenv.outputs.MINIO_TAG }}" + image-ref: "docker.io/kotsadm/minio:alpha" format: 'template' template: '@/contrib/sarif.tpl' output: 'minio-scan-output.sarif' @@ -187,7 +175,7 @@ jobs: id: scan uses: aquasecurity/trivy-action@master with: - image-ref: "docker.io/kotsadm/dex:${{ steps.dotenv.outputs.dex_tag }}" + image-ref: "docker.io/kotsadm/dex:alpha" format: 'template' template: '@/contrib/sarif.tpl' output: 'dex-scan-output.sarif' diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 187b34a0ba..869c668c72 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -268,7 +268,7 @@ jobs: runs-on: ubuntu-20.04 needs: [ can-run-ci, generate-tag ] permissions: - id-token: write # required to be able to assume the GCP SA identity to pull Chainguard packages. + id-token: write # required to be able to assume the GCP SA identity to pull private Chainguard packages. steps: - uses: actions/setup-go@v4 with: @@ -427,8 +427,9 @@ jobs: with: path: .image.env - - uses: ./.github/actions/build-push-minio-image + - uses: ./.github/actions/build-push-image-with-apko with: + apko-config: deploy/minio/apko.yaml image-name: ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.MINIO_TAG }} @@ -444,8 +445,9 @@ jobs: with: path: .image.env - - uses: ./.github/actions/build-push-rqlite-image + - uses: ./.github/actions/build-push-image-with-apko with: + apko-config: deploy/rqlite/apko.yaml image-name: ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} @@ -461,8 +463,9 @@ jobs: with: path: .image.env - - uses: ./.github/actions/build-push-dex-image + - uses: ./.github/actions/build-push-image-with-apko with: + apko-config: deploy/dex/apko.yaml image-name: ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 29a9633010..5f78c7a0f3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -6,7 +6,6 @@ on: - "v*.*.*" branches: - main - - build-rqlite-with-apko jobs: generate-tag: @@ -161,7 +160,7 @@ jobs: runs-on: ubuntu-20.04 needs: [generate-tag] permissions: - id-token: write # required to be able to assume the GCP SA identity to pull Chainguard packages. + id-token: write # required to be able to assume the GCP SA identity to pull private Chainguard packages. steps: - name: Checkout uses: actions/checkout@v4 @@ -182,20 +181,23 @@ jobs: registry-username: ${{ secrets.DOCKERHUB_USER }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} - - uses: ./.github/actions/build-push-minio-image + - uses: ./.github/actions/build-push-image-with-apko with: + apko-config: deploy/minio/apko.yaml image-name: index.docker.io/kotsadm/minio:${{ steps.dotenv.outputs.MINIO_TAG }} registry-username: ${{ secrets.DOCKERHUB_USER }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} - - uses: ./.github/actions/build-push-rqlite-image + - uses: ./.github/actions/build-push-image-with-apko with: + apko-config: deploy/rqlite/apko.yaml image-name: index.docker.io/kotsadm/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} registry-username: ${{ secrets.DOCKERHUB_USER }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} - - uses: ./.github/actions/build-push-dex-image + - uses: ./.github/actions/build-push-image-with-apko with: + apko-config: deploy/dex/apko.yaml image-name: index.docker.io/kotsadm/dex:${{ steps.dotenv.outputs.DEX_TAG }} registry-username: ${{ secrets.DOCKERHUB_USER }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} diff --git a/cmd/imagedeps/testdata/rqlite/constants.go b/cmd/imagedeps/testdata/rqlite/constants.go index 0318baee55..decfb545f0 100644 --- a/cmd/imagedeps/testdata/rqlite/constants.go +++ b/cmd/imagedeps/testdata/rqlite/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Rqlite = "rqlite/rqlite:8.0.1" + Rqlite = "kotsadm/rqlite:8.0.1" ) diff --git a/cmd/imagedeps/testdata/rqlite/input-spec b/cmd/imagedeps/testdata/rqlite/input-spec index c4772a98f3..fb3664fced 100644 --- a/cmd/imagedeps/testdata/rqlite/input-spec +++ b/cmd/imagedeps/testdata/rqlite/input-spec @@ -1 +1 @@ -rqlite rqlite/rqlite ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ +rqlite kotsadm/rqlite ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ diff --git a/migrations/kustomize/overlays/dev/rqlite.yaml b/migrations/kustomize/overlays/dev/rqlite.yaml index e23b13defe..4bf62ace05 100644 --- a/migrations/kustomize/overlays/dev/rqlite.yaml +++ b/migrations/kustomize/overlays/dev/rqlite.yaml @@ -59,8 +59,8 @@ spec: spec: containers: - name: rqlite - image: rqlite/rqlite:7.9.2 - imagePullPolicy: IfNotPresent + image: kotsadm/rqlite:alpha + imagePullPolicy: Always args: - -disco-mode=dns - -disco-config={"name":"kotsadm-rqlite-headless"} diff --git a/migrations/kustomize/overlays/okteto/rqlite.yaml b/migrations/kustomize/overlays/okteto/rqlite.yaml index e23b13defe..4bf62ace05 100644 --- a/migrations/kustomize/overlays/okteto/rqlite.yaml +++ b/migrations/kustomize/overlays/okteto/rqlite.yaml @@ -59,8 +59,8 @@ spec: spec: containers: - name: rqlite - image: rqlite/rqlite:7.9.2 - imagePullPolicy: IfNotPresent + image: kotsadm/rqlite:alpha + imagePullPolicy: Always args: - -disco-mode=dns - -disco-config={"name":"kotsadm-rqlite-headless"} From 8a067f44d085d437d290eedbcf115f85af2c3d16 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Mon, 11 Dec 2023 15:25:52 +0000 Subject: [PATCH 14/22] run regression tests --- .github/workflows/release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5f78c7a0f3..dee8bd7b7b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -6,6 +6,7 @@ on: - "v*.*.*" branches: - main + - build-rqlite-with-apko jobs: generate-tag: From c0582e6f6d2d29a7d67d84b23f96b5af8d636bbd Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Mon, 11 Dec 2023 15:36:07 +0000 Subject: [PATCH 15/22] fix image-exists github output --- .github/actions/build-push-image-with-apko/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-push-image-with-apko/action.yml b/.github/actions/build-push-image-with-apko/action.yml index adfe9a5428..38cc422a69 100644 --- a/.github/actions/build-push-image-with-apko/action.yml +++ b/.github/actions/build-push-image-with-apko/action.yml @@ -27,9 +27,9 @@ runs: run: | set -euo pipefail if docker manifest inspect ${{ inputs.image-name }} > /dev/null 2>&1; then - echo "image-exists=true" >> $GITHUB_ENV + echo "image-exists='true'" >> "$GITHUB_ENV" else - echo "image-exists=false" >> $GITHUB_ENV + echo "image-exists='false'" >> "$GITHUB_ENV" fi - uses: chainguard-images/actions/apko-publish@main From 42da70b4a259d63fcb3cadcaaec0aadbd53c2ddb Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Mon, 11 Dec 2023 15:40:06 +0000 Subject: [PATCH 16/22] fix image-exists github output --- .github/actions/build-push-image-with-apko/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/build-push-image-with-apko/action.yml b/.github/actions/build-push-image-with-apko/action.yml index 38cc422a69..ba3dff1343 100644 --- a/.github/actions/build-push-image-with-apko/action.yml +++ b/.github/actions/build-push-image-with-apko/action.yml @@ -27,13 +27,13 @@ runs: run: | set -euo pipefail if docker manifest inspect ${{ inputs.image-name }} > /dev/null 2>&1; then - echo "image-exists='true'" >> "$GITHUB_ENV" + echo "image-exists=true" >> "$GITHUB_OUTPUT" else - echo "image-exists='false'" >> "$GITHUB_ENV" + echo "image-exists=false" >> "$GITHUB_OUTPUT" fi - uses: chainguard-images/actions/apko-publish@main - if: steps.check-image-exists.outputs.image-exists == 'false' + if: ${{ steps.check-image-exists.outputs.image-exists == 'false' }} with: config: ${{ inputs.apko-config }} archs: x86_64 From 02e450c7fa7983d9b2478ecb3cd6205abd2bdd5f Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Mon, 11 Dec 2023 16:01:14 +0000 Subject: [PATCH 17/22] refactor release workflow --- .github/workflows/regression.yaml | 9 +-- .github/workflows/release.yaml | 97 +++++++++++++++++++++---------- Makefile | 3 + migrations/Makefile | 2 - 4 files changed, 70 insertions(+), 41 deletions(-) diff --git a/.github/workflows/regression.yaml b/.github/workflows/regression.yaml index cb303f7a83..666823560f 100644 --- a/.github/workflows/regression.yaml +++ b/.github/workflows/regression.yaml @@ -90,15 +90,10 @@ jobs: with: name: kots path: ./automation/jumpbox/bin - - name: Download kotsadm image + - name: Download kotsadm release uses: actions/download-artifact@v3 with: - name: kotsadm-image - path: ./automation/jumpbox/docker-archive - - name: Download migrations - uses: actions/download-artifact@v3 - with: - name: migrations-image + name: kotsadm-release path: ./automation/jumpbox/docker-archive - name: Make kotsadm airgap archive with minio image working-directory: automation/jumpbox diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index dee8bd7b7b..b81fd1f77d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -63,7 +63,7 @@ jobs: echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" - build-schema-migrations: + build-kotsadm-migrations: runs-on: ubuntu-20.04 needs: [generate-tag] steps: @@ -80,11 +80,6 @@ jobs: GIT_TAG: ${{ needs.generate-tag.outputs.tag }} DOCKER_CONFIG: ./.docker run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make -C migrations schema-release - - name: Upload airgap image - uses: actions/upload-artifact@v3 - with: - name: migrations-image - path: ./migrations/bin/docker-archive build-web: runs-on: ubuntu-20.04 @@ -157,11 +152,8 @@ jobs: name: kots path: ./bin/kots - build-kotsadm: + build-minio: runs-on: ubuntu-20.04 - needs: [generate-tag] - permissions: - id-token: write # required to be able to assume the GCP SA identity to pull private Chainguard packages. steps: - name: Checkout uses: actions/checkout@v4 @@ -172,16 +164,6 @@ jobs: with: path: .image.env - - uses: ./.github/actions/build-push-kotsadm-image - with: - chainguard-gcp-wif-pool: ${{ secrets.CHAINGUARD_GCP_WIF_POOL }} - chainguard-gcp-sa: ${{ secrets.CHAINGUARD_GCP_SA }} - chainguard-gcp-project-id: ${{ secrets.CHAINGUARD_GCP_PROJECT_ID }} - image-name: index.docker.io/kotsadm/kotsadm:${{ needs.generate-tag.outputs.tag }} - git-tag: ${{ needs.generate-tag.outputs.tag }} - registry-username: ${{ secrets.DOCKERHUB_USER }} - registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} - - uses: ./.github/actions/build-push-image-with-apko with: apko-config: deploy/minio/apko.yaml @@ -189,6 +171,18 @@ jobs: registry-username: ${{ secrets.DOCKERHUB_USER }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + build-rqlite: + runs-on: ubuntu-20.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Read image tags from env file + uses: falti/dotenv-action@v1 + id: dotenv + with: + path: .image.env + - uses: ./.github/actions/build-push-image-with-apko with: apko-config: deploy/rqlite/apko.yaml @@ -196,6 +190,18 @@ jobs: registry-username: ${{ secrets.DOCKERHUB_USER }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + build-dex: + runs-on: ubuntu-20.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Read image tags from env file + uses: falti/dotenv-action@v1 + id: dotenv + with: + path: .image.env + - uses: ./.github/actions/build-push-image-with-apko with: apko-config: deploy/dex/apko.yaml @@ -203,16 +209,48 @@ jobs: registry-username: ${{ secrets.DOCKERHUB_USER }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + build-kotsadm: + runs-on: ubuntu-20.04 + needs: [generate-tag] + permissions: + id-token: write # required to be able to assume the GCP SA identity to pull private Chainguard packages. + steps: + - name: Checkout + uses: actions/checkout@v4 + + - uses: ./.github/actions/build-push-kotsadm-image + with: + chainguard-gcp-wif-pool: ${{ secrets.CHAINGUARD_GCP_WIF_POOL }} + chainguard-gcp-sa: ${{ secrets.CHAINGUARD_GCP_SA }} + chainguard-gcp-project-id: ${{ secrets.CHAINGUARD_GCP_PROJECT_ID }} + image-name: index.docker.io/kotsadm/kotsadm:${{ needs.generate-tag.outputs.tag }} + git-tag: ${{ needs.generate-tag.outputs.tag }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + + build-release: + runs-on: ubuntu-20.04 + needs: [build-kotsadm-migrations, build-kotsadm, build-minio, build-rqlite, build-dex] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Read image tags from env file + uses: falti/dotenv-action@v1 + id: dotenv + with: + path: .image.env + - name: Build tagged release env: GIT_TAG: ${{ needs.generate-tag.outputs.tag }} DOCKER_CONFIG: ./.docker run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make build-release - - name: Upload airgap image + - name: Upload kotsadm release uses: actions/upload-artifact@v3 with: - name: kotsadm-image + name: kotsadm-release path: ./bin/docker-archive goreleaser: @@ -275,7 +313,7 @@ jobs: generate-kurl-addon: runs-on: ubuntu-20.04 - needs: [ generate-tag, build-kurl-proxy, build-schema-migrations, build-kots, build-kotsadm ] + needs: [ generate-tag, build-kurl-proxy, build-kots, build-release ] outputs: addon_package_url: ${{ steps.addon-generate.outputs.addon_package_url }} env: @@ -384,17 +422,12 @@ jobs: build-airgap: runs-on: ubuntu-20.04 if: github.ref_type != 'branch' - needs: [build-kotsadm, goreleaser, build-schema-migrations, generate-tag] + needs: [goreleaser, generate-tag, build-release] steps: - - name: Download migrations - uses: actions/download-artifact@v3 - with: - name: migrations-image - path: ./docker-archive - - name: Download kotsadm image + - name: Download kotsadm release uses: actions/download-artifact@v3 with: - name: kotsadm-image + name: kotsadm-release path: ./docker-archive - name: Make kotsadm airgap archive with minio image run: | @@ -442,7 +475,7 @@ jobs: regression-test: if: github.ref_type == 'branch' - needs: [ regression-test-setup, generate-tag, build-kots, build-kotsadm, generate-kurl-addon ] + needs: [ regression-test-setup, generate-tag, build-kots, generate-kurl-addon, build-release ] uses: ./.github/workflows/regression.yaml with: version_tag_old: ${{ needs.regression-test-setup.outputs.last_release_tag }} diff --git a/Makefile b/Makefile index d37bc28cb2..49dbbf09cd 100644 --- a/Makefile +++ b/Makefile @@ -131,6 +131,9 @@ build-release: mkdir -p bin/docker-archive/kotsadm skopeo copy docker://kotsadm/kotsadm:${GIT_TAG} docker-archive:bin/docker-archive/kotsadm/${GIT_TAG} + mkdir -p bin/docker-archive/kotsadm-migrations + skopeo copy docker://kotsadm/kotsadm-migrations:${GIT_TAG} docker-archive:bin/docker-archive/kotsadm-migrations/${GIT_TAG} + mkdir -p bin/docker-archive/dex skopeo copy docker://kotsadm/dex:${DEX_TAG} docker-archive:bin/docker-archive/dex/${DEX_TAG} diff --git a/migrations/Makefile b/migrations/Makefile index 3e38a4e8af..2954166162 100644 --- a/migrations/Makefile +++ b/migrations/Makefile @@ -9,8 +9,6 @@ schema-alpha: build_schema .PHONY: schema-release schema-release: IMAGE = kotsadm/${PROJECT_NAME}:${GIT_TAG} schema-release: build_schema - mkdir -p bin/docker-archive/${PROJECT_NAME} - skopeo copy docker-daemon:kotsadm/${PROJECT_NAME}:${GIT_TAG} docker-archive:bin/docker-archive/${PROJECT_NAME}/${GIT_TAG} build_schema: docker build --pull --build-arg SCHEMAHERO_TAG=${SCHEMAHERO_TAG} -f deploy/Dockerfile -t ${IMAGE} . From a0520b050cac4b834eff7e01658b52b7d7fc9263 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Mon, 11 Dec 2023 16:07:50 +0000 Subject: [PATCH 18/22] echo image exists output --- .github/actions/build-push-image-with-apko/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/build-push-image-with-apko/action.yml b/.github/actions/build-push-image-with-apko/action.yml index ba3dff1343..e2b474b56b 100644 --- a/.github/actions/build-push-image-with-apko/action.yml +++ b/.github/actions/build-push-image-with-apko/action.yml @@ -27,8 +27,10 @@ runs: run: | set -euo pipefail if docker manifest inspect ${{ inputs.image-name }} > /dev/null 2>&1; then + echo "Image already exists. Will not overwrite." echo "image-exists=true" >> "$GITHUB_OUTPUT" else + echo "Image does not exist. Will build and push." echo "image-exists=false" >> "$GITHUB_OUTPUT" fi From c0cd1617c65d0e8937845db3fe769f3d02ba4097 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Mon, 11 Dec 2023 16:33:18 +0000 Subject: [PATCH 19/22] fix build-release --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b81fd1f77d..49c7a13fa4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -230,7 +230,7 @@ jobs: build-release: runs-on: ubuntu-20.04 - needs: [build-kotsadm-migrations, build-kotsadm, build-minio, build-rqlite, build-dex] + needs: [generate-tag, build-kotsadm-migrations, build-kotsadm, build-minio, build-rqlite, build-dex] steps: - name: Checkout uses: actions/checkout@v4 From 48f8e1abed984e101e4c7a0d010d5a90fa1a5ac6 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Mon, 11 Dec 2023 18:02:09 +0000 Subject: [PATCH 20/22] remove test branch --- .github/workflows/release.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 49c7a13fa4..999c4c097e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -6,7 +6,6 @@ on: - "v*.*.*" branches: - main - - build-rqlite-with-apko jobs: generate-tag: From de394fe04e084d0572fe56180601b0d4a5b6256e Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Tue, 12 Dec 2023 18:49:29 +0000 Subject: [PATCH 21/22] build kotsadm-migrations and use rqlite 7.21.4 --- .../action.yml | 62 +++++++++++++++++++ .github/workflows/alpha.yaml | 29 ++++----- .github/workflows/build-test.yaml | 18 ++---- .github/workflows/release.yaml | 15 ++--- .image.env | 2 +- Makefile | 2 +- cmd/imagedeps/README.md | 4 +- cmd/imagedeps/main_test.go | 2 +- cmd/imagedeps/testdata/rqlite/.image.env | 2 +- cmd/imagedeps/testdata/rqlite/constants.go | 2 +- deploy/rqlite/apko.yaml | 2 +- migrations/Makefile | 9 --- migrations/deploy/apko.yaml.tmpl | 36 +++++++++++ migrations/deploy/melange.yaml.tmpl | 32 ++++++++++ pkg/image/constants.go | 2 +- 15 files changed, 159 insertions(+), 60 deletions(-) create mode 100644 .github/actions/build-push-kotsadm-migrations-image/action.yml create mode 100644 migrations/deploy/apko.yaml.tmpl create mode 100644 migrations/deploy/melange.yaml.tmpl diff --git a/.github/actions/build-push-kotsadm-migrations-image/action.yml b/.github/actions/build-push-kotsadm-migrations-image/action.yml new file mode 100644 index 0000000000..858ac584db --- /dev/null +++ b/.github/actions/build-push-kotsadm-migrations-image/action.yml @@ -0,0 +1,62 @@ +name: 'Build and push kotsadm-migrations image' +description: 'Composite action for building and pushing kotsadm-migrations image' +inputs: + image-name: + description: 'Full destination kotsadm-migrations image name' + required: true + + git-tag: + description: 'Git tag' + required: true + + registry-username: + description: 'Username to login to registry' + default: '' + required: false + + registry-password: + description: 'Password to login to registry' + default: '' + required: false + +runs: + using: "composite" + steps: + - name: load environment variables from .image.env + uses: falti/dotenv-action@v1 + id: dotenv + with: + path: .image.env + + - uses: shrink/actions-docker-extract@v3 + with: + image: schemahero/schemahero:${{ steps.dotenv.outputs.SCHEMAHERO_TAG }} + path: /schemahero + destination: migrations + + - name: template melange and apko configs + shell: bash + run: | + export GIT_TAG=${{ inputs.git-tag }} + envsubst '${GIT_TAG}' < migrations/deploy/melange.yaml.tmpl > migrations/deploy/melange.yaml + envsubst '${GIT_TAG}' < migrations/deploy/apko.yaml.tmpl > migrations/deploy/apko.yaml + + - id: cache-dir + shell: bash + run: echo "cache_dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" + + - uses: chainguard-dev/actions/melange-build@main + with: + config: migrations/deploy/melange.yaml + archs: x86_64 + sign-with-temporary-key: true + cache-dir: ${{ steps.cache-dir.outputs.cache_dir }} + + - uses: chainguard-images/actions/apko-publish@main + with: + config: migrations/deploy/apko.yaml + archs: x86_64 + tag: ${{ inputs.image-name }} + vcs-url: true + generic-user: ${{ inputs.registry-username }} + generic-pass: ${{ inputs.registry-password }} diff --git a/.github/workflows/alpha.yaml b/.github/workflows/alpha.yaml index 8ac5131e43..d06d5149c5 100644 --- a/.github/workflows/alpha.yaml +++ b/.github/workflows/alpha.yaml @@ -19,24 +19,17 @@ jobs: uses: ./.github/actions/version-tag - build-schema-migrations: + build-kotsadm-migrations: runs-on: ubuntu-20.04 + needs: [generate-tag] steps: - - name: Checkout - uses: actions/checkout@v4 - - - uses: azure/docker-login@v1 - env: - DOCKER_CONFIG: ./migrations/.docker - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build - env: - DOCKER_CONFIG: ./.docker - run: | - mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make -C migrations schema-alpha + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-push-kotsadm-migrations-image + with: + image-name: index.docker.io/kotsadm/kotsadm-migrations:alpha + git-tag: ${{ needs.generate-tag.outputs.tag }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} build-rqlite: @@ -261,9 +254,9 @@ jobs: sarif_file: kotsadm-scan-output.sarif - scan_migrations: + scan_kotsadm_migrations: runs-on: ubuntu-20.04 - needs: [build-schema-migrations] + needs: [build-kotsadm-migrations] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 869c668c72..80adb0f0f8 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -383,7 +383,7 @@ jobs: build-migrations: runs-on: ubuntu-20.04 - needs: [ can-run-ci ] + needs: [ can-run-ci, generate-tag ] steps: - name: setup env run: | @@ -399,20 +399,10 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: load environment variables from .image.env - uses: falti/dotenv-action@v1 - id: dotenv + - uses: ./.github/actions/build-push-kotsadm-migrations-image with: - path: .image.env - - - name: build and push migrations for e2e - uses: docker/build-push-action@v5 - with: - tags: ttl.sh/automated-${{ github.run_id }}/kotsadm-migrations:24h - context: ./migrations - file: ./migrations/deploy/Dockerfile - push: true - build-args: SCHEMAHERO_TAG=${{ steps.dotenv.outputs.schemahero_tag }} + image-name: ttl.sh/automated-${{ github.run_id }}/kotsadm-migrations:24h + git-tag: ${{ needs.generate-tag.outputs.tag }} push-minio: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 999c4c097e..daabb41492 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -68,17 +68,12 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - uses: azure/docker-login@v1 - env: - DOCKER_CONFIG: ./migrations/.docker + - uses: ./.github/actions/build-push-kotsadm-migrations-image with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: "Release schema migrations on tag" - env: - GIT_TAG: ${{ needs.generate-tag.outputs.tag }} - DOCKER_CONFIG: ./.docker - run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make -C migrations schema-release + image-name: index.docker.io/kotsadm/kotsadm-migrations:${{ needs.generate-tag.outputs.tag }} + git-tag: ${{ needs.generate-tag.outputs.tag }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} build-web: runs-on: ubuntu-20.04 diff --git a/.image.env b/.image.env index f6d5f1c69f..18d2f5d4e9 100644 --- a/.image.env +++ b/.image.env @@ -2,7 +2,7 @@ # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. MINIO_TAG='0.20231101.183725' -RQLITE_TAG='8.0.1' +RQLITE_TAG='7.21.4' DEX_TAG='2.37.0' SCHEMAHERO_TAG='0.17.0' LVP_TAG='v0.5.5' \ No newline at end of file diff --git a/Makefile b/Makefile index 49dbbf09cd..7642f5ba30 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ include Makefile.build.mk CURRENT_USER := $(shell id -u -n) MINIO_TAG ?= 0.20231101.183725 -RQLITE_TAG ?= 8.0.1 +RQLITE_TAG ?= 7.21.4 DEX_TAG ?= 2.37.0 LVP_TAG ?= v0.5.5 diff --git a/cmd/imagedeps/README.md b/cmd/imagedeps/README.md index 50b797671f..732d036ee6 100644 --- a/cmd/imagedeps/README.md +++ b/cmd/imagedeps/README.md @@ -31,7 +31,7 @@ dex kotsadm/dex The preceding image spec will produce the following environment and Go files. ```shell MINIO_TAG='0.20231101.183725' -RQLITE_TAG='8.0.1' +RQLITE_TAG='7.21.4' DEX_TAG='2.37.0' ``` ```go @@ -39,7 +39,7 @@ package image const ( Minio = "kotsadm/minio:0.20231101.183725" - Rqlite = "kotsadm/rqlite:8.0.1" + Rqlite = "kotsadm/rqlite:7.21.4" Dex = "kotsadm/dex:2.37.0" ) ``` diff --git a/cmd/imagedeps/main_test.go b/cmd/imagedeps/main_test.go index 26dea136a6..5354fee1a8 100644 --- a/cmd/imagedeps/main_test.go +++ b/cmd/imagedeps/main_test.go @@ -14,7 +14,7 @@ import ( var ( minioTag = "0.20231101.183725" - rqliteTag = "8.0.1" + rqliteTag = "7.21.4" dexTag = "2.37.0" schemaheroTags = []string{ diff --git a/cmd/imagedeps/testdata/rqlite/.image.env b/cmd/imagedeps/testdata/rqlite/.image.env index 73a2d7b8ac..d6f44088a3 100644 --- a/cmd/imagedeps/testdata/rqlite/.image.env +++ b/cmd/imagedeps/testdata/rqlite/.image.env @@ -1,4 +1,4 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -RQLITE_TAG='8.0.1' \ No newline at end of file +RQLITE_TAG='7.21.4' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/rqlite/constants.go b/cmd/imagedeps/testdata/rqlite/constants.go index decfb545f0..455a24f3a8 100644 --- a/cmd/imagedeps/testdata/rqlite/constants.go +++ b/cmd/imagedeps/testdata/rqlite/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Rqlite = "kotsadm/rqlite:8.0.1" + Rqlite = "kotsadm/rqlite:7.21.4" ) diff --git a/deploy/rqlite/apko.yaml b/deploy/rqlite/apko.yaml index 7097434175..d57a18497c 100644 --- a/deploy/rqlite/apko.yaml +++ b/deploy/rqlite/apko.yaml @@ -4,7 +4,7 @@ contents: keyring: - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub packages: - - rqlite~8.0.1 + - rqlite~7.21.4 - rqlite-oci-entrypoint - bash - busybox diff --git a/migrations/Makefile b/migrations/Makefile index 2954166162..f28e7bce5e 100644 --- a/migrations/Makefile +++ b/migrations/Makefile @@ -1,15 +1,6 @@ SHELL:=/bin/bash -PROJECT_NAME ?= kotsadm-migrations SCHEMAHERO_TAG ?= 0.17.0 -.PHONY: schema-alpha -schema-alpha: IMAGE = kotsadm/${PROJECT_NAME}:alpha -schema-alpha: build_schema - -.PHONY: schema-release -schema-release: IMAGE = kotsadm/${PROJECT_NAME}:${GIT_TAG} -schema-release: build_schema - build_schema: docker build --pull --build-arg SCHEMAHERO_TAG=${SCHEMAHERO_TAG} -f deploy/Dockerfile -t ${IMAGE} . docker push ${IMAGE} diff --git a/migrations/deploy/apko.yaml.tmpl b/migrations/deploy/apko.yaml.tmpl new file mode 100644 index 0000000000..fb1796b3b3 --- /dev/null +++ b/migrations/deploy/apko.yaml.tmpl @@ -0,0 +1,36 @@ +contents: + repositories: + - https://packages.wolfi.dev/os + - ./packages/ + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + - ./melange.rsa.pub + packages: + - kotsadm-migrations-head # This is expected to be built locally by `melange`. + - bash + - busybox + - curl + - git + - wolfi-baselayout + +accounts: + groups: + - groupname: schemahero + gid: 1001 + users: + - username: schemahero + uid: 1001 + gid: 1001 + run-as: schemahero + +environment: + VERSION: ${GIT_TAG} + +entrypoint: + command: /schemahero + +cmd: apply + +archs: + - x86_64 + - aarch64 diff --git a/migrations/deploy/melange.yaml.tmpl b/migrations/deploy/melange.yaml.tmpl new file mode 100644 index 0000000000..88365e8866 --- /dev/null +++ b/migrations/deploy/melange.yaml.tmpl @@ -0,0 +1,32 @@ +package: + name: kotsadm-migrations-head + version: ${GIT_TAG} + epoch: 0 + description: kotsadm-migrations package + copyright: + - license: Apache-2.0 + +environment: + contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - ca-certificates-bundle + - busybox + - git + - go + environment: + GOMODCACHE: '/var/cache/melange' + +pipeline: + - runs: | + set -x + export DESTDIR="${{targets.destdir}}" + mkdir -p "${DESTDIR}" + + mv migrations/tables "${DESTDIR}/tables" + + # expected to have already been extracted from the schemahero image + mv migrations/schemahero "${DESTDIR}/schemahero" diff --git a/pkg/image/constants.go b/pkg/image/constants.go index fae7b78d0a..009a983805 100644 --- a/pkg/image/constants.go +++ b/pkg/image/constants.go @@ -6,7 +6,7 @@ package image const ( Minio = "kotsadm/minio:0.20231101.183725" - Rqlite = "kotsadm/rqlite:8.0.1" + Rqlite = "kotsadm/rqlite:7.21.4" Dex = "kotsadm/dex:2.37.0" Schemahero = "schemahero/schemahero:0.17.0" Lvp = "replicated/local-volume-provider:v0.5.5" From 0877842b9f0e3d5595a18fb105d26084dbe10797 Mon Sep 17 00:00:00 2001 From: Salah Aldeen Al Saleh Date: Tue, 12 Dec 2023 18:51:34 +0000 Subject: [PATCH 22/22] only define archs in github action --- deploy/apko.yaml.tmpl | 4 ---- deploy/dex/apko.yaml | 4 ---- deploy/minio/apko.yaml | 4 ---- deploy/rqlite/apko.yaml | 4 ---- kurl_proxy/deploy/apko.yaml.tmpl | 4 ---- migrations/deploy/apko.yaml.tmpl | 4 ---- 6 files changed, 24 deletions(-) diff --git a/deploy/apko.yaml.tmpl b/deploy/apko.yaml.tmpl index 276ab70a76..0a711b7e93 100644 --- a/deploy/apko.yaml.tmpl +++ b/deploy/apko.yaml.tmpl @@ -52,7 +52,3 @@ entrypoint: command: /kotsadm cmd: api - -archs: - - x86_64 - - aarch64 diff --git a/deploy/dex/apko.yaml b/deploy/dex/apko.yaml index 8c92702b41..5827458515 100644 --- a/deploy/dex/apko.yaml +++ b/deploy/dex/apko.yaml @@ -24,7 +24,3 @@ entrypoint: command: docker-entrypoint cmd: dex serve /etc/dex/config.docker.yaml - -archs: - - x86_64 - - aarch64 diff --git a/deploy/minio/apko.yaml b/deploy/minio/apko.yaml index 2937015a7b..868c6dc88e 100644 --- a/deploy/minio/apko.yaml +++ b/deploy/minio/apko.yaml @@ -22,7 +22,3 @@ accounts: entrypoint: command: minio - -archs: - - x86_64 - - aarch64 diff --git a/deploy/rqlite/apko.yaml b/deploy/rqlite/apko.yaml index d57a18497c..5f56ef4c35 100644 --- a/deploy/rqlite/apko.yaml +++ b/deploy/rqlite/apko.yaml @@ -24,7 +24,3 @@ entrypoint: command: docker-entrypoint.sh cmd: rqlite - -archs: - - x86_64 - - aarch64 diff --git a/kurl_proxy/deploy/apko.yaml.tmpl b/kurl_proxy/deploy/apko.yaml.tmpl index 5dd997f702..e4a32e04f8 100644 --- a/kurl_proxy/deploy/apko.yaml.tmpl +++ b/kurl_proxy/deploy/apko.yaml.tmpl @@ -27,7 +27,3 @@ environment: VERSION: ${GIT_TAG} cmd: /kurl_proxy - -archs: - - x86_64 - - aarch64 diff --git a/migrations/deploy/apko.yaml.tmpl b/migrations/deploy/apko.yaml.tmpl index fb1796b3b3..0e83a9c503 100644 --- a/migrations/deploy/apko.yaml.tmpl +++ b/migrations/deploy/apko.yaml.tmpl @@ -30,7 +30,3 @@ entrypoint: command: /schemahero cmd: apply - -archs: - - x86_64 - - aarch64