diff --git a/.github/workflows/alpha.yaml b/.github/workflows/alpha.yaml index 59d9185f6c..be7edf38cf 100644 --- a/.github/workflows/alpha.yaml +++ b/.github/workflows/alpha.yaml @@ -96,7 +96,7 @@ jobs: path: ./kurl_proxy/bin - build_go_api: + build-kots: runs-on: ubuntu-20.04 needs: [build_web, build_kurl_proxy, generate-tag] steps: @@ -133,39 +133,26 @@ jobs: - name: Upload Go API artifact uses: actions/upload-artifact@v3 with: - name: go_api + name: kots path: ./bin - release_go_api_alpha: + build-kotsadm: runs-on: ubuntu-20.04 - needs: [build_web, build_go_api, generate-tag] + needs: [generate-tag] + permissions: + id-token: write # required to be able to assume the GCP SA identity to pull Chainguard packages. steps: - - - name: Checkout - uses: actions/checkout@v4 - - - name: Download go_api artifact - uses: actions/download-artifact@v3 - with: - name: go_api - path: ./bin - - - name: Add executable permissions - run: | - chmod a+x ./bin/kotsadm - chmod a+x ./bin/kots - - - uses: azure/docker-login@v1 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build alpha release - env: - GIT_TAG: ${{ needs.generate-tag.outputs.tag }} - run: | - mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make build-alpha + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-push-kotsadm-image + with: + chainguard-gcp-wif-pool: ${{ secrets.CHAINGUARD_GCP_WIF_POOL }} + chainguard-gcp-sa: ${{ secrets.CHAINGUARD_GCP_SA }} + chainguard-gcp-project-id: ${{ secrets.CHAINGUARD_GCP_PROJECT_ID }} + image-name: index.docker.io/kotsadm/kotsadm:alpha + git-tag: ${{ needs.generate-tag.outputs.tag }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} build_kurl_proxy_alpha: @@ -210,7 +197,7 @@ jobs: id: scan uses: aquasecurity/trivy-action@master with: - image-ref: "rqlite/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}" + image-ref: "cgr.dev/chainguard/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}" format: 'template' template: '@/contrib/sarif.tpl' output: 'rqlite-scan-output.sarif' @@ -237,7 +224,7 @@ jobs: id: scan uses: aquasecurity/trivy-action@master with: - image-ref: "docker.io/minio/minio:${{ steps.dotenv.outputs.minio_tag }}" + image-ref: "cgr.dev/chainguard/minio:${{ steps.dotenv.outputs.minio_tag }}" format: 'template' template: '@/contrib/sarif.tpl' output: 'minio-scan-output.sarif' @@ -265,7 +252,7 @@ jobs: id: scan uses: aquasecurity/trivy-action@master with: - image-ref: "ghcr.io/dexidp/dex:${{ steps.dotenv.outputs.dex_tag }}" + image-ref: "cgr.dev/chainguard/dex:${{ steps.dotenv.outputs.dex_tag }}" format: 'template' template: '@/contrib/sarif.tpl' output: 'dex-scan-output.sarif' @@ -330,7 +317,7 @@ jobs: scan_kotsadm: runs-on: ubuntu-20.04 - needs: [release_go_api_alpha] + needs: [build-kotsadm] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 6c401ab34b..426086e56e 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -431,8 +431,8 @@ jobs: - name: push minio for e2e run: | - docker pull minio/minio:${{ steps.dotenv.outputs.minio_tag }} - docker tag minio/minio:${{ steps.dotenv.outputs.minio_tag }} ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.minio_tag }} + docker pull cgr.dev/chainguard/minio:${{ steps.dotenv.outputs.minio_tag }} + docker tag cgr.dev/chainguard/minio:${{ steps.dotenv.outputs.minio_tag }} ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.minio_tag }} docker push ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.minio_tag }} @@ -450,8 +450,8 @@ jobs: - name: push rqlite for CI run: | - docker pull rqlite/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} - docker tag rqlite/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} + docker pull cgr.dev/chainguard/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} + docker tag cgr.dev/chainguard/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} docker push ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} @@ -469,7 +469,8 @@ jobs: - name: push dex for CI run: | - docker build --pull -f deploy/dex.Dockerfile -t ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }} --build-arg TAG=${{ steps.dotenv.outputs.DEX_TAG }} . + docker pull cgr.dev/chainguard/dex:${{ steps.dotenv.outputs.DEX_TAG }} + docker tag cgr.dev/chainguard/dex:${{ steps.dotenv.outputs.DEX_TAG }} ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }} docker push ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 25c6aab205..c14f664a92 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -186,7 +186,7 @@ jobs: id-token: write # required to be able to assume the GCP SA identity to pull Chainguard packages. steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: ./.github/actions/build-push-kotsadm-image with: chainguard-gcp-wif-pool: ${{ secrets.CHAINGUARD_GCP_WIF_POOL }} diff --git a/.image.env b/.image.env index cb5fd3bde3..dd203aa0a0 100644 --- a/.image.env +++ b/.image.env @@ -1,8 +1,8 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -MINIO_TAG='RELEASE.2023-11-11T08-14-41Z' +MINIO_TAG='0.20231101-dev' RQLITE_TAG='7.21.4' -DEX_TAG='v2.37.0' +DEX_TAG='2.37.0' SCHEMAHERO_TAG='0.16.0' LVP_TAG='v0.5.5' \ No newline at end of file diff --git a/Makefile b/Makefile index b2c6216431..43698014e4 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ include Makefile.build.mk CURRENT_USER := $(shell id -u -n) -MINIO_TAG ?= RELEASE.2023-11-11T08-14-41Z +MINIO_TAG ?= 0.20231101-dev RQLITE_TAG ?= 7.21.4 -DEX_TAG ?= v2.37.0 +DEX_TAG ?= 2.37.0 LVP_TAG ?= v0.5.5 define sendMetrics @@ -118,31 +118,36 @@ build-ttl.sh: build all-ttl.sh: build-ttl.sh source .image.env && IMAGE=ttl.sh/${CURRENT_USER}/kotsadm-migrations:24h make -C migrations build_schema - docker pull minio/minio:${MINIO_TAG} - docker tag minio/minio:${MINIO_TAG} ttl.sh/${CURRENT_USER}/minio:${MINIO_TAG} + docker pull cgr.dev/chainguard/minio:${MINIO_TAG} + docker tag cgr.dev/chainguard/minio:${MINIO_TAG} ttl.sh/${CURRENT_USER}/minio:${MINIO_TAG} docker push ttl.sh/${CURRENT_USER}/minio:${MINIO_TAG} - docker pull rqlite/rqlite:${RQLITE_TAG} - docker tag rqlite/rqlite:${RQLITE_TAG} ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG} + docker pull cgr.dev/chainguard/rqlite:${RQLITE_TAG} + docker tag cgr.dev/chainguard/rqlite:${RQLITE_TAG} ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG} docker push ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG} -.PHONY: build-alpha -build-alpha: - docker build --pull -f deploy/Dockerfile --build-arg version=${GIT_TAG} -t kotsadm/kotsadm:alpha . - docker push kotsadm/kotsadm:alpha - .PHONY: build-release build-release: mkdir -p bin/docker-archive/kotsadm skopeo copy docker://kotsadm/kotsadm:${GIT_TAG} docker-archive:bin/docker-archive/kotsadm/${GIT_TAG} - docker build --pull -f deploy/dex.Dockerfile -t kotsadm/dex:${DEX_TAG} --build-arg TAG=${DEX_TAG} . - docker push kotsadm/dex:${DEX_TAG} - mkdir -p bin/docker-archive/dex - skopeo copy docker://kotsadm/dex:${DEX_TAG} docker-archive:bin/docker-archive/dex/${DEX_TAG} - mkdir -p bin/docker-archive/minio - skopeo copy docker://minio/minio:${MINIO_TAG} docker-archive:bin/docker-archive/minio/${MINIO_TAG} + docker pull cgr.dev/chainguard/minio:${MINIO_TAG} + docker tag cgr.dev/chainguard/minio:${MINIO_TAG} kotsadm/minio:${MINIO_TAG} + docker push kotsadm/minio:${MINIO_TAG} + skopeo copy docker-daemon:kotsadm/minio:${MINIO_TAG} docker-archive:bin/docker-archive/minio/${MINIO_TAG} + + mkdir -p bin/docker-archive/rqlite + docker pull cgr.dev/chainguard/rqlite:${RQLITE_TAG} + docker tag cgr.dev/chainguard/rqlite:${RQLITE_TAG} kotsadm/rqlite:${RQLITE_TAG} + docker push kotsadm/rqlite:${RQLITE_TAG} + skopeo copy docker-daemon:kotsadm/rqlite:${RQLITE_TAG} docker-archive:bin/docker-archive/rqlite/${RQLITE_TAG} + + mkdir -p bin/docker-archive/dex + docker pull cgr.dev/chainguard/dex:${DEX_TAG} + docker tag cgr.dev/chainguard/dex:${DEX_TAG} kotsadm/dex:${DEX_TAG} + docker push kotsadm/dex:${DEX_TAG} + skopeo copy docker-daemon:kotsadm/dex:${DEX_TAG} docker-archive:bin/docker-archive/dex/${DEX_TAG} mkdir -p bin/docker-archive/local-volume-provider skopeo copy docker://replicated/local-volume-provider:${LVP_TAG} docker-archive:bin/docker-archive/local-volume-provider/${LVP_TAG} diff --git a/cmd/imagedeps/README.md b/cmd/imagedeps/README.md index 8962ea024d..4c25084161 100644 --- a/cmd/imagedeps/README.md +++ b/cmd/imagedeps/README.md @@ -20,27 +20,27 @@ is useful to restrict release tags to a major version, or to filter out garbage | Name | Image URI | Matcher Regexp (Optional) | |------|--------------------|----------| -| Name of the image for example **minio** | Untagged image reference **ghcr.io/dexidp/dex**| An optional regular expression, only matching tags will be included. | +| Name of the image for example **minio** | Untagged image reference **cgr.dev/chainguard/minio**| An optional regular expression, only matching tags will be included. | ### Sample image-spec ```text -minio minio/minio -rqlite rqlite/rqlite -dex ghcr.io/dexidp/dex +minio cgr.dev/chainguard/minio +rqlite cgr.dev/chainguard/rqlite +dex cgr.dev/chainguard/dex ``` The preceding image spec will produce the following environment and Go files. ```shell -MINIO_TAG='RELEASE.2021-09-15T04-54-25Z' -RQLITE_TAG='7.7.0' +MINIO_TAG='0.20231025-dev' +RQLITE_TAG='7.21.4' DEX_TAG='v2.30.0' ``` ```go package image const ( - Minio = "minio/minio:RELEASE.2021-09-15T04-54-25Z" - Rqlite = "rqlite/rqlite:7.7.0" - Dex = "ghcr.io/dexidp/dex:v2.30.0" + Minio = "cgr.dev/chainguard/minio:0.20231025-dev" + Rqlite = "cgr.dev/chainguard/rqlite:7.21.4" + Dex = "cgr.dev/chainguard/dex:2.30.0" ) ``` diff --git a/cmd/imagedeps/image-spec b/cmd/imagedeps/image-spec index 66a0ae252a..4e7c9b694d 100644 --- a/cmd/imagedeps/image-spec +++ b/cmd/imagedeps/image-spec @@ -1,5 +1,5 @@ -minio minio/minio -rqlite rqlite/rqlite ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ -dex ghcr.io/dexidp/dex +minio cgr.dev/chainguard/minio ^\b0\.\d+\b-dev$ +rqlite cgr.dev/chainguard/rqlite ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ +dex cgr.dev/chainguard/dex ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ schemahero schemahero/schemahero ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ lvp replicated/local-volume-provider ^v([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ \ No newline at end of file diff --git a/cmd/imagedeps/main_test.go b/cmd/imagedeps/main_test.go index e1ac5eb665..713addb013 100644 --- a/cmd/imagedeps/main_test.go +++ b/cmd/imagedeps/main_test.go @@ -12,16 +12,72 @@ import ( "github.com/stretchr/testify/require" ) -var releaseTags = []string{ - "RELEASE.2022-06-11T19-55-32Z.fips", - "RELEASE.2021-09-09T21-37-06Z.xxx", - "RELEASE.2021-09-09T21-37-05Z", - "RELEASE.2021-09-09T21-37-04Z", -} -var semVerTags = []string{ - "0.12.7", "0.12.6", "0.12.5", - "0.12.4", "0.12.3", "0.12.2", -} +var ( + minioTags = []string{ + "sha256-00428f99c05677c91ad393c3017376e800d601708baa36e51091df3b9a67b324.att", + "latest-dev", + "latest", + "0.20231025.063325-r0-dev", + "0.20231025.063325-r0", + "0.20231025.063325-dev", + "0.20231025.063325", + "0.20231025-dev", + "0.20231025", + "0.20230904.195737-r1-dev", + "0.20230904.195737-r1", + "0.20230904.195737-dev", + "0.20230904.195737", + "0.20230904-dev", + "0.20230904", + "0-dev", + "0", + } + + schemaheroTags = []string{ + "0.13.2", + "0.13.1", + "0.12.7", + "0.12.2", + } + + rqliteTags = []string{ + "sha256-00122e405b3fa3b5105b0468f1fb72dcb32474968a971c45906a702120d55b58.att", + "latest-dev", + "latest", + "7", + "7-dev", + "7.7.0", + "7.7.0-dev", + "7.7.0-r2", + "7.7.0-r2-dev", + "7.6.2", + "7.6.1", + "7.6.0", + "6.10.2", + "6.10.1", + "6.8.2", + } + + dexTags = []string{ + "sha256-002adc734b3d83bb6be291b49eb8f3f95b905c411d404c2f4b52a759140739c9.att", + "latest-dev", + "latest", + "2.37.0", + "2.37.0-r3-dev", + "2.37.0-r3", + "2.37.0-dev", + "2.36.0", + "2.35.3", + "2.35.2", + "2.35.1", + } + + lvpTags = []string{ + "v0.3.3", + "v0.3.2", + "v0.3.1", + } +) func makeReleases(tags []string) []*github.RepositoryRelease { var releases []*github.RepositoryRelease @@ -46,29 +102,21 @@ func TestFunctional(t *testing.T) { expectError bool }{ { - name: "basic", + name: "minio", fn: getTagFinder( - withGithubReleaseTagFinder( - func(_ string, _ string) ([]*github.RepositoryRelease, error) { - return makeReleases(releaseTags), nil + withRepoGetTags( + func(_ string) ([]string, error) { + return minioTags, nil }, ), ), }, { - name: "with-overrides", + name: "schemahero", fn: getTagFinder( withRepoGetTags( func(_ string) ([]string, error) { - return []string{ - "0.13.2", "0.13.1", - "0.12.7", "0.12.2", - }, nil - }, - ), - withGithubReleaseTagFinder( - func(_ string, _ string) ([]*github.RepositoryRelease, error) { - return makeReleases(releaseTags), nil + return schemaheroTags, nil }, ), ), @@ -82,30 +130,17 @@ func TestFunctional(t *testing.T) { fn: getTagFinder( withRepoGetTags( func(_ string) ([]string, error) { - return []string{ - "7.7.0", "7.6.1", "7.6.0", - "6.10.2", "6.10.1", "6.8.2", - }, nil - }, - ), - ), - }, - { - name: "filter-github", - fn: getTagFinder( - withGithubReleaseTagFinder( - func(_ string, _ string) ([]*github.RepositoryRelease, error) { - return makeReleases(releaseTags), nil + return rqliteTags, nil }, ), ), }, { - name: "schemahero", + name: "dex", fn: getTagFinder( withRepoGetTags( func(_ string) ([]string, error) { - return semVerTags, nil + return dexTags, nil }, ), ), @@ -115,9 +150,7 @@ func TestFunctional(t *testing.T) { fn: getTagFinder( withRepoGetTags( func(_ string) ([]string, error) { - return []string{ - "v0.3.3", - }, nil + return lvpTags, nil }, ), ), diff --git a/cmd/imagedeps/tag-finder.go b/cmd/imagedeps/tag-finder.go index eac106a303..1d97be31ea 100644 --- a/cmd/imagedeps/tag-finder.go +++ b/cmd/imagedeps/tag-finder.go @@ -2,7 +2,9 @@ package main import ( "context" + "encoding/json" "fmt" + "io" "net/http" "os" "path" @@ -13,6 +15,7 @@ import ( semver "github.com/Masterminds/semver/v3" "github.com/google/go-github/v39/github" "github.com/heroku/docker-registry-client/registry" + "github.com/pkg/errors" "golang.org/x/oauth2" ) @@ -119,17 +122,17 @@ func getTagFinder(opts ...func(c *configuration)) tagFinderFn { switch imageName { case minioReference: - latestReleaseTag, err = getLatestTagFromGithub(config.releaseFinder, "minio", "minio", matcherFn) + latestReleaseTag, err = getLatestTagFromRegistry("cgr.dev/chainguard/minio", config.repositoryTagsFinder, matcherFn) if err != nil { - return nil, fmt.Errorf("failed to get release tag for minio/minio %w", err) + return nil, fmt.Errorf("failed to get release tag for %s %w", imageName, err) } case dexReference: - latestReleaseTag, err = getLatestTagFromGithub(config.releaseFinder, "dexidp", "dex", matcherFn) + latestReleaseTag, err = getLatestTagFromRegistry("cgr.dev/chainguard/dex", config.repositoryTagsFinder, matcherFn) if err != nil { - return nil, fmt.Errorf("failed to get release tag for dexidp/dex %w", err) + return nil, fmt.Errorf("failed to get release tag for %s %w", imageName, err) } case rqliteReference: - latestReleaseTag, err = getLatestTagFromRegistry("rqlite/rqlite", config.repositoryTagsFinder, matcherFn) + latestReleaseTag, err = getLatestTagFromRegistry("cgr.dev/chainguard/rqlite", config.repositoryTagsFinder, matcherFn) if err != nil { return nil, fmt.Errorf("failed to get release tag for %s %w", imageName, err) } @@ -256,21 +259,120 @@ func getReleases(owner, repo string) ([]*github.RepositoryRelease, error) { // getRegistryTags queries a Docker Registry HTTP API V2 compliant registry to get the tags for an image. func getRegistryTags(untaggedRef string) ([]string, error) { + parts := strings.Split(untaggedRef, "/") + + if len(parts) > 0 && parts[0] == "cgr.dev" { + // this is chainguard's registry and it only accepts a token (not username/password) + token, err := getCGRToken(untaggedRef) + if err != nil { + return nil, fmt.Errorf("could not get cgr token %w", err) + } + tags, err := getCGRImageTags(untaggedRef, token) + if err != nil { + return nil, fmt.Errorf("could not get tags from cgr %w", err) + } + return tags, nil + } + registryUri := dockerRegistryUrl imageRef := untaggedRef userName, password := "", "" - parts := strings.Split(untaggedRef, "/") + if len(parts) > 2 { registryUri = fmt.Sprintf("https://%s", parts[0]) imageRef = path.Join(parts[1:]...) } + hub, err := registry.New(registryUri, userName, password) if err != nil { return nil, fmt.Errorf("could not connect to registry %q %w", registryUri, err) } + tags, err := hub.Tags(imageRef) if err != nil { return nil, fmt.Errorf("could not fetch tags for image %q %w", imageRef, err) } + return tags, nil } + +func getCGRToken(untaggedRef string) (string, error) { + parts := strings.Split(untaggedRef, "/") + if len(parts) < 2 { + return "", fmt.Errorf("invalid ref %q", untaggedRef) + } + + repo := strings.Join(parts[1:], "/") + scope := fmt.Sprintf("repository:%s:pull", repo) + url := fmt.Sprintf("https://cgr.dev/token?scope=%s", scope) + + resp, err := http.Get(url) + if err != nil { + return "", errors.Wrap(err, "failed to get token") + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read body %w", err) + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to get token: %s", body) + } + + var token struct { + Token string `json:"token"` + } + if err := json.Unmarshal(body, &token); err != nil { + return "", fmt.Errorf("failed to unmarshal body %w", err) + } + + return token.Token, nil +} + +func getCGRImageTags(untaggedRef string, token string) ([]string, error) { + parts := strings.Split(untaggedRef, "/") + if len(parts) < 2 { + return nil, fmt.Errorf("invalid ref %q", untaggedRef) + } + + repo := strings.Join(parts[1:], "/") + url := fmt.Sprintf("https://cgr.dev/v2/%s/tags/list", repo) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request %w", err) + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, errors.Wrap(err, "failed to get tags") + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read body %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get tags: %s", body) + } + + var tags struct { + Tags []string `json:"tags"` + } + if err := json.Unmarshal(body, &tags); err != nil { + return nil, fmt.Errorf("failed to unmarshal body %w", err) + } + + // reverse array order so that most recent is first + for i := len(tags.Tags)/2 - 1; i >= 0; i-- { + opp := len(tags.Tags) - 1 - i + tags.Tags[i], tags.Tags[opp] = tags.Tags[opp], tags.Tags[i] + } + + return tags.Tags, nil +} diff --git a/cmd/imagedeps/testdata/basic/input-spec b/cmd/imagedeps/testdata/basic/input-spec deleted file mode 100644 index 7c0313fd1f..0000000000 --- a/cmd/imagedeps/testdata/basic/input-spec +++ /dev/null @@ -1 +0,0 @@ -minio minio/minio \ No newline at end of file diff --git a/cmd/imagedeps/testdata/filter-github/.image.env b/cmd/imagedeps/testdata/dex/.image.env similarity index 83% rename from cmd/imagedeps/testdata/filter-github/.image.env rename to cmd/imagedeps/testdata/dex/.image.env index 1055c8ca12..3077a28796 100644 --- a/cmd/imagedeps/testdata/filter-github/.image.env +++ b/cmd/imagedeps/testdata/dex/.image.env @@ -1,4 +1,4 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -MINIO_TAG='RELEASE.2021-09-09T21-37-06Z.xxx' \ No newline at end of file +DEX_TAG='2.37.0' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/filter-github/constants.go b/cmd/imagedeps/testdata/dex/constants.go similarity index 81% rename from cmd/imagedeps/testdata/filter-github/constants.go rename to cmd/imagedeps/testdata/dex/constants.go index 4c28919a57..4537788f47 100644 --- a/cmd/imagedeps/testdata/filter-github/constants.go +++ b/cmd/imagedeps/testdata/dex/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Minio = "minio/minio:RELEASE.2021-09-09T21-37-06Z.xxx" + Dex = "cgr.dev/chainguard/dex:2.37.0" ) diff --git a/cmd/imagedeps/testdata/dex/input-spec b/cmd/imagedeps/testdata/dex/input-spec new file mode 100644 index 0000000000..5a572f7155 --- /dev/null +++ b/cmd/imagedeps/testdata/dex/input-spec @@ -0,0 +1 @@ +dex cgr.dev/chainguard/dex ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ \ No newline at end of file diff --git a/cmd/imagedeps/testdata/filter-github/input-spec b/cmd/imagedeps/testdata/filter-github/input-spec deleted file mode 100644 index 80c831f8a5..0000000000 --- a/cmd/imagedeps/testdata/filter-github/input-spec +++ /dev/null @@ -1 +0,0 @@ -minio minio/minio xxx$ \ No newline at end of file diff --git a/cmd/imagedeps/testdata/basic/.image.env b/cmd/imagedeps/testdata/minio/.image.env similarity index 83% rename from cmd/imagedeps/testdata/basic/.image.env rename to cmd/imagedeps/testdata/minio/.image.env index 2999afe7b5..05c7a93417 100644 --- a/cmd/imagedeps/testdata/basic/.image.env +++ b/cmd/imagedeps/testdata/minio/.image.env @@ -1,4 +1,4 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -MINIO_TAG='RELEASE.2022-06-11T19-55-32Z.fips' \ No newline at end of file +MINIO_TAG='0.20231025-dev' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/basic/constants.go b/cmd/imagedeps/testdata/minio/constants.go similarity index 81% rename from cmd/imagedeps/testdata/basic/constants.go rename to cmd/imagedeps/testdata/minio/constants.go index 240e8c0597..8b7ad5cb11 100644 --- a/cmd/imagedeps/testdata/basic/constants.go +++ b/cmd/imagedeps/testdata/minio/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Minio = "minio/minio:RELEASE.2022-06-11T19-55-32Z.fips" + Minio = "cgr.dev/chainguard/minio:0.20231025-dev" ) diff --git a/cmd/imagedeps/testdata/minio/input-spec b/cmd/imagedeps/testdata/minio/input-spec new file mode 100644 index 0000000000..afcdce5ca2 --- /dev/null +++ b/cmd/imagedeps/testdata/minio/input-spec @@ -0,0 +1 @@ +minio cgr.dev/chainguard/minio ^\b0\.\d+\b-dev$ \ No newline at end of file diff --git a/cmd/imagedeps/testdata/rqlite/constants.go b/cmd/imagedeps/testdata/rqlite/constants.go index ab99db1066..872adeba02 100644 --- a/cmd/imagedeps/testdata/rqlite/constants.go +++ b/cmd/imagedeps/testdata/rqlite/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Rqlite = "rqlite/rqlite:7.7.0" + Rqlite = "cgr.dev/chainguard/rqlite:7.7.0" ) diff --git a/cmd/imagedeps/testdata/rqlite/input-spec b/cmd/imagedeps/testdata/rqlite/input-spec index c4772a98f3..fc65a896c9 100644 --- a/cmd/imagedeps/testdata/rqlite/input-spec +++ b/cmd/imagedeps/testdata/rqlite/input-spec @@ -1 +1 @@ -rqlite rqlite/rqlite ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ +rqlite cgr.dev/chainguard/rqlite ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ diff --git a/cmd/imagedeps/testdata/schemahero/.image.env b/cmd/imagedeps/testdata/schemahero/.image.env index 931c6000ca..4615bd4e65 100644 --- a/cmd/imagedeps/testdata/schemahero/.image.env +++ b/cmd/imagedeps/testdata/schemahero/.image.env @@ -1,4 +1,4 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -SCHEMAHERO_TAG='0.12.7' \ No newline at end of file +SCHEMAHERO_TAG='0.13.2' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/schemahero/constants.go b/cmd/imagedeps/testdata/schemahero/constants.go index 50a8eca1f2..3b8bc02ce8 100644 --- a/cmd/imagedeps/testdata/schemahero/constants.go +++ b/cmd/imagedeps/testdata/schemahero/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Schemahero = "schemahero/schemahero:0.12.7" + Schemahero = "schemahero/schemahero:0.13.2" ) diff --git a/cmd/imagedeps/testdata/with-overrides/replacers/expected/test.Dockerfile b/cmd/imagedeps/testdata/schemahero/replacers/expected/test.Dockerfile similarity index 100% rename from cmd/imagedeps/testdata/with-overrides/replacers/expected/test.Dockerfile rename to cmd/imagedeps/testdata/schemahero/replacers/expected/test.Dockerfile diff --git a/cmd/imagedeps/testdata/with-overrides/replacers/expected/test.mk b/cmd/imagedeps/testdata/schemahero/replacers/expected/test.mk similarity index 100% rename from cmd/imagedeps/testdata/with-overrides/replacers/expected/test.mk rename to cmd/imagedeps/testdata/schemahero/replacers/expected/test.mk diff --git a/cmd/imagedeps/testdata/with-overrides/replacers/input/test.Dockerfile b/cmd/imagedeps/testdata/schemahero/replacers/input/test.Dockerfile similarity index 100% rename from cmd/imagedeps/testdata/with-overrides/replacers/input/test.Dockerfile rename to cmd/imagedeps/testdata/schemahero/replacers/input/test.Dockerfile diff --git a/cmd/imagedeps/testdata/with-overrides/replacers/input/test.mk b/cmd/imagedeps/testdata/schemahero/replacers/input/test.mk similarity index 100% rename from cmd/imagedeps/testdata/with-overrides/replacers/input/test.mk rename to cmd/imagedeps/testdata/schemahero/replacers/input/test.mk diff --git a/cmd/imagedeps/testdata/with-overrides/.image.env b/cmd/imagedeps/testdata/with-overrides/.image.env deleted file mode 100644 index f8ab2a50cc..0000000000 --- a/cmd/imagedeps/testdata/with-overrides/.image.env +++ /dev/null @@ -1,5 +0,0 @@ -# Generated file, do not modify. This file is generated from a text file containing a list of images. The -# most recent tag is interpolated from the source repository and used to generate a fully qualified image -# name. -MINIO_TAG='RELEASE.2022-06-11T19-55-32Z.fips' -SCHEMAHERO_TAG='0.13.2' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/with-overrides/constants.go b/cmd/imagedeps/testdata/with-overrides/constants.go deleted file mode 100644 index 99df410f88..0000000000 --- a/cmd/imagedeps/testdata/with-overrides/constants.go +++ /dev/null @@ -1,10 +0,0 @@ -package image - -// Generated file, do not modify. This file is generated from a text file containing a list of images. The -// most recent tag is interpolated from the source repository and used to generate a fully qualified -// image name. - -const ( - Minio = "minio/minio:RELEASE.2022-06-11T19-55-32Z.fips" - Schemahero = "schemahero/schemahero:0.13.2" -) diff --git a/cmd/imagedeps/testdata/with-overrides/input-spec b/cmd/imagedeps/testdata/with-overrides/input-spec deleted file mode 100644 index b2a4505a31..0000000000 --- a/cmd/imagedeps/testdata/with-overrides/input-spec +++ /dev/null @@ -1,2 +0,0 @@ -minio minio/minio -schemahero schemahero/schemahero ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ diff --git a/deploy/dex.Dockerfile b/deploy/dex.Dockerfile deleted file mode 100644 index 7db7f0ff67..0000000000 --- a/deploy/dex.Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -ARG TAG=v2.32.0 -FROM ghcr.io/dexidp/dex:$TAG diff --git a/deploy/kurl/kotsadm/template/base/Manifest b/deploy/kurl/kotsadm/template/base/Manifest index 0e57a590cf..f1fb302415 100644 --- a/deploy/kurl/kotsadm/template/base/Manifest +++ b/deploy/kurl/kotsadm/template/base/Manifest @@ -1,7 +1,7 @@ image kotsadm-migrations __KOTSADM_MIGRATIONS_IMAGE__ image kotsadm __KOTSADM_IMAGE__ image kurl-proxy __KURL_PROXY_IMAGE__ -image rqlite rqlite/rqlite:__RQLITE_TAG__ +image rqlite __RQLITE_IMAGE__ image dex __DEX_IMAGE__ asset kots.tar.gz __KOTSADM_BINARY__ diff --git a/deploy/kurl/kotsadm/template/base/rqlite.yaml b/deploy/kurl/kotsadm/template/base/rqlite.yaml index 2a78dd37b1..d4fa133d75 100644 --- a/deploy/kurl/kotsadm/template/base/rqlite.yaml +++ b/deploy/kurl/kotsadm/template/base/rqlite.yaml @@ -50,7 +50,7 @@ spec: - kotsadm-rqlite topologyKey: "kubernetes.io/hostname" containers: - - image: rqlite/rqlite:__RQLITE_TAG__ + - image: __RQLITE_IMAGE__ name: rqlite args: - -disco-mode=dns diff --git a/deploy/kurl/kotsadm/template/generate.sh b/deploy/kurl/kotsadm/template/generate.sh index 234ea3cd60..a3f82b516a 100755 --- a/deploy/kurl/kotsadm/template/generate.sh +++ b/deploy/kurl/kotsadm/template/generate.sh @@ -28,7 +28,8 @@ function generate() { sed -i -e "s|__KOTSADM_BINARY__|$kotsadm_binary|g" "${dir}/Manifest" # The following environment variables will be exported by the .image.env file - find "$dir" -type f -exec sed -i -e "s|__RQLITE_TAG__|$RQLITE_TAG|g" {} \; + local rqlite_image="$kotsadm_image_registry/$kotsadm_image_namespace/rqlite:$RQLITE_TAG" + find "$dir" -type f -exec sed -i -e "s|__RQLITE_IMAGE__|$rqlite_image|g" {} \; local dex_image="$kotsadm_image_registry/$kotsadm_image_namespace/dex:$DEX_TAG" find "$dir" -type f -exec sed -i -e "s|__DEX_IMAGE__|$dex_image|g" {} \; } diff --git a/integration/database/rqlite_migration_test.go b/integration/database/rqlite_migration_test.go index 78765e0032..ac4cf78dc5 100644 --- a/integration/database/rqlite_migration_test.go +++ b/integration/database/rqlite_migration_test.go @@ -71,11 +71,14 @@ func TestMigrateFromPostgresToRqlite(t *testing.T) { rqliteTag, _ := image.GetTag(image.Rqlite) rqliteRunOptions := &dockertest.RunOptions{ Name: "rqlite", - Repository: "rqlite/rqlite", + Repository: "cgr.dev/chainguard/rqlite", Tag: rqliteTag, Mounts: []string{ fmt.Sprintf("%s:/auth/config.json", rqliteAuthConfigPath), }, + ExposedPorts: []string{ + "4001/tcp", + }, PortBindings: map[docker.Port][]docker.PortBinding{ "4001/tcp": { { diff --git a/migrations/Makefile b/migrations/Makefile index 011acb5480..662b4c3d14 100644 --- a/migrations/Makefile +++ b/migrations/Makefile @@ -1,6 +1,5 @@ SHELL:=/bin/bash PROJECT_NAME ?= kotsadm-migrations -RQLITE_TAG ?= 7.21.4 SCHEMAHERO_TAG ?= 0.16.0 .PHONY: schema-alpha @@ -12,8 +11,6 @@ schema-release: IMAGE = kotsadm/${PROJECT_NAME}:${GIT_TAG} schema-release: build_schema mkdir -p bin/docker-archive/${PROJECT_NAME} skopeo copy docker-daemon:kotsadm/${PROJECT_NAME}:${GIT_TAG} docker-archive:bin/docker-archive/${PROJECT_NAME}/${GIT_TAG} - mkdir -p bin/docker-archive/rqlite - skopeo copy docker://rqlite/rqlite:${RQLITE_TAG} docker-archive:bin/docker-archive/rqlite/${RQLITE_TAG} build_schema: docker build --pull --build-arg SCHEMAHERO_TAG=${SCHEMAHERO_TAG} -f deploy/Dockerfile -t ${IMAGE} . diff --git a/migrations/kustomize/overlays/dev/rqlite.yaml b/migrations/kustomize/overlays/dev/rqlite.yaml index 3014cb5c0c..bb2a830557 100644 --- a/migrations/kustomize/overlays/dev/rqlite.yaml +++ b/migrations/kustomize/overlays/dev/rqlite.yaml @@ -59,7 +59,7 @@ spec: spec: containers: - name: rqlite - image: rqlite/rqlite:7.9.2 + image: cgr.dev/chainguard/rqlite:7.21.4 imagePullPolicy: IfNotPresent args: - -disco-mode=dns diff --git a/migrations/kustomize/overlays/okteto/rqlite.yaml b/migrations/kustomize/overlays/okteto/rqlite.yaml index 3014cb5c0c..bb2a830557 100644 --- a/migrations/kustomize/overlays/okteto/rqlite.yaml +++ b/migrations/kustomize/overlays/okteto/rqlite.yaml @@ -59,7 +59,7 @@ spec: spec: containers: - name: rqlite - image: rqlite/rqlite:7.9.2 + image: cgr.dev/chainguard/rqlite:7.21.4 imagePullPolicy: IfNotPresent args: - -disco-mode=dns diff --git a/pkg/image/constants.go b/pkg/image/constants.go index f1b491f53d..0440c91d5c 100644 --- a/pkg/image/constants.go +++ b/pkg/image/constants.go @@ -5,9 +5,9 @@ package image // image name. const ( - Minio = "minio/minio:RELEASE.2023-11-11T08-14-41Z" - Rqlite = "rqlite/rqlite:7.21.4" - Dex = "ghcr.io/dexidp/dex:v2.37.0" + Minio = "cgr.dev/chainguard/minio:0.20231101-dev" + Rqlite = "cgr.dev/chainguard/rqlite:7.21.4" + Dex = "cgr.dev/chainguard/dex:2.37.0" Schemahero = "schemahero/schemahero:0.16.0" Lvp = "replicated/local-volume-provider:v0.5.5" ) diff --git a/pkg/kotsadm/minio.go b/pkg/kotsadm/minio.go index 7385b989e9..6c440b701e 100644 --- a/pkg/kotsadm/minio.go +++ b/pkg/kotsadm/minio.go @@ -25,6 +25,7 @@ import ( ) var ( + MinioCGRImageTagRegexp = regexp.MustCompile(`:0\.\d+`) MinioImageTagDateRegexp = regexp.MustCompile(`RELEASE\.(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z)`) // MigrateToMinioXlBeforeTime is the time that the minio version was released that removed the legacy backend // that we need to migrate from: https://github.com/minio/minio/releases/tag/RELEASE.2022-10-29T06-21-33Z @@ -127,6 +128,7 @@ func ensureMinioStatefulset(deployOptions types.DeployOptions, clientset kuberne existingMinio.Spec.Template.Spec.Volumes = desiredMinio.Spec.Template.Spec.DeepCopy().Volumes existingMinio.Spec.Template.Spec.Containers[0].Image = desiredMinio.Spec.Template.Spec.Containers[0].Image existingMinio.Spec.Template.Spec.Containers[0].VolumeMounts = desiredMinio.Spec.Template.Spec.Containers[0].DeepCopy().VolumeMounts + existingMinio.Spec.Template.Spec.Containers[0].Command = desiredMinio.Spec.Template.Spec.Containers[0].Command existingMinio.Spec.Template.Spec.InitContainers = desiredMinio.Spec.Template.Spec.DeepCopy().InitContainers _, err = clientset.AppsV1().StatefulSets(deployOptions.Namespace).Update(ctx, existingMinio, metav1.UpdateOptions{}) @@ -393,6 +395,12 @@ func IsMinioXlMigrationNeeded(clientset kubernetes.Interface, namespace string) // imageNeedsMinioXlMigration returns true if the minio image is older than the migrate before time (2022-10-29T06-21-33Z). func imageNeedsMinioXlMigration(minioImage string) (bool, error) { + isCGRImage := len(MinioCGRImageTagRegexp.FindStringSubmatch(minioImage)) > 0 + if isCGRImage { + // chainguard minio images are all new and don't need to be migrated + return false, nil + } + existingImageTagDateMatch := MinioImageTagDateRegexp.FindStringSubmatch(minioImage) if len(existingImageTagDateMatch) != 2 { return false, errors.New("failed to parse existing image tag date") diff --git a/pkg/kotsadm/minio_test.go b/pkg/kotsadm/minio_test.go index f10fec56b1..6c80e1a321 100644 --- a/pkg/kotsadm/minio_test.go +++ b/pkg/kotsadm/minio_test.go @@ -91,6 +91,13 @@ func Test_IsMinioXlMigrationNeeded(t *testing.T) { wantMinioImage: "minio/minio:RELEASE.2023-02-10T18-48-39Z", wantErr: false, }, + { + name: "should not migrate cgr image", + clientset: fake.NewSimpleClientset(minioStsWithImage("cgr.dev/chainguard/minio:0.20231025-dev")), + wantMigration: false, + wantMinioImage: "cgr.dev/chainguard/minio:0.20231025-dev", + wantErr: false, + }, { name: "should not migrate if no minio", clientset: fake.NewSimpleClientset(), diff --git a/pkg/kotsadm/objects/images.go b/pkg/kotsadm/objects/images.go index 1d097d2611..926b2eb78b 100644 --- a/pkg/kotsadm/objects/images.go +++ b/pkg/kotsadm/objects/images.go @@ -18,8 +18,8 @@ func GetAdminConsoleImages(deployOptions types.DeployOptions) map[string]string rqliteTag, _ := image.GetTag(image.Rqlite) dexTag, _ := image.GetTag(image.Dex) - minioImage := fmt.Sprintf("minio/minio:%s", minioTag) - rqliteImage := fmt.Sprintf("rqlite/rqlite:%s", rqliteTag) + minioImage := fmt.Sprintf("kotsadm/minio:%s", minioTag) + rqliteImage := fmt.Sprintf("kotsadm/rqlite:%s", rqliteTag) dexImage := fmt.Sprintf("kotsadm/dex:%s", dexTag) if s := kotsadmversion.KotsadmPullSecret(deployOptions.Namespace, deployOptions.RegistryConfig); s != nil { @@ -43,12 +43,15 @@ func GetAdminConsoleImages(deployOptions types.DeployOptions) map[string]string } func GetOriginalAdminConsoleImages(deployOptions types.DeployOptions) map[string]string { - dexTag, _ := image.GetTag(image.Dex) // dex image is special; we host a copy + minioTag, _ := image.GetTag(image.Minio) + rqliteTag, _ := image.GetTag(image.Rqlite) + dexTag, _ := image.GetTag(image.Dex) + return map[string]string{ "kotsadm-migrations": fmt.Sprintf("kotsadm/kotsadm-migrations:%s", kotsadmversion.KotsadmTag(deployOptions.RegistryConfig)), "kotsadm": fmt.Sprintf("kotsadm/kotsadm:%s", kotsadmversion.KotsadmTag(deployOptions.RegistryConfig)), - "minio": image.Minio, - "rqlite": image.Rqlite, + "minio": fmt.Sprintf("kotsadm/minio:%s", minioTag), + "rqlite": fmt.Sprintf("kotsadm/rqlite:%s", rqliteTag), "dex": fmt.Sprintf("kotsadm/dex:%s", dexTag), } } diff --git a/pkg/kotsadm/objects/minio_objects.go b/pkg/kotsadm/objects/minio_objects.go index 56c12594c7..4ae3acfc76 100644 --- a/pkg/kotsadm/objects/minio_objects.go +++ b/pkg/kotsadm/objects/minio_objects.go @@ -136,7 +136,7 @@ func MinioStatefulset(deployOptions types.DeployOptions, size resource.Quantity) Command: []string{ "/bin/sh", "-ce", - "/usr/bin/docker-entrypoint.sh minio -C /home/minio/.minio/ --quiet server /export", + "minio -C /home/minio/.minio/ --quiet server /export", }, Ports: []corev1.ContainerPort{ { diff --git a/pkg/kotsadm/objects/scripts/import-minio-data.sh b/pkg/kotsadm/objects/scripts/import-minio-data.sh index 65acbe11fd..2eb1aef9eb 100644 --- a/pkg/kotsadm/objects/scripts/import-minio-data.sh +++ b/pkg/kotsadm/objects/scripts/import-minio-data.sh @@ -33,7 +33,7 @@ shopt -s dotglob rm -rfv /export/* echo "starting new minio instance" -/bin/sh -ce "/usr/bin/docker-entrypoint.sh minio -C /home/minio/.minio/ server /export" & +/bin/sh -ce "minio -C /home/minio/.minio/ server /export" & MINIO_PID=$! # alias the minio instance diff --git a/pkg/snapshot/filesystem_minio.go b/pkg/snapshot/filesystem_minio.go index cfebe071bd..f8b0c03d0d 100644 --- a/pkg/snapshot/filesystem_minio.go +++ b/pkg/snapshot/filesystem_minio.go @@ -269,7 +269,7 @@ func fileSystemMinioDeploymentResource(clientset kubernetes.Interface, secretChe if err != nil { return nil, errors.Wrap(err, "failed to get minio image tag") } - minioImage := fmt.Sprintf("minio/minio:%s", minioTag) + minioImage := fmt.Sprintf("kotsadm/minio:%s", minioTag) imagePullSecrets := []corev1.LocalObjectReference{} isKurl, err := kurl.IsKurl(clientset)