chore: fix LAM workflow (#875)

replicatedhq · Jul 29, 2024 · c501cbc · c501cbc
1 parent 79cdc2c
commit c501cbc
Show file tree

Hide file tree

Showing 25 changed files with 361 additions and 241 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -27,6 +27,7 @@ jobs:
         uses: golangci/golangci-lint-action@v6
         with:
           version: v1.54
+
   tests:
     name: Unit tests
     runs-on: ubuntu-latest
@@ -43,11 +44,17 @@ jobs:
 
   build-images:
     name: Build Images
+    outputs:
+      local-artifact-mirror: ${{ steps.local-artifact-mirror.outputs.image }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - uses: actions/cache@v4
+        with:
+          fetch-depth: 0 # fetch all history so that we can get the previous tag
+
+      - name: Cache Melange
+        uses: actions/cache@v4
         with:
           path: |
             build/.melange-cache
@@ -60,16 +67,10 @@ jobs:
         uses: chainguard-dev/actions/setup-melange@main
 
       - name: Build and push local-artifact-mirror image
+        id: local-artifact-mirror
         run: |
-          export SHORT_SHA=dev-${GITHUB_SHA::7}
-          export LOCAL_ARTIFACT_MIRROR_IMAGE=replicated/embedded-cluster-local-artifact-mirror-staging
-          export REGISTRY=docker.io
-          export USERNAME=${{ secrets.DOCKERHUB_USER }}
-          export PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
-          make apko
-          make build-and-push-local-artifact-mirror-image VERSION="${SHORT_SHA}"
-          make build-and-push-local-artifact-mirror-image VERSION="${SHORT_SHA}-previous-k0s"
-          make build-and-push-local-artifact-mirror-image VERSION="${SHORT_SHA}-upgrade"
+          make -C local-artifact-mirror apko build-ttl.sh
+          echo "image=$(cat local-artifact-mirror/build/image)" >> $GITHUB_OUTPUT
 
   buildtools:
     name: Build Buildtools
@@ -93,6 +94,7 @@ jobs:
   build:
     name: Build
     runs-on: ubuntu-latest
+    needs: [build-images]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -106,7 +108,7 @@ jobs:
       - name: Build Linux AMD64 and Output Metadata
         run: |
           export SHORT_SHA=dev-${GITHUB_SHA::7}
-          export LOCAL_ARTIFACT_MIRROR_IMAGE=replicated/embedded-cluster-local-artifact-mirror-staging
+          export LOCAL_ARTIFACT_MIRROR_IMAGE=${{ needs.build-images.outputs.local-artifact-mirror }}
           make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) K0S_BINARY_SOURCE_OVERRIDE=$(make print-PREVIOUS_K0S_BINARY_SOURCE_OVERRIDE) VERSION="${SHORT_SHA}-previous-k0s"
           tar -C output/bin -czvf embedded-cluster-linux-amd64-previous-k0s.tgz embedded-cluster
           ./output/bin/embedded-cluster version metadata > metadata-previous-k0s.json
@@ -123,7 +125,7 @@ jobs:
           path: |
             output/bin/embedded-cluster
       - name: Publish development release
-        if: github.branch == 'main'
+        if: github.ref == 'refs/heads/main'
         uses: marvinpinto/action-automatic-releases@latest
         with:
           automatic_release_tag: development
@@ -136,7 +138,7 @@ jobs:
       - name: Build CI binary
         run: |
           export SHORT_SHA=dev-${GITHUB_SHA::7}
-          export LOCAL_ARTIFACT_MIRROR_IMAGE=replicated/embedded-cluster-local-artifact-mirror-staging
+          export LOCAL_ARTIFACT_MIRROR_IMAGE=${{ needs.build-images.outputs.local-artifact-mirror }}
           echo "# channel release object" > e2e/kots-release-install/release.yaml
           echo 'channelID: "2cHXb1RCttzpR0xvnNWyaZCgDBP"' >> e2e/kots-release-install/release.yaml
           echo 'channelSlug: "ci"' >> e2e/kots-release-install/release.yaml
@@ -374,6 +376,7 @@ jobs:
       - e2e
       - sanitize
       - tests
+      - check-images
     if: always()
     steps:
       # https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context

diff --git a/.github/workflows/image-scan.yaml b/.github/workflows/image-scan.yaml
@@ -5,15 +5,49 @@ on:
     - cron: '0 0 * * *'
   workflow_dispatch: {}
   push:
+    # TODO: add this once we have all images in the metadata.yaml
+    # paths:
+    #   - '**/metadata.yaml'
     branches:
       - main
 
 permissions:
   security-events: write
 
 jobs:
+
+  build-images:
+    outputs:
+      local-artifact-mirror: ${{ steps.local-artifact-mirror.outputs.image }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # fetch all history so that we can get the previous tag
+
+      - name: Cache Melange
+        uses: actions/cache@v4
+        with:
+          path: |
+            build/.melange-cache
+          key: melange-cache
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Setup Melange
+        uses: chainguard-dev/actions/setup-melange@main
+
+      - name: Build and push local-artifact-mirror image
+        id: local-artifact-mirror
+        run: |
+          make -C local-artifact-mirror apko build-ttl.sh
+          echo "image=$(cat local-artifact-mirror/build/image)" >> $GITHUB_OUTPUT
+
   output-matrix:
     runs-on: ubuntu-latest
+    needs: [build-images]
     outputs:
       matrix: ${{ steps.build-matrix.outputs.matrix }}
     steps:
@@ -27,6 +61,7 @@ jobs:
 
       - name: Build
         run: |
+          export LOCAL_ARTIFACT_MIRROR_IMAGE=${{ needs.build-images.outputs.local-artifact-mirror }}
           make embedded-cluster-linux-amd64
 
       - name: List images

diff --git a/.github/workflows/release-prod.yaml b/.github/workflows/release-prod.yaml
@@ -6,62 +6,82 @@ on:
 permissions:
   contents: write
 jobs:
-  releaser:
+
+  get-tag:
     runs-on: ubuntu-latest
+    outputs:
+      tag-name: ${{ steps.get-tag.outputs.tag-name }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+
       - name: Extract tag name
+        id: get-tag
         run: |
           # remove the "refs/tags/" prefix to get the tag that was pushed
           export RAW_TAG=${GITHUB_REF#refs/tags/}
           # add a 'v' prefix to the tag if it doesn't already have one
           export V_TAG=$(echo "$RAW_TAG" | sed 's/^[^v]/v&/')
-          # store the tag name in a github environment variable for later steps
-          echo "TAG_NAME=${V_TAG}" >> $GITHUB_ENV
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version-file: go.mod
+          # store the tag name in an output for later steps
+          echo "tag-name=${V_TAG}" >> $GITHUB_OUTPUT
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+  build-images:
+    runs-on: ubuntu-latest
+    needs: [get-tag]
+    outputs:
+      local-artifact-mirror: ${{ steps.local-artifact-mirror.outputs.image }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
 
-      - uses: actions/cache@v4
+      - name: Cache Melange
+        uses: actions/cache@v4
         with:
           path: |
             build/.melange-cache
           key: melange-cache
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Setup Melange
         uses: chainguard-dev/actions/setup-melange@main
 
-      - name: Build and push local-artifact-mirror image (production)
+      - name: Build and push local-artifact-mirror image
+        id: local-artifact-mirror
+        env:
+          REGISTRY: docker.io
+          USERNAME: ${{ secrets.DOCKERHUB_USER }}
+          PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
         run: |
-          export REGISTRY=docker.io
-          export USERNAME=${{ secrets.DOCKERHUB_USER }}
-          export PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
-          make apko
-          make build-and-push-local-artifact-mirror-image VERSION=$TAG_NAME
+          make -C local-artifact-mirror apko build-and-push-local-artifact-mirror-image \
+            PACKAGE_VERSION=${{ needs.get-tag.outputs.tag-name }}
+          echo "image=$(cat local-artifact-mirror/build/image)" >> $GITHUB_OUTPUT
 
-      - name: Build and push local-artifact-mirror image (staging)
-        run: |
-          export LOCAL_ARTIFACT_MIRROR_IMAGE=replicated/embedded-cluster-local-artifact-mirror-staging
-          export REGISTRY=docker.io
-          export USERNAME=${{ secrets.DOCKERHUB_USER }}
-          export PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
-          make apko
-          make build-and-push-local-artifact-mirror-image VERSION=$TAG_NAME
+  release:
+    runs-on: ubuntu-latest
+    needs: [get-tag, build-images]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
       - name: Build and push utils image (production)
         run: |
           export REGISTRY=docker.io
           export USERNAME=${{ secrets.DOCKERHUB_USER }}
           export PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
           make apko
-          make build-and-push-utils-image VERSION=$TAG_NAME
+          make build-and-push-utils-image VERSION=${{ needs.get-tag.outputs.tag-name }}
 
       - name: Build and push utils image (staging)
         run: |
@@ -70,12 +90,15 @@ jobs:
           export USERNAME=${{ secrets.DOCKERHUB_USER }}
           export PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
           make apko
-          make build-and-push-utils-image VERSION=$TAG_NAME
+          make build-and-push-utils-image VERSION=${{ needs.get-tag.outputs.tag-name }}
 
       - name: Build linux-amd64
         run: |
-          make embedded-cluster-linux-amd64 VERSION=$TAG_NAME
+          make embedded-cluster-linux-amd64 \
+            VERSION=${{ needs.get-tag.outputs.tag-name }} \
+            LOCAL_ARTIFACT_MIRROR_IMAGE=${{ needs.build-images.outputs.local-artifact-mirror }}
           tar -C output/bin -czvf embedded-cluster-linux-amd64.tgz embedded-cluster
+
       - name: Output Metadata
         run: |
           ./output/bin/embedded-cluster version metadata > metadata.json
@@ -86,17 +109,19 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
           AWS_REGION: "us-east-1"
         run: |
-          export EC_VERSION="$TAG_NAME"
+          export EC_VERSION="${{ needs.get-tag.outputs.tag-name }}"
           ./scripts/cache-files.sh
+
       - name: Cache Prod Files
         env:
           S3_BUCKET: "tf-embedded-cluster-binaries"
           AWS_ACCESS_KEY_ID: ${{ secrets.PROD_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.PROD_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
           AWS_REGION: "us-east-1"
         run: |
-          export EC_VERSION="$TAG_NAME"
+          export EC_VERSION="${{ needs.get-tag.outputs.tag-name }}"
           ./scripts/cache-files.sh
+
       - name: Publish release
         uses: marvinpinto/action-automatic-releases@latest
         with: