Skip to content

handle unmodified-binary tests and in-docker upgrade tests in postrel… #251

handle unmodified-binary tests and in-docker upgrade tests in postrel…

handle unmodified-binary tests and in-docker upgrade tests in postrel… #251

Workflow file for this run

name: Release
on:
push:
tags:
- '[0-9]+\.[0-9]+\.[0-9]+\+k8s-[0-9]+\.[0-9]+'
- '[0-9]+\.[0-9]+\.[0-9]+\+k8s-[0-9]+\.[0-9]+-*'
permissions:
contents: write
jobs:
get-tag:
runs-on: ubuntu-latest
outputs:
tag-name: ${{ steps.get-tag.outputs.tag-name }}
k0s_version: ${{ steps.export.outputs.k0s_version }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Extract tag name
id: get-tag
run: |
# remove the "refs/tags/" prefix to get the tag that was pushed
export RAW_TAG=${{ github.ref_name }}
# add a 'v' prefix to the tag if it doesn't already have one
export V_TAG=$(echo "$RAW_TAG" | sed 's/^[^v]/v&/')
# store the tag name in an output for later steps
echo "tag-name=${V_TAG}" >> $GITHUB_OUTPUT
- name: Export k0s version
id: export
run: |
K0S_VERSION="$(make print-K0S_VERSION)"
echo "K0S_VERSION=\"$K0S_VERSION\""
echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"
buildtools:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache-dependency-path: "**/*.sum"
- name: Compile buildtools
run: |
make buildtools
- name: Upload buildtools artifact
uses: actions/upload-artifact@v4
with:
name: buildtools
path: output/bin/buildtools
publish-operator-image:
runs-on: ubuntu-latest
needs: [get-tag]
outputs:
image: ${{ steps.operator-image.outputs.image }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install dagger
run: |
curl -fsSL https://dl.dagger.io/dagger/install.sh | sh
sudo mv ./bin/dagger /usr/local/bin/dagger
- name: Build and push operator image
id: operator-image
env:
REGISTRY_SERVER: docker.io
REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USER }}
REGISTRY_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
run: |
make -C operator build-and-push-operator-image \
PACKAGE_VERSION=${{ needs.get-tag.outputs.tag-name }}
echo "image=$(cat operator/build/image)" >> $GITHUB_OUTPUT
publish-operator-chart:
runs-on: ubuntu-latest
needs: [get-tag, publish-operator-image]
outputs:
chart: ${{ steps.operator-chart.outputs.chart }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build and push operator chart
id: operator-chart
env:
HELM_USER: ${{secrets.REPLICATED_LIBRARY_SERVICE_ACCOUNT}}
HELM_PASS: ${{secrets.REPLICATED_LIBRARY_SERVICE_ACCOUNT}}
HELM_REGISTRY: registry.replicated.com
run: |
make -C operator build-chart \
PACKAGE_VERSION=${{ needs.get-tag.outputs.tag-name }}
echo "chart=$(cat operator/build/chart)" >> $GITHUB_OUTPUT
publish-images:
runs-on: ubuntu-latest
needs: [get-tag]
outputs:
local-artifact-mirror: ${{ steps.local-artifact-mirror.outputs.image }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install dagger
run: |
curl -fsSL https://dl.dagger.io/dagger/install.sh | sh
sudo mv ./bin/dagger /usr/local/bin/dagger
- name: Build and push local-artifact-mirror image
id: local-artifact-mirror
env:
REGISTRY_SERVER: docker.io
REGISTRY_USERNAME: ${{ secrets.DOCKERHUB_USER }}
REGISTRY_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
run: |
make -C local-artifact-mirror build-and-push-local-artifact-mirror-image \
PACKAGE_VERSION=${{ needs.get-tag.outputs.tag-name }}
echo "image=$(cat local-artifact-mirror/build/image)" >> $GITHUB_OUTPUT
release:
runs-on: ubuntu-latest
needs: [get-tag, buildtools, publish-images, publish-operator-image, publish-operator-chart]
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Cache embedded bins
uses: actions/cache@v4
with:
path: |
output/bins
key: bins-cache
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache-dependency-path: "**/*.sum"
- name: Download buildtools artifact
uses: actions/download-artifact@v4
with:
name: buildtools
path: output/bin
- name: Update embedded-cluster-operator metadata.yaml
env:
IMAGES_REGISTRY_SERVER: index.docker.io
IMAGES_REGISTRY_USER: ${{ secrets.DOCKERHUB_USER }}
IMAGES_REGISTRY_PASS: ${{ secrets.DOCKERHUB_PASSWORD }}
OPERATOR_CHART: ${{ needs.publish-operator-chart.outputs.chart }}
OPERATOR_IMAGE: ${{ needs.publish-operator-image.outputs.image }}
run: |
./scripts/ci-update-operator-metadata.sh
- name: Build linux-amd64
run: |
mkdir -p build
make embedded-cluster-linux-amd64 \
VERSION=${{ needs.get-tag.outputs.tag-name }} \
LOCAL_ARTIFACT_MIRROR_IMAGE=proxy.replicated.com/anonymous/${{ needs.publish-images.outputs.local-artifact-mirror }}
tar -C output/bin -czvf build/embedded-cluster-linux-amd64.tgz embedded-cluster
- name: Output Metadata
run: |
mkdir -p build
./output/bin/embedded-cluster version metadata > build/metadata.json
- name: Cache Staging Files
env:
S3_BUCKET: "tf-staging-embedded-cluster-bin"
AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
AWS_REGION: "us-east-1"
run: |
export EC_VERSION="${{ needs.get-tag.outputs.tag-name }}"
mkdir -p operator/build
echo "${{ needs.publish-operator-image.outputs.image }}" > "operator/build/image-$EC_VERSION"
./scripts/ci-upload-binaries.sh
- name: Cache Prod Files
env:
S3_BUCKET: "tf-embedded-cluster-binaries"
AWS_ACCESS_KEY_ID: ${{ secrets.PROD_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.PROD_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }}
AWS_REGION: "us-east-1"
run: |
export EC_VERSION="${{ needs.get-tag.outputs.tag-name }}"
mkdir -p operator/build
echo "${{ needs.publish-operator-image.outputs.image }}" > "operator/build/image-$EC_VERSION"
./scripts/ci-upload-binaries.sh
- name: Publish release
uses: marvinpinto/action-automatic-releases@latest
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
prerelease: true
files: |
build/*.tgz
build/metadata.json
find-previous-stable:
name: Determine previous stable version
runs-on: ubuntu-latest
needs:
- get-tag
outputs:
ec_version: ${{ steps.export.outputs.ec_version }}
k0s_version: ${{ steps.export.outputs.k0s_version }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Export k0s version
id: export
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
k0s_majmin_version="$(make print-PREVIOUS_K0S_VERSION | sed 's/v\([0-9]*\.[0-9]*\).*/\1/')"
if [ "$k0s_majmin_version" == "1.28" ]; then
k0s_majmin_version="1.29"
fi
EC_VERSION="$(gh release list --repo replicatedhq/embedded-cluster \
--exclude-drafts --exclude-pre-releases --json name \
--jq '.[] | .name' \
| grep "k8s-${k0s_majmin_version}" \
| head -n1)"
gh release download "$EC_VERSION" --repo replicatedhq/embedded-cluster --pattern 'metadata.json'
K0S_VERSION="$(jq -r '.Versions.Kubernetes' metadata.json)"
echo "EC_VERSION=\"$EC_VERSION\""
echo "K0S_VERSION=\"$K0S_VERSION\""
echo "ec_version=$EC_VERSION" >> "$GITHUB_OUTPUT"
echo "k0s_version=$K0S_VERSION" >> "$GITHUB_OUTPUT"
release-app:
name: Create app releases
runs-on: ubuntu-latest
permissions:
pull-requests: write
needs:
- release
- get-tag
- find-previous-stable
steps:
- name: Checkout
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install replicated CLI
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release download --repo replicatedhq/replicated --pattern '*linux_amd64.tar.gz' --output replicated.tar.gz
tar xf replicated.tar.gz replicated && rm replicated.tar.gz
mv replicated /usr/local/bin/replicated
- name: Create CI releases
env:
REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
APP_CHANNEL: CI
USES_DEV_BUCKET: "0"
run: |
# re-promote a release containing an old version of embedded-cluster to test upgrades
export APP_VERSION="appver-${{ github.ref_name }}-pre-minio-removal"
replicated release promote 807 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}"
# re-promote a release containing an old version of embedded-cluster to test upgrades
export APP_VERSION="appver-${{ github.ref_name }}-1.8.0-k8s-1.28"
replicated release promote 11615 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${APP_VERSION}"
replicated release promote 11615 2eAqMYG1IEtX8cwpaO1kgNV6EB3 --version "${APP_VERSION}"
# promote a release containing the previous stable version of embedded-cluster to test upgrades
export EC_VERSION="${{ needs.find-previous-stable.outputs.ec_version }}"
export APP_VERSION="appver-${{ github.ref_name }}-previous-stable"
export RELEASE_YAML_DIR=e2e/kots-release-install-stable
./scripts/ci-release-app.sh
# install the previous k0s version to ensure an upgrade occurs
# we do not actually run k0s upgrade tests on prerelease at present (as we don't build a previous k0s binary)
export EC_VERSION="${{ github.ref_name }}"
export APP_VERSION="appver-${{ github.ref_name }}-previous-k0s"
export RELEASE_YAML_DIR=e2e/kots-release-install
./scripts/ci-release-app.sh
# install the current k0s version
export EC_VERSION="${{ github.ref_name }}"
export APP_VERSION="appver-${{ github.ref_name }}"
export RELEASE_YAML_DIR=e2e/kots-release-install
./scripts/ci-release-app.sh
# then a noop upgrade
export EC_VERSION="${{ github.ref_name }}"
export APP_VERSION="appver-${{ github.ref_name }}-noop"
export RELEASE_YAML_DIR=e2e/kots-release-install
./scripts/ci-release-app.sh
# and finally an app upgrade
export EC_VERSION="${{ github.ref_name }}"
export APP_VERSION="appver-${{ github.ref_name }}-upgrade"
export RELEASE_YAML_DIR=e2e/kots-release-upgrade
./scripts/ci-release-app.sh
- name: Create airgap releases
env:
REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
APP_CHANNEL: CI-airgap
USES_DEV_BUCKET: "0"
run: |
# promote a release containing the previous stable version of embedded-cluster to test upgrades
export EC_VERSION="${{ needs.find-previous-stable.outputs.ec_version }}"
export APP_VERSION="appver-${{ github.ref_name }}-previous-stable"
export RELEASE_YAML_DIR=e2e/kots-release-install-stable
./scripts/ci-release-app.sh
# promote a release with the current k0s version, but call it the previous version to test noop upgrades
export EC_VERSION="${{ github.ref_name }}"
export APP_VERSION="appver-${{ github.ref_name }}-previous-k0s"
export RELEASE_YAML_DIR=e2e/kots-release-install
./scripts/ci-release-app.sh
# promote a release with the current k0s version
export EC_VERSION="${{ github.ref_name }}"
export APP_VERSION="appver-${{ github.ref_name }}"
export RELEASE_YAML_DIR=e2e/kots-release-install
./scripts/ci-release-app.sh
# and finally an app upgrade
export EC_VERSION="${{ github.ref_name }}"
export APP_VERSION="appver-${{ github.ref_name }}-upgrade"
export RELEASE_YAML_DIR=e2e/kots-release-upgrade
./scripts/ci-release-app.sh
- name: Create Stable release
env:
REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
REPLICATED_API_ORIGIN: "https://api.staging.replicated.com/vendor"
APP_CHANNEL: Stable
USES_DEV_BUCKET: "0"
run: |
# create a release in the stable channel of our test app for use by CMX
export EC_VERSION="${{ github.ref_name }}"
export APP_VERSION="${{ github.ref_name }}"
export RELEASE_YAML_DIR=e2e/kots-release-install
./scripts/ci-release-app.sh
download-current:
name: Download the current release binary
runs-on: ubuntu-latest
needs:
- release-app
- get-tag
steps:
- name: Download current binary
env:
LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }}
run: |
export APP_VERSION="appver-${{ github.ref_name }}"
curl -L "https://ec-e2e-replicated-app.testcluster.net/embedded/embedded-cluster-smoke-test-staging-app/ci/${APP_VERSION}" -H "Authorization: $LICENSE_ID" -o embedded-cluster-smoke-test-staging-app-ci.tgz
tar -xzf embedded-cluster-smoke-test-staging-app-ci.tgz
mv embedded-cluster-smoke-test-staging-app embedded-cluster
mkdir -p output/bin
mv embedded-cluster output/bin
# download the embedded-cluster binary from the github release
curl -L "https://github.com/replicatedhq/embedded-cluster/releases/download/${{ github.ref_name }}/embedded-cluster-linux-amd64.tgz" -o embedded-cluster-github.tgz
tar -xzf embedded-cluster-github.tgz
mv embedded-cluster output/bin/embedded-cluster-original
- name: Upload release
uses: actions/upload-artifact@v4
with:
name: current-release
path: |
output/bin/embedded-cluster
# e2e-docker runs the e2e tests inside a docker container rather than a full VM
e2e-docker:
name: E2E docker
runs-on: ubuntu-latest
needs:
- release
- release-app
- get-tag
- download-current
- find-previous-stable
strategy:
fail-fast: false
matrix:
test:
- TestPreflights
- TestMaterialize
- TestHostPreflightCustomSpec
- TestHostPreflightInBuiltSpec
- TestSingleNodeInstallation
- TestSingleNodeInstallationAlmaLinux8
- TestSingleNodeInstallationDebian11
- TestSingleNodeInstallationDebian12
- TestSingleNodeInstallationCentos9Stream
- TestSingleNodeUpgradePreviousStable
- TestInstallFromReplicatedApp
- TestUpgradeFromReplicatedApp
- TestInstallWithoutEmbed
- TestUpgradeEC18FromReplicatedApp
- TestResetAndReinstall
- TestOldVersionUpgrade
- TestInstallSnapshotFromReplicatedApp
- TestMultiNodeInstallation
- TestMultiNodeHAInstallation
- TestSingleNodeDisasterRecovery
- TestSingleNodeResumeDisasterRecovery
- TestMultiNodeHADisasterRecovery
- TestSingleNodeInstallationNoopUpgrade
- TestCustomCIDR
- TestLocalArtifactMirror
- TestMultiNodeReset
- TestCollectSupportBundle
- TestUnsupportedOverrides
- TestHostCollectSupportBundleInCluster
- TestInstallWithConfigValues
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Download binary
uses: actions/download-artifact@v4
with:
name: current-release
path: output/bin
- name: Setup go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache-dependency-path: "**/*.sum"
- name: Login to DockerHub to avoid rate limiting
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
- name: Free up runner disk space
uses: ./.github/actions/free-disk-space
- name: Write license files
run: |
echo "${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}" | base64 --decode > e2e/license.yaml
echo "${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE }}" | base64 --decode > e2e/snapshot-license.yaml
- name: Run test
env:
SHORT_SHA: ${{ github.ref_name }}
LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }}
AIRGAP_LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }}
SNAPSHOT_LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }}
AIRGAP_SNAPSHOT_LICENSE_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_SNAPSHOT_LICENSE_ID }}
DR_AWS_S3_ENDPOINT: https://s3.amazonaws.com
DR_AWS_S3_REGION: us-east-1
DR_AWS_S3_BUCKET: kots-testim-snapshots
DR_AWS_S3_PREFIX: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}
DR_AWS_S3_PREFIX_AIRGAP: ${{ matrix.test }}-${{ github.run_id }}-${{ github.run_attempt }}-airgap
DR_AWS_ACCESS_KEY_ID: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
DR_AWS_SECRET_ACCESS_KEY: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
EXPECT_K0S_VERSION: ${{ needs.get-tag.outputs.k0s_version }}
EXPECT_K0S_VERSION_PREVIOUS: ${{ needs.get-tag.outputs.k0s_version }}
EXPECT_K0S_VERSION_PREVIOUS_STABLE: ${{ needs.find-previous-stable.outputs.k0s_version }}
EXPECT_EMBEDDED_CLUSTER_UPGRADE_TARGET_VERSION: ${{ github.ref_name }}
run: |
make e2e-test TEST_NAME=${{ matrix.test }}
- name: Troubleshoot
if: ${{ !cancelled() }}
uses: ./.github/actions/e2e-troubleshoot
with:
test-name: '${{ matrix.test }}'
e2e:
name: E2E
runs-on: ${{ matrix.runner || 'ubuntu-22.04' }}
needs:
- release
- release-app
- get-tag
- download-current
- find-previous-stable
strategy:
fail-fast: false
matrix:
test:
- TestVersion
- TestCommandsRequireSudo
- TestResetAndReinstallAirgap
- TestSingleNodeAirgapUpgrade
- TestSingleNodeAirgapUpgradeConfigValues
- TestSingleNodeAirgapUpgradeCustomCIDR
- TestSingleNodeDisasterRecoveryWithProxy
- TestProxiedEnvironment
- TestProxiedCustomCIDR
- TestInstallWithPrivateCAs
- TestInstallWithMITMProxy
include:
- test: TestMultiNodeAirgapUpgrade
runner: embedded-cluster
- test: TestMultiNodeAirgapUpgradeSameK0s
runner: embedded-cluster
- test: TestMultiNodeAirgapUpgradePreviousStable
runner: embedded-cluster
- test: TestAirgapUpgradeFromEC18
runner: embedded-cluster
- test: TestSingleNodeAirgapDisasterRecovery
runner: embedded-cluster
- test: TestMultiNodeAirgapHAInstallation
runner: embedded-cluster
- test: TestMultiNodeAirgapHADisasterRecovery
runner: embedded-cluster
- test: TestFiveNodesAirgapUpgrade
runner: embedded-cluster
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Download current binary
uses: actions/download-artifact@v4
with:
name: current-release
path: output/bin
- uses: ./.github/actions/e2e
with:
test-name: '${{ matrix.test }}'
is-large-runner: ${{ matrix.runner == 'embedded-cluster' }}
airgap-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }}
snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }}
snapshot-license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE }}
airgap-snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_SNAPSHOT_LICENSE_ID }}
license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }}
license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE }}
dr-aws-access-key-id: ${{ secrets.TESTIM_AWS_ACCESS_KEY_ID }}
dr-aws-secret-access-key: ${{ secrets.TESTIM_AWS_SECRET_ACCESS_KEY }}
version-specifier: ${{ github.ref_name }}
k0s-version: ${{ needs.get-tag.outputs.k0s_version }}
k0s-version-previous: ${{ needs.get-tag.outputs.k0s_version }} # we do not run k8s upgrade tests on release
k0s-version-previous-stable: ${{ needs.find-previous-stable.outputs.k0s_version }}
upgrade-target-ec-version: ${{ github.ref_name }}
# this job will validate that all the tests passed
validate-release-success:
name: Validate success
runs-on: ubuntu-20.04
needs:
- e2e
- e2e-docker
- release
- release-app
if: always()
steps:
# https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context
- name: fail if e2e job was not successful
if: needs.e2e.result != 'success'
run: exit 1
- name: fail if e2e-docker job was not successful
if: needs.e2e-docker.result != 'success'
run: exit 1
- name: succeed if everything else passed
run: echo "Validation succeeded"