add airgap upgrade CI test (#518)

replicatedhq · Apr 22, 2024 · bac06c8 · bac06c8
1 parent 5a2e411
commit bac06c8
Show file tree

Hide file tree

Showing 17 changed files with 526 additions and 127 deletions.
diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml
@@ -50,13 +50,14 @@ jobs:
       uses: actions/setup-go@v5
       with:
         go-version: "1.21.0"
-    - name: Build Linux AMD64
+    - name: Build Linux AMD64 and Output Metadata
       run: |
         export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
-        make embedded-cluster-linux-amd64 VERSION="${SHORT_SHA}"
+        make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) VERSION="${SHORT_SHA}-previous-k0s"
+        tar -C output/bin -czvf embedded-cluster-linux-amd64-previous-k0s.tgz embedded-cluster
+        ./output/bin/embedded-cluster version metadata > metadata-previous-k0s.json
+        make -B embedded-cluster-linux-amd64 VERSION="${SHORT_SHA}"
         tar -C output/bin -czvf embedded-cluster-linux-amd64.tgz embedded-cluster
-    - name: Output Metadata
-      run: |
         ./output/bin/embedded-cluster version metadata > metadata.json
     - name: Install Replicated CLI
       run: |
@@ -93,7 +94,9 @@ jobs:
         echo "versionLabel: \"appver-${SHORT_SHA}\"" >> e2e/kots-release-install/release.yaml
         cat e2e/kots-release-install/release.yaml
         cp output/bin/embedded-cluster output/bin/embedded-cluster-original
-        make embedded-release VERSION="${SHORT_SHA}" # this is done after the metadata.json is generated so as to not include additional charts
+        make -B embedded-release K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) VERSION="${SHORT_SHA}-previous-k0s"
+        mv output/bin/embedded-cluster output/bin/embedded-cluster-previous-k0s
+        make -B embedded-release VERSION="${SHORT_SHA}" # this is done after the metadata.json is generated so as to not include additional charts
     - name: Cache files for integration test
       env:
         S3_BUCKET: "tf-staging-embedded-cluster-bin"
@@ -105,6 +108,7 @@ jobs:
         export EC_VERSION="v${SHORT_SHA}"
         ./scripts/cache-files.sh
         ./scripts/create-upgrade-release.sh
+        ./scripts/create-previous-k0s-release.sh
 
     - name: Create Airgap Release
       env:
@@ -114,11 +118,12 @@ jobs:
       run: |
         export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
         echo "${SHORT_SHA}"
-        sed -i "s/__version_string__/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml
-        sed -i "s/__version_string__/${SHORT_SHA}-upgrade/g" e2e/kots-release-upgrade/cluster-config.yaml
+        # airgap tests install the previous k0s version to ensure an upgrade occurs
+        sed -i "s/${SHORT_SHA}/${SHORT_SHA}-previous-k0s/g" e2e/kots-release-install/cluster-config.yaml
         
         rm e2e/kots-release-install/release.yaml
         replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}"
+        replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI-airgap --version "appver-${SHORT_SHA}-upgrade"
 
     - name: Create download link message text
       run: |
@@ -149,6 +154,7 @@ jobs:
         path: |
           output/bin/embedded-cluster
           output/bin/embedded-cluster-original
+          output/bin/embedded-cluster-previous-k0s
           output/bin/embedded-cluster-release-builder
 
   e2e:
@@ -177,7 +183,7 @@ jobs:
           - TestOldVersionUpgrade
           - TestMaterialize
           - TestLocalArtifactMirror
-          - TestSingleNodeAirgapInstallationUbuntuJammy
+          - TestSingleNodeAirgapUpgradeUbuntuJammy
     steps:
     - name: Checkout
       uses: actions/checkout@v4

diff --git a/.github/workflows/release-dev.yaml b/.github/workflows/release-dev.yaml
@@ -17,13 +17,14 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: "1.21"
-      - name: Build Linux AMD64
+      - name: Build Linux AMD64 and Output Metadata
         run: |
           export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
-          make embedded-cluster-linux-amd64 VERSION="${SHORT_SHA}"
+          make -B embedded-cluster-linux-amd64 K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) VERSION="${SHORT_SHA}-previous-k0s"
+          tar -C output/bin -czvf embedded-cluster-linux-amd64-previous-k0s.tgz embedded-cluster
+          ./output/bin/embedded-cluster version metadata > metadata-previous-k0s.json
+          make -B embedded-cluster-linux-amd64 VERSION="${SHORT_SHA}"
           tar -C output/bin -czvf embedded-cluster-linux-amd64.tgz embedded-cluster
-      - name: Output Metadata
-        run: |
           ./output/bin/embedded-cluster version metadata > metadata.json
       - name: Publish development release
         uses: marvinpinto/action-automatic-releases@latest
@@ -70,7 +71,9 @@ jobs:
           echo "versionLabel: \"appver-${SHORT_SHA}\"" >> e2e/kots-release-install/release.yaml
           cat e2e/kots-release-install/release.yaml
           cp output/bin/embedded-cluster output/bin/embedded-cluster-original
-          make embedded-release VERSION="${SHORT_SHA}" # this is done after the metadata.json is generated so as to not include additional charts
+          make -B embedded-release K0S_VERSION=$(make print-PREVIOUS_K0S_VERSION) VERSION="${SHORT_SHA}-previous-k0s"
+          mv output/bin/embedded-cluster output/bin/embedded-cluster-previous-k0s
+          make -B embedded-release VERSION="${SHORT_SHA}" # this is done after the metadata.json is generated so as to not include additional charts
       - name: Cache files for integration test
         env:
           S3_BUCKET: "tf-staging-embedded-cluster-bin"
@@ -82,6 +85,7 @@ jobs:
           export EC_VERSION="v${SHORT_SHA}"
           ./scripts/cache-files.sh
           ./scripts/create-upgrade-release.sh
+          ./scripts/create-previous-k0s-release.sh
       - name: Create Airgap Release
         env:
           REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
@@ -90,11 +94,12 @@ jobs:
         run: |
           export SHORT_SHA=dev-$(git rev-parse --short=7 HEAD)
           echo "${SHORT_SHA}"
-          sed -i "s/__version_string__/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml
-          sed -i "s/__version_string__/${SHORT_SHA}-upgrade/g" e2e/kots-release-upgrade/cluster-config.yaml
+          # airgap tests install the previous k0s version to ensure an upgrade occurs
+          sed -i "s/${SHORT_SHA}/${SHORT_SHA}-previous-k0s/g" e2e/kots-release-install/cluster-config.yaml
           
           rm e2e/kots-release-install/release.yaml
           replicated release create --yaml-dir e2e/kots-release-install --promote CI-airgap --version "appver-${SHORT_SHA}"
+          replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI-airgap --version "appver-${SHORT_SHA}-upgrade"
 
       - name: upload binary
         uses: actions/upload-artifact@v4
@@ -103,6 +108,7 @@ jobs:
           path: |
             output/bin/embedded-cluster
             output/bin/embedded-cluster-original
+            output/bin/embedded-cluster-previous-k0s
             output/bin/embedded-cluster-release-builder
   e2e:
     runs-on: ubuntu-latest
@@ -130,7 +136,7 @@ jobs:
           - TestOldVersionUpgrade
           - TestMaterialize
           - TestLocalArtifactMirror
-          - TestSingleNodeAirgapInstallationUbuntuJammy
+          - TestSingleNodeAirgapUpgradeUbuntuJammy
     steps:
       - name: Checkout
         uses: actions/checkout@v4

diff --git a/Makefile b/Makefile
@@ -22,6 +22,7 @@ REGISTRY_CHART_VERSION = 2.2.3
 REGISTRY_IMAGE_VERSION = 2.8.3
 KUBECTL_VERSION = v1.30.0
 K0S_VERSION = v1.29.4+k0s.0
+PREVIOUS_K0S_VERSION ?= v1.28.9+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.87.0
 KOTS_VERSION = v$(shell echo $(ADMIN_CONSOLE_CHART_VERSION) | sed 's/\([0-9]\+\.[0-9]\+\.[0-9]\+\).*/\1/')
@@ -167,3 +168,6 @@ scan:
 		--severity="HIGH,CRITICAL" \
 		--ignore-unfixed \
 		./
+
+print-%:
+	@echo -n $($*)
diff --git a/e2e/cluster/cluster.go b/e2e/cluster/cluster.go
@@ -58,7 +58,8 @@ type Input struct {
 	LicensePath                       string
 	EmbeddedClusterPath               string
 	EmbeddedClusterReleaseBuilderPath string // used to replace the release in the binary
-	AirgapBundlePath                  string
+	AirgapInstallBundlePath           string
+	AirgapUpgradeBundlePath           string
 	Image                             string
 	network                           string
 	T                                 *testing.T
@@ -196,27 +197,33 @@ func NewTestCluster(in *Input) *Output {
 	nodes := CreateNodes(in)
 	for _, node := range nodes {
 		CopyFilesToNode(in, node)
+		CopyScriptsToNode(in, node)
 		if in.CreateRegularUser {
 			CreateRegularUser(in, node)
 		}
 	}
-	if in.WithProxy {
-		CreateProxy(in)
-	}
-	return &Output{
+	out := &Output{
 		T:       in.T,
 		Nodes:   nodes,
 		network: in.network,
 		id:      in.id,
 	}
+	if in.WithProxy {
+		out.Proxy = CreateProxy(in)
+		CopyScriptsToNode(in, out.Proxy)
+		if in.CreateRegularUser {
+			CreateRegularUser(in, out.Proxy)
+		}
+	}
+	return out
 }
 
 // CreateProxy creates a node that attaches to both networks (external and internal),
 // once this is done we install squid and configure it to be a proxy. We also make
 // sure that all nodes are configured to use the proxy as default gateway. Internet
 // won't work on them by design (exception made for DNS requests and http requests
 // using the proxy). Proxy is accessible from the cluster nodes on 10.0.0.254:3128.
-func CreateProxy(in *Input) {
+func CreateProxy(in *Input) string {
 	client, err := lxd.ConnectLXDUnix(lxdSocket, nil)
 	if err != nil {
 		in.T.Fatalf("Failed to connect to LXD: %v", err)
@@ -272,6 +279,7 @@ func CreateProxy(in *Input) {
 		}
 	}
 	ConfigureProxy(in)
+	return name
 }
 
 // ConfigureProxy installs squid and iptables on the target node. Configures the needed
@@ -372,8 +380,7 @@ func CreateRegularUser(in *Input, node string) {
 }
 
 // CopyFilesToNode copies the files needed for the cluster to the node. Copies
-// the provided ssh key, the embedded-cluster binary and also all scripts from the
-// scripts directory (they are all placed under /usr/local/bin inside the node).
+// the provided ssh key and the embedded-cluster release files.
 func CopyFilesToNode(in *Input, node string) {
 	client, err := lxd.ConnectLXDUnix(lxdSocket, nil)
 	if err != nil {
@@ -400,11 +407,24 @@ func CopyFilesToNode(in *Input, node string) {
 			Mode:       0755,
 		},
 		{
-			SourcePath: in.AirgapBundlePath,
+			SourcePath: in.AirgapInstallBundlePath,
 			DestPath:   "/tmp/ec-release.tgz",
 			Mode:       0755,
 		},
+		{
+			SourcePath: in.AirgapUpgradeBundlePath,
+			DestPath:   "/tmp/ec-release-upgrade.tgz",
+			Mode:       0755,
+		},
 	}
+	for _, file := range files {
+		CopyFileToNode(in, node, file)
+	}
+}
+
+// CopyScriptsToNode copies all scripts from the scripts directory
+// (they are all placed under /usr/local/bin inside the node).
+func CopyScriptsToNode(in *Input, node string) {
 	scriptFiles, err := scripts.FS.ReadDir(".")
 	if err != nil {
 		in.T.Fatalf("Failed to read scripts directory: %v", err)
@@ -423,13 +443,11 @@ func CopyFilesToNode(in *Input, node string) {
 			in.T.Fatalf("Failed to copy script %s: %v", script.Name(), err)
 		}
 		fp.Close()
-		files = append(files, File{
+		file := File{
 			SourcePath: tmp.Name(),
 			DestPath:   fmt.Sprintf("/usr/local/bin/%s", script.Name()),
 			Mode:       0755,
-		})
-	}
-	for _, file := range files {
+		}
 		CopyFileToNode(in, node, file)
 	}
 }