Merge remote-tracking branch 'origin/main' into k0s-1-28-9

.github/actions/e2e/action.yml

@@ -4,6 +4,9 @@ inputs:
   test-name:
     description: 'individual test to run'
     required: true
+  is-large-runner:
+    description: 'Whether the test is running on a large runner'
+    required: true
   airgap-license-id:
     description: 'airgap-enabled license id to use for e2e tests'
     required: true
@@ -61,6 +64,7 @@ runs:
            external_ids:ovn-encap-type=geneve                      \
            external_ids:ovn-encap-ip=127.0.0.1
   - name: Free up runner disk space
+    if: ${{ inputs.is-large-runner == 'false' }}
     shell: bash
     run: |
       df -h

.github/workflows/pull-request.yaml

@@ -194,11 +194,15 @@ jobs:
           - TestMultiNodeHAInstallation
           - TestMultiNodeAirgapHAInstallation
           - TestMultiNodeAirgapUpgradeSameK0s
+          - TestMultiNodeHADisasterRecovery
+          - TestMultiNodeAirgapHADisasterRecovery
         include:
           - test: TestMultiNodeAirgapUpgrade
             runner: embedded-cluster
           - test: TestMultiNodeAirgapHAInstallation
             runner: embedded-cluster
+          - test: TestMultiNodeAirgapHADisasterRecovery
+            runner: embedded-cluster
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -211,6 +215,7 @@ jobs:
     - uses: ./.github/actions/e2e
       with:
         test-name: '${{ matrix.test }}'
+        is-large-runner: ${{ matrix.runner == 'embedded-cluster' }}
         airgap-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }}
         snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }}
         snapshot-license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE }}

.github/workflows/release-dev.yaml

@@ -147,11 +147,15 @@ jobs:
           - TestMultiNodeHAInstallation
           - TestMultiNodeAirgapHAInstallation
           - TestMultiNodeAirgapUpgradeSameK0s
+          - TestMultiNodeHADisasterRecovery
+          - TestMultiNodeAirgapHADisasterRecovery
         include:
           - test: TestMultiNodeAirgapUpgrade
             runner: embedded-cluster
           - test: TestMultiNodeAirgapHAInstallation
             runner: embedded-cluster
+          - test: TestMultiNodeAirgapHADisasterRecovery
+            runner: embedded-cluster
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -163,6 +167,7 @@ jobs:
       - uses: ./.github/actions/e2e
         with:
           test-name: '${{ matrix.test }}'
+          is-large-runner: ${{ matrix.runner == 'embedded-cluster' }}
           airgap-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_AIRGAP_LICENSE_ID }}
           snapshot-license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE_ID }}
           snapshot-license: ${{ secrets.STAGING_EMBEDDED_CLUSTER_SNAPSHOT_LICENSE }}

.github/workflows/release-prod.yaml

@@ -14,7 +14,13 @@ jobs:
         with:
           fetch-depth: 0
       - name: Extract tag name
-        run: echo "TAG_NAME=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+        run: |
+          # remove the "refs/tags/" prefix to get the tag that was pushed
+          export RAW_TAG=${GITHUB_REF#refs/tags/}
+          # add a 'v' prefix to the tag if it doesn't already have one
+          export V_TAG=$(echo "$RAW_TAG" | sed 's/^[^v]/v&/')
+          # store the tag name in a github environment variable for later steps
+          echo "TAG_NAME=${V_TAG}" >> $GITHUB_ENV
       - name: Set up Go
         uses: actions/setup-go@v5
         with:

Makefile

@@ -4,12 +4,12 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_URL = oci://registry.replicated.com/library
 ADMIN_CONSOLE_CHART_NAME = admin-console
-ADMIN_CONSOLE_CHART_VERSION = 1.109.9-build.1
+ADMIN_CONSOLE_CHART_VERSION = 1.109.12
 ADMIN_CONSOLE_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
 EMBEDDED_OPERATOR_CHART_NAME = embedded-cluster-operator
-EMBEDDED_OPERATOR_CHART_VERSION = 0.34.6
+EMBEDDED_OPERATOR_CHART_VERSION = 0.34.9
 EMBEDDED_OPERATOR_UTILS_IMAGE = busybox:1.36.1
 EMBEDDED_CLUSTER_OPERATOR_IMAGE_OVERRIDE =
 OPENEBS_CHART_URL = https://openebs.github.io/openebs
@@ -18,7 +18,7 @@ OPENEBS_CHART_VERSION = 4.0.1
 OPENEBS_UTILS_VERSION = 4.0.0
 SEAWEEDFS_CHART_URL = https://seaweedfs.github.io/seaweedfs/helm
 SEAWEEDFS_CHART_NAME = seaweedfs/seaweedfs
-SEAWEEDFS_CHART_VERSION = 3.67.0
+SEAWEEDFS_CHART_VERSION = 3.68.0
 REGISTRY_CHART_URL = https://helm.twun.io
 REGISTRY_CHART_NAME = twuni/docker-registry
 REGISTRY_CHART_VERSION = 2.2.3

cmd/embedded-cluster/assets/resource-modifiers.yaml

@@ -0,0 +1,54 @@
+version: v1
+resourceModifierRules:
+# convert kotsadm components (rqlite) to non-HA mode
+# as kotsadm will always be restored to a single node
+# because it is used during the restore process to add nodes
+- conditions:
+    groupResource: statefulsets.apps
+    resourceNameRegex: "^kotsadm-rqlite$"
+    namespaces:
+    - kotsadm
+  patches:
+  - operation: replace
+    path: "/spec/replicas"
+    value: 1
+  - operation: replace
+    path: "/spec/template/spec/containers/0/args/2"
+    value: "-bootstrap-expect=1"
+# decouple kotsadm components PVCs from nodes
+# this allows the PVCs to be created on the correct nodes
+# when restoring HA kotsadm to a single node and then converting it to HA again
+- conditions:
+    groupResource: persistentvolumeclaims
+    resourceNameRegex: "kotsadm-rqlite"
+    namespaces:
+    - kotsadm
+  mergePatches:
+  - patchData: |
+      {
+        "metadata": {
+          "annotations": {
+            "volume.kubernetes.io/selected-node": null
+          }
+        }
+      }
+# preserve the registry service IP from the original cluster
+- conditions:
+    groupResource: services
+    resourceNameRegex: "^registry$"
+    namespaces:
+    - registry
+  patches:
+  - operation: replace
+    path: "/spec/clusterIP"
+    value: "__REGISTRY_SERVICE_IP__"
+# preserve the seaweedfs s3 service IP from the original cluster
+- conditions:
+    groupResource: services
+    resourceNameRegex: "^ec-seaweedfs-s3$"
+    namespaces:
+    - seaweedfs
+  patches:
+  - operation: replace
+    path: "/spec/clusterIP"
+    value: "__SEAWEEDFS_S3_SERVICE_IP__"

cmd/embedded-cluster/install.go

@@ -79,20 +79,6 @@ func configureNetworkManager(c *cli.Context) error {
 	return nil
 }
 
-// runPostInstall is a helper function that run things just after the k0s install
-// command ran.
-func runPostInstall() error {
-	src := "/etc/systemd/system/k0scontroller.service"
-	dst := fmt.Sprintf("/etc/systemd/system/%s.service", defaults.BinaryName())
-	if err := os.Symlink(src, dst); err != nil {
-		return fmt.Errorf("failed to create symlink: %w", err)
-	}
-	if _, err := helpers.RunCommand("systemctl", "daemon-reload"); err != nil {
-		return fmt.Errorf("unable to get reload systemctl daemon: %w", err)
-	}
-	return installAndEnableLocalArtifactMirror()
-}
-
 // RunHostPreflights runs the host preflights we found embedded in the binary
 // on all configured hosts. We attempt to read HostPreflights from all the
 // embedded Helm Charts and from the Kots Application Release files.
@@ -505,9 +491,9 @@ var installCommand = &cli.Command{
 			metrics.ReportApplyFinished(c, err)
 			return err
 		}
-		logrus.Debugf("running post install")
-		if err := runPostInstall(); err != nil {
-			err := fmt.Errorf("unable to run post install: %w", err)
+		logrus.Debugf("creating systemd unit files")
+		if err := createSystemdUnitFiles(false); err != nil {
+			err := fmt.Errorf("unable to create systemd unit files: %w", err)
 			metrics.ReportApplyFinished(c, err)
 			return err
 		}

cmd/embedded-cluster/join.go

@@ -18,8 +18,9 @@ import (
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	k8syaml "sigs.k8s.io/yaml"
 
@@ -212,7 +213,8 @@ var joinCommand = &cli.Command{
 		}
 
 		logrus.Debugf("creating systemd unit files")
-		if err := createSystemdUnitFiles(jcmd.K0sJoinCommand); err != nil {
+		// both controller and worker nodes will have 'worker' in the join command
+		if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller")); err != nil {
 			err := fmt.Errorf("unable to create systemd unit files: %w", err)
 			metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
 			return err
@@ -386,22 +388,22 @@ func systemdUnitFileName() string {
 
 // createSystemdUnitFiles links the k0s systemd unit file. this also creates a new
 // systemd unit file for the local artifact mirror service.
-func createSystemdUnitFiles(fullcmd string) error {
+func createSystemdUnitFiles(isWorker bool) error {
 	dst := systemdUnitFileName()
-	if _, err := os.Stat(dst); err == nil {
+	if _, err := os.Lstat(dst); err == nil {
 		if err := os.Remove(dst); err != nil {
 			return err
 		}
 	}
 	src := "/etc/systemd/system/k0scontroller.service"
-	if strings.Contains(fullcmd, "worker") {
+	if isWorker {
 		src = "/etc/systemd/system/k0sworker.service"
 	}
 	if err := os.Symlink(src, dst); err != nil {
-		return err
+		return fmt.Errorf("failed to create symlink: %w", err)
 	}
 	if _, err := helpers.RunCommand("systemctl", "daemon-reload"); err != nil {
-		return err
+		return fmt.Errorf("unable to get reload systemctl daemon: %w", err)
 	}
 	return installAndEnableLocalArtifactMirror()
 }
@@ -459,17 +461,16 @@ func canEnableHA(ctx context.Context, kcli client.Client) (bool, error) {
 	if installation.Spec.HighAvailability {
 		return false, nil
 	}
-	var nodes corev1.NodeList
-	labelSelector := labels.Set(map[string]string{
-		"node-role.kubernetes.io/control-plane": "true",
-	}).AsSelector()
-	if err := kcli.List(ctx, &nodes, &client.ListOptions{LabelSelector: labelSelector}); err != nil {
-		return false, fmt.Errorf("unable to list nodes: %w", err)
+	if err := kcli.Get(ctx, types.NamespacedName{Name: ecRestoreStateCMName, Namespace: "embedded-cluster"}, &corev1.ConfigMap{}); err == nil {
+		return false, nil // cannot enable HA during a restore
+	} else if !errors.IsNotFound(err) {
+		return false, fmt.Errorf("unable to get restore state configmap: %w", err)
 	}
-	if len(nodes.Items) < 3 {
-		return false, nil
+	ncps, err := kubeutils.NumOfControlPlaneNodes(ctx, kcli)
+	if err != nil {
+		return false, fmt.Errorf("unable to check control plane nodes: %w", err)
 	}
-	return true, nil
+	return ncps >= 3, nil
 }
 
 // enableHA enables high availability in the installation object

cmd/embedded-cluster/join_test.go

@@ -190,6 +190,24 @@ func Test_canEnableHA(t *testing.T) {
 			},
 			want: false,
 		},
+		{
+			name: "high availability is not enabled and there is three or more controller nodes but a restore is in progress",
+			args: args{
+				kcli: fake.NewClientBuilder().WithScheme(scheme).WithObjects(
+					&embeddedclusterv1beta1.Installation{
+						ObjectMeta: metav1.ObjectMeta{Name: "test-installation"},
+						Spec:       embeddedclusterv1beta1.InstallationSpec{HighAvailability: false},
+					},
+					&corev1.ConfigMap{
+						ObjectMeta: metav1.ObjectMeta{Name: ecRestoreStateCMName, Namespace: "embedded-cluster"},
+					},
+					&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node1", Labels: controllerLabels}},
+					&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node2", Labels: controllerLabels}},
+					&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node3", Labels: controllerLabels}},
+				).Build(),
+			},
+			want: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {