From 930f4fcfd3f9b43722c642593382bfb46e689e9b Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Wed, 1 May 2024 20:45:31 +0800 Subject: [PATCH] backup more namespaces in Embedded Cluster (#4583) * backup more namespaces in Embedded Cluster * backup installation kinds * allow any kots.io/backup labels to be backed up in embedded cluster * tag installation kind with 'infrastructure' tag * refactors and improvements * you CAN have multiple label selectors * no double slash label * change to a replicated.com label * simplify what labels we back up * use ec-install label value for installation object * pass entire labelselector to k8s * proper labelselector fix * add 'this is an embedded cluster' annotation to backup only if true * remove unused const * logic issue * add a unit test that actually validates that the label selector generation works * add the embedded cluster ID to the backup metadata --- pkg/embeddedcluster/util.go | 3 + pkg/kotsadmsnapshot/backup.go | 85 ++++++++--- pkg/kotsadmsnapshot/backup_test.go | 217 ++++++++++++++++++++++++++++- pkg/util/util.go | 4 + 4 files changed, 288 insertions(+), 21 deletions(-) diff --git a/pkg/embeddedcluster/util.go b/pkg/embeddedcluster/util.go index 4eedbe8cc4..dbe56f9c54 100644 --- a/pkg/embeddedcluster/util.go +++ b/pkg/embeddedcluster/util.go @@ -135,6 +135,9 @@ func startClusterUpgrade(ctx context.Context, newcfg embeddedclusterv1beta1.Conf newins := embeddedclusterv1beta1.Installation{ ObjectMeta: metav1.ObjectMeta{ Name: time.Now().Format("20060102150405"), + Labels: map[string]string{ + "replicated.com/disaster-recovery": "ec-install", + }, }, Spec: embeddedclusterv1beta1.InstallationSpec{ ClusterID: current.Spec.ClusterID, diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 94ac4e9ddc..15380df70e 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -126,7 +126,7 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b includedNamespaces = append(includedNamespaces, veleroBackup.Spec.IncludedNamespaces...) includedNamespaces = append(includedNamespaces, kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) - veleroBackup.Spec.IncludedNamespaces = prepareIncludedNamespaces(includedNamespaces) + veleroBackup.Spec.IncludedNamespaces = prepareIncludedNamespaces(includedNamespaces, util.IsEmbeddedCluster()) snapshotTrigger := "manual" if isScheduled { @@ -359,6 +359,10 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre backupAnnotations["kots.io/kotsadm-image"] = kotsadmImage backupAnnotations["kots.io/kotsadm-deploy-namespace"] = kotsadmNamespace backupAnnotations["kots.io/apps-sequences"] = marshalledAppsSequences + if util.IsEmbeddedCluster() { + backupAnnotations["kots.io/embedded-cluster"] = "true" + backupAnnotations["kots.io/embedded-cluster-id"] = util.EmbeddedClusterID() + } includeClusterResources := true veleroBackup := &velerov1.Backup{ @@ -370,17 +374,12 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre }, Spec: velerov1.BackupSpec{ StorageLocation: "default", - IncludedNamespaces: prepareIncludedNamespaces(includedNamespaces), + IncludedNamespaces: prepareIncludedNamespaces(includedNamespaces, util.IsEmbeddedCluster()), ExcludedNamespaces: excludedNamespaces, IncludeClusterResources: &includeClusterResources, - LabelSelector: &metav1.LabelSelector{ - // app label selectors are not supported and we can't merge them since that might exclude kotsadm components - MatchLabels: map[string]string{ - kotsadmtypes.BackupLabel: kotsadmtypes.BackupLabelValue, - }, - }, - OrderedResources: backupOrderedResources, - Hooks: backupHooks, + LabelSelector: instanceBackupLabelSelector(util.IsEmbeddedCluster()), + OrderedResources: backupOrderedResources, + Hooks: backupHooks, }, } @@ -951,7 +950,8 @@ func mergeLabelSelector(kots metav1.LabelSelector, app metav1.LabelSelector) met // Prepares the list of unique namespaces that will be included in a backup. Empty namespaces are excluded. // If a wildcard is specified, any specific namespaces will not be included since the backup will include all namespaces. // Velero does not allow for both a wildcard and specific namespaces and will consider the backup invalid if both are present. -func prepareIncludedNamespaces(namespaces []string) []string { +// If this is an embedded-cluster installation, the "embedded-cluster", "openebs" and "kube-system" namespaces will be included. +func prepareIncludedNamespaces(namespaces []string, isEC bool) []string { uniqueNamespaces := make(map[string]bool) for _, n := range namespaces { if n == "" { @@ -962,6 +962,12 @@ func prepareIncludedNamespaces(namespaces []string) []string { uniqueNamespaces[n] = true } + if isEC { + uniqueNamespaces["embedded-cluster"] = true + uniqueNamespaces["kube-system"] = true + uniqueNamespaces["openebs"] = true + } + includedNamespaces := make([]string, len(uniqueNamespaces)) i := 0 for k := range uniqueNamespaces { @@ -977,18 +983,34 @@ func excludeShutdownPodsFromBackup(ctx context.Context, clientset kubernetes.Int "status.phase": string(corev1.PodFailed), } - podListOption := metav1.ListOptions{ - LabelSelector: labels.SelectorFromSet(veleroBackup.Spec.LabelSelector.MatchLabels).String(), - FieldSelector: fields.SelectorFromSet(selectorMap).String(), + labelSets := []string{} + if veleroBackup.Spec.LabelSelector.MatchLabels != nil && len(veleroBackup.Spec.LabelSelector.MatchLabels) != 0 { + labelSets = []string{labels.SelectorFromSet(veleroBackup.Spec.LabelSelector.MatchLabels).String()} + } else { + for _, expr := range veleroBackup.Spec.LabelSelector.MatchExpressions { + if expr.Operator != metav1.LabelSelectorOpIn { + return fmt.Errorf("unsupported operator %s in label selector %q", expr.Operator, veleroBackup.Spec.LabelSelector.String()) + } + for _, value := range expr.Values { + labelSets = append(labelSets, fmt.Sprintf("%s=%s", expr.Key, value)) + } + } } - for _, namespace := range veleroBackup.Spec.IncludedNamespaces { - if namespace == "*" { - namespace = "" // specifying an empty ("") namespace in client-go retrieves resources from all namespaces + for _, labelSet := range labelSets { + podListOption := metav1.ListOptions{ + LabelSelector: labelSet, + FieldSelector: fields.SelectorFromSet(selectorMap).String(), } - if err := excludeShutdownPodsFromBackupInNamespace(ctx, clientset, namespace, podListOption); err != nil { - return errors.Wrap(err, "failed to exclude shutdown pods from backup") + for _, namespace := range veleroBackup.Spec.IncludedNamespaces { + if namespace == "*" { + namespace = "" // specifying an empty ("") namespace in client-go retrieves resources from all namespaces + } + + if err := excludeShutdownPodsFromBackupInNamespace(ctx, clientset, namespace, podListOption); err != nil { + return errors.Wrap(err, "failed to exclude shutdown pods from backup") + } } } @@ -1023,3 +1045,28 @@ func excludeShutdownPodsFromBackupInNamespace(ctx context.Context, clientset kub } return nil } + +func instanceBackupLabelSelector(isEmbeddedCluster bool) *metav1.LabelSelector { + if isEmbeddedCluster { // only DR on embedded-cluster + return &metav1.LabelSelector{ + MatchLabels: map[string]string{}, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "replicated.com/disaster-recovery", + Operator: metav1.LabelSelectorOpIn, + Values: []string{ + "infra", + "app", + "ec-install", + }, + }, + }, + } + } + + return &metav1.LabelSelector{ + MatchLabels: map[string]string{ + kotsadmtypes.BackupLabel: kotsadmtypes.BackupLabelValue, + }, + } +} diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 01b1babf5b..d5f9256b0a 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -6,6 +6,7 @@ import ( kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" corev1 "k8s.io/api/core/v1" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" @@ -23,6 +24,7 @@ func TestPrepareIncludedNamespaces(t *testing.T) { name string namespaces []string want []string + isEC bool }{ { name: "empty", @@ -74,11 +76,23 @@ func TestPrepareIncludedNamespaces(t *testing.T) { namespaces: []string{"*", "", "test"}, want: []string{"*"}, }, + { + name: "wildcard with embedded cluster", + namespaces: []string{"*", "test"}, + want: []string{"*"}, + isEC: true, + }, + { + name: "embedded-cluster install", + namespaces: []string{"test", "abcapp"}, + want: []string{"test", "abcapp", "embedded-cluster", "kube-system", "openebs"}, + isEC: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := prepareIncludedNamespaces(tt.namespaces) + got := prepareIncludedNamespaces(tt.namespaces, tt.isEC) if !assert.ElementsMatch(t, tt.want, got) { t.Errorf("prepareIncludedNamespaces() = %v, want %v", got, tt.want) } @@ -238,6 +252,16 @@ var appSlugLabelSelector = &metav1.LabelSelector{ }, } +var appSlugMatchExpression = &metav1.LabelSelector{ + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "kots.io/app-slug", + Operator: metav1.LabelSelectorOpIn, + Values: []string{"abc-slug", "test-slug", "xyz-slug"}, + }, + }, +} + var appSlugPodListOption = metav1.ListOptions{ LabelSelector: labels.SelectorFromSet(appSlugLabelSelector.MatchLabels).String(), FieldSelector: fields.SelectorFromSet(selectorMap).String(), @@ -439,7 +463,7 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { wantErr: false, }, { - name: "expect no error when shutdown pods are found and updated for kotsadm backup label and nanespace is *", + name: "expect no error when shutdown pods are found and updated for kotsadm backup label and namespace is *", args: args{ ctx: context.TODO(), clientset: mockK8sClientWithShutdownPods(), @@ -452,6 +476,20 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { }, wantErr: false, }, + { + name: "expect no error when shutdown pods are found and updated for app slug match expression", + args: args{ + ctx: context.TODO(), + clientset: mockK8sClientWithShutdownPods(), + veleroBackup: &velerov1.Backup{ + Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"test"}, + LabelSelector: appSlugMatchExpression, + }, + }, + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -461,3 +499,178 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { }) } } + +func Test_excludeShutdownPodsFromBackup_check(t *testing.T) { + res := []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "different-app-test-pod", + Namespace: "test", + Labels: map[string]string{ + "kots.io/app-slug": "not-test-slug", + }, + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: "Failed", + Reason: "Shutdown", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other-included-app-test-pod", + Namespace: "test", + Labels: map[string]string{ + "kots.io/app-slug": "abc-slug", + }, + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: "Failed", + Reason: "Shutdown", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "running-test-pod", + Namespace: "test", + Labels: map[string]string{ + "kots.io/app-slug": "test-slug", + }, + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: "Running", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "already-labelled-test-pod", + Namespace: "test", + Labels: map[string]string{ + "kots.io/app-slug": "test-slug", + "velero.io/exclude-from-backup": "true", + }, + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: "Failed", + Reason: "Shutdown", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "needs-label-test-pod", + Namespace: "test", + Labels: map[string]string{ + "kots.io/app-slug": "test-slug", + }, + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{ + Phase: "Failed", + Reason: "Shutdown", + }, + }, + } + + type args struct { + veleroBackup *velerov1.Backup + } + tests := []struct { + name string + args args + resources []runtime.Object + wantExcluded []string + }{ + { + name: "expect label selector to work", + wantExcluded: []string{"already-labelled-test-pod", "needs-label-test-pod"}, + args: args{ + veleroBackup: &velerov1.Backup{ + Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"test"}, + LabelSelector: appSlugLabelSelector, + }, + }, + }, + resources: res, + }, + { + name: "expect match expression to work", + wantExcluded: []string{"other-included-app-test-pod", "already-labelled-test-pod", "needs-label-test-pod"}, + args: args{ + veleroBackup: &velerov1.Backup{ + Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"test"}, + LabelSelector: appSlugMatchExpression, + }, + }, + }, + resources: res, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := require.New(t) + mockClient := fake.NewSimpleClientset(tt.resources...) + + err := excludeShutdownPodsFromBackup(context.TODO(), mockClient, tt.args.veleroBackup) + req.NoError(err) + + // count the number of pods with exclude annotation + testPods, err := mockClient.CoreV1().Pods("test").List(context.TODO(), metav1.ListOptions{}) + req.NoError(err) + + foundExcluded := []string{} + for _, pod := range testPods.Items { + if _, ok := pod.Labels["velero.io/exclude-from-backup"]; ok { + foundExcluded = append(foundExcluded, pod.Name) + } + } + + req.ElementsMatch(tt.wantExcluded, foundExcluded) + }) + } +} + +func Test_instanceBackupLabelSelector(t *testing.T) { + tests := []struct { + name string + isEmbeddedCluster bool + want *metav1.LabelSelector + }{ + { + name: "not embedded cluster", + isEmbeddedCluster: false, + want: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "kots.io/backup": "velero", + }, + }, + }, + { + name: "embedded cluster", + isEmbeddedCluster: true, + want: &metav1.LabelSelector{ + MatchLabels: map[string]string{}, + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "replicated.com/disaster-recovery", + Operator: metav1.LabelSelectorOpIn, + Values: []string{ + "infra", + "app", + "ec-install", + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, instanceBackupLabelSelector(tt.isEmbeddedCluster), "instanceBackupLabelSelector(%v)", tt.isEmbeddedCluster) + }) + } +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 2948ddce09..b8962882f8 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -169,6 +169,10 @@ func IsEmbeddedCluster() bool { return os.Getenv("EMBEDDED_CLUSTER_ID") != "" } +func EmbeddedClusterID() string { + return os.Getenv("EMBEDDED_CLUSTER_ID") +} + func GetValueFromMapPath(m interface{}, path []string) interface{} { if len(path) == 0 { return nil