From 3829fb090b5bf066052e91e7b0f479e0358f7569 Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Fri, 26 Jul 2024 10:45:14 -0700 Subject: [PATCH] Add support for Velero 1.14 (#4782) --- pkg/embeddedcluster/util.go | 6 +- pkg/kotsadmsnapshot/backup.go | 71 ++++++---- pkg/kotsadmsnapshot/backup_test.go | 202 +++++++++++++++++++++++++++-- 3 files changed, 238 insertions(+), 41 deletions(-) diff --git a/pkg/embeddedcluster/util.go b/pkg/embeddedcluster/util.go index 8a7ece4667..1ca7267e23 100644 --- a/pkg/embeddedcluster/util.go +++ b/pkg/embeddedcluster/util.go @@ -18,8 +18,8 @@ import ( ) const ( - seaweedfsNamespace = "seaweedfs" - seaweedfsS3SVCName = "ec-seaweedfs-s3" + SeaweedfsNamespace = "seaweedfs" + SeaweedfsS3SVCName = "ec-seaweedfs-s3" ) // ErrNoInstallations is returned when no installation object is found in the cluster. @@ -101,7 +101,7 @@ func ClusterConfig(ctx context.Context, kbClient kbclient.Client) (*embeddedclus } func GetSeaweedFSS3ServiceIP(ctx context.Context, kbClient kbclient.Client) (string, error) { - nsn := k8stypes.NamespacedName{Name: seaweedfsS3SVCName, Namespace: seaweedfsNamespace} + nsn := k8stypes.NamespacedName{Name: SeaweedfsS3SVCName, Namespace: SeaweedfsNamespace} var svc corev1.Service if err := kbClient.Get(ctx, nsn, &svc); err != nil && !k8serrors.IsNotFound(err) { return "", fmt.Errorf("failed to get seaweedfs s3 service: %w", err) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index c323250773..10df057231 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -12,6 +12,7 @@ import ( units "github.com/docker/go-units" "github.com/pkg/errors" + embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1" downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types" apptypes "github.com/replicatedhq/kots/pkg/app/types" "github.com/replicatedhq/kots/pkg/embeddedcluster" @@ -35,6 +36,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled bool) (*velerov1.Backup, error) { @@ -128,7 +130,7 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b includedNamespaces = append(includedNamespaces, veleroBackup.Spec.IncludedNamespaces...) includedNamespaces = append(includedNamespaces, kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) - veleroBackup.Spec.IncludedNamespaces = prepareIncludedNamespaces(includedNamespaces, util.IsEmbeddedCluster()) + veleroBackup.Spec.IncludedNamespaces = prepareIncludedNamespaces(includedNamespaces) snapshotTrigger := "manual" if isScheduled { @@ -382,21 +384,14 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre if err != nil { return nil, fmt.Errorf("failed to get current installation: %w", err) } - seaweedFSS3ServiceIP, err := embeddedcluster.GetSeaweedFSS3ServiceIP(ctx, kbClient) + ecAnnotations, err := ecBackupAnnotations(ctx, kbClient, installation) if err != nil { - return nil, fmt.Errorf("failed to get seaweedfs s3 service ip: %w", err) + return nil, fmt.Errorf("failed to get embedded cluster backup annotations: %w", err) } - if seaweedFSS3ServiceIP != "" { - backupAnnotations["kots.io/embedded-cluster-seaweedfs-s3-ip"] = seaweedFSS3ServiceIP - } - backupAnnotations["kots.io/embedded-cluster"] = "true" - backupAnnotations["kots.io/embedded-cluster-id"] = util.EmbeddedClusterID() - backupAnnotations["kots.io/embedded-cluster-version"] = util.EmbeddedClusterVersion() - backupAnnotations["kots.io/embedded-cluster-is-ha"] = strconv.FormatBool(installation.Spec.HighAvailability) - if installation.Spec.Network != nil { - backupAnnotations["kots.io/embedded-cluster-pod-cidr"] = installation.Spec.Network.PodCIDR - backupAnnotations["kots.io/embedded-cluster-service-cidr"] = installation.Spec.Network.ServiceCIDR + for k, v := range ecAnnotations { + backupAnnotations[k] = v } + includedNamespaces = append(includedNamespaces, ecIncludedNamespaces(installation)...) } includeClusterResources := true @@ -409,7 +404,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre }, Spec: velerov1.BackupSpec{ StorageLocation: "default", - IncludedNamespaces: prepareIncludedNamespaces(includedNamespaces, util.IsEmbeddedCluster()), + IncludedNamespaces: prepareIncludedNamespaces(includedNamespaces), ExcludedNamespaces: excludedNamespaces, IncludeClusterResources: &includeClusterResources, OrLabelSelectors: instanceBackupLabelSelectors(util.IsEmbeddedCluster()), @@ -985,11 +980,47 @@ func mergeLabelSelector(kots metav1.LabelSelector, app metav1.LabelSelector) met return kots } +// ecBackupAnnotations returns the annotations that should be added to an embedded cluster backup +func ecBackupAnnotations(ctx context.Context, kbClient kbclient.Client, in *embeddedclusterv1beta1.Installation) (map[string]string, error) { + annotations := map[string]string{} + + seaweedFSS3ServiceIP, err := embeddedcluster.GetSeaweedFSS3ServiceIP(ctx, kbClient) + if err != nil { + return nil, fmt.Errorf("failed to get seaweedfs s3 service ip: %w", err) + } + if seaweedFSS3ServiceIP != "" { + annotations["kots.io/embedded-cluster-seaweedfs-s3-ip"] = seaweedFSS3ServiceIP + } + + annotations["kots.io/embedded-cluster"] = "true" + annotations["kots.io/embedded-cluster-id"] = util.EmbeddedClusterID() + annotations["kots.io/embedded-cluster-version"] = util.EmbeddedClusterVersion() + annotations["kots.io/embedded-cluster-is-ha"] = strconv.FormatBool(in.Spec.HighAvailability) + + if in.Spec.Network != nil { + annotations["kots.io/embedded-cluster-pod-cidr"] = in.Spec.Network.PodCIDR + annotations["kots.io/embedded-cluster-service-cidr"] = in.Spec.Network.ServiceCIDR + } + + return annotations, nil +} + +// ecIncludedNamespaces returns the namespaces that should be included in an embedded cluster backup +func ecIncludedNamespaces(in *embeddedclusterv1beta1.Installation) []string { + includedNamespaces := []string{"embedded-cluster", "kube-system", "openebs"} + if in.Spec.AirGap { + includedNamespaces = append(includedNamespaces, "registry") + if in.Spec.HighAvailability { + includedNamespaces = append(includedNamespaces, "seaweedfs") + } + } + return includedNamespaces +} + // Prepares the list of unique namespaces that will be included in a backup. Empty namespaces are excluded. // If a wildcard is specified, any specific namespaces will not be included since the backup will include all namespaces. // Velero does not allow for both a wildcard and specific namespaces and will consider the backup invalid if both are present. -// If this is an embedded-cluster installation, the "embedded-cluster", "openebs" and "kube-system" namespaces will be included. -func prepareIncludedNamespaces(namespaces []string, isEC bool) []string { +func prepareIncludedNamespaces(namespaces []string) []string { uniqueNamespaces := make(map[string]bool) for _, n := range namespaces { if n == "" { @@ -1000,14 +1031,6 @@ func prepareIncludedNamespaces(namespaces []string, isEC bool) []string { uniqueNamespaces[n] = true } - if isEC { - uniqueNamespaces["embedded-cluster"] = true - uniqueNamespaces["kube-system"] = true - uniqueNamespaces["openebs"] = true - uniqueNamespaces["registry"] = true - uniqueNamespaces["seaweedfs"] = true - } - includedNamespaces := make([]string, len(uniqueNamespaces)) i := 0 for k := range uniqueNamespaces { diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 9386d2c790..dc6cb5fb5f 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -4,6 +4,8 @@ import ( "context" "testing" + embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1" + "github.com/replicatedhq/kots/pkg/embeddedcluster" kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -17,6 +19,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" coretest "k8s.io/client-go/testing" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + fakekbclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestPrepareIncludedNamespaces(t *testing.T) { @@ -24,7 +28,6 @@ func TestPrepareIncludedNamespaces(t *testing.T) { name string namespaces []string want []string - isEC bool }{ { name: "empty", @@ -76,23 +79,11 @@ func TestPrepareIncludedNamespaces(t *testing.T) { namespaces: []string{"*", "", "test"}, want: []string{"*"}, }, - { - name: "wildcard with embedded cluster", - namespaces: []string{"*", "test"}, - want: []string{"*"}, - isEC: true, - }, - { - name: "embedded-cluster install", - namespaces: []string{"test", "abcapp"}, - want: []string{"test", "abcapp", "embedded-cluster", "kube-system", "openebs", "registry", "seaweedfs"}, - isEC: true, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := prepareIncludedNamespaces(tt.namespaces, tt.isEC) + got := prepareIncludedNamespaces(tt.namespaces) if !assert.ElementsMatch(t, tt.want, got) { t.Errorf("prepareIncludedNamespaces() = %v, want %v", got, tt.want) } @@ -690,3 +681,186 @@ func Test_instanceBackupLabelSelectors(t *testing.T) { }) } } + +func Test_ecBackupAnnotations(t *testing.T) { + scheme := runtime.NewScheme() + corev1.AddToScheme(scheme) + embeddedclusterv1beta1.AddToScheme(scheme) + + tests := []struct { + name string + kbClient kbclient.Client + in *embeddedclusterv1beta1.Installation + env map[string]string + want map[string]string + }{ + { + name: "basic", + kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).Build(), + in: &embeddedclusterv1beta1.Installation{}, + env: map[string]string{ + "EMBEDDED_CLUSTER_ID": "embedded-cluster-id", + "EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version", + }, + want: map[string]string{ + "kots.io/embedded-cluster": "true", + "kots.io/embedded-cluster-id": "embedded-cluster-id", + "kots.io/embedded-cluster-version": "embedded-cluster-version", + "kots.io/embedded-cluster-is-ha": "false", + }, + }, + { + name: "online ha", + kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).Build(), + in: &embeddedclusterv1beta1.Installation{ + Spec: embeddedclusterv1beta1.InstallationSpec{ + HighAvailability: true, + }, + }, + env: map[string]string{ + "EMBEDDED_CLUSTER_ID": "embedded-cluster-id", + "EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version", + }, + want: map[string]string{ + "kots.io/embedded-cluster": "true", + "kots.io/embedded-cluster-id": "embedded-cluster-id", + "kots.io/embedded-cluster-version": "embedded-cluster-version", + "kots.io/embedded-cluster-is-ha": "true", + }, + }, + { + name: "airgap ha", + kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).WithObjects( + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: embeddedcluster.SeaweedfsS3SVCName, + Namespace: embeddedcluster.SeaweedfsNamespace, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.96.0.10", + }, + }, + ).Build(), + in: &embeddedclusterv1beta1.Installation{ + Spec: embeddedclusterv1beta1.InstallationSpec{ + HighAvailability: true, + AirGap: true, + }, + }, + env: map[string]string{ + "EMBEDDED_CLUSTER_ID": "embedded-cluster-id", + "EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version", + }, + want: map[string]string{ + "kots.io/embedded-cluster": "true", + "kots.io/embedded-cluster-id": "embedded-cluster-id", + "kots.io/embedded-cluster-version": "embedded-cluster-version", + "kots.io/embedded-cluster-is-ha": "true", + "kots.io/embedded-cluster-seaweedfs-s3-ip": "10.96.0.10", + }, + }, + { + name: "with pod and service cidrs", + kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).Build(), + in: &embeddedclusterv1beta1.Installation{ + Spec: embeddedclusterv1beta1.InstallationSpec{ + Network: &embeddedclusterv1beta1.NetworkSpec{ + PodCIDR: "10.128.0.0/20", + ServiceCIDR: "10.129.0.0/20", + }, + }, + }, + env: map[string]string{ + "EMBEDDED_CLUSTER_ID": "embedded-cluster-id", + "EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version", + }, + want: map[string]string{ + "kots.io/embedded-cluster": "true", + "kots.io/embedded-cluster-id": "embedded-cluster-id", + "kots.io/embedded-cluster-version": "embedded-cluster-version", + "kots.io/embedded-cluster-is-ha": "false", + "kots.io/embedded-cluster-pod-cidr": "10.128.0.0/20", + "kots.io/embedded-cluster-service-cidr": "10.129.0.0/20", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := require.New(t) + for k, v := range tt.env { + t.Setenv(k, v) + } + got, err := ecBackupAnnotations(context.TODO(), tt.kbClient, tt.in) + req.NoError(err) + req.Equal(tt.want, got) + }) + } +} + +func Test_ecIncludedNamespaces(t *testing.T) { + tests := []struct { + name string + in *embeddedclusterv1beta1.Installation + want []string + }{ + { + name: "online", + in: &embeddedclusterv1beta1.Installation{}, + want: []string{ + "embedded-cluster", + "kube-system", + "openebs", + }, + }, + { + name: "online ha", + in: &embeddedclusterv1beta1.Installation{ + Spec: embeddedclusterv1beta1.InstallationSpec{ + HighAvailability: true, + }, + }, + want: []string{ + "embedded-cluster", + "kube-system", + "openebs", + }, + }, + { + name: "airgap", + in: &embeddedclusterv1beta1.Installation{ + Spec: embeddedclusterv1beta1.InstallationSpec{ + AirGap: true, + }, + }, + want: []string{ + "embedded-cluster", + "kube-system", + "openebs", + "registry", + }, + }, + { + name: "airgap ha", + in: &embeddedclusterv1beta1.Installation{ + Spec: embeddedclusterv1beta1.InstallationSpec{ + HighAvailability: true, + AirGap: true, + }, + }, + want: []string{ + "embedded-cluster", + "kube-system", + "openebs", + "registry", + "seaweedfs", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := require.New(t) + got := ecIncludedNamespaces(tt.in) + req.Equal(tt.want, got) + }) + } +}