diff --git a/migrations/tables/app_version.yaml b/migrations/tables/app_version.yaml index f24b8116d2..b9d2054800 100644 --- a/migrations/tables/app_version.yaml +++ b/migrations/tables/app_version.yaml @@ -67,6 +67,8 @@ spec: type: text - name: backup_spec type: text + - name: restore_spec + type: text - name: identity_spec type: text - name: branding_archive diff --git a/pkg/handlers/backup.go b/pkg/handlers/backup.go index 2d850f2e1c..ba3713d36a 100644 --- a/pkg/handlers/backup.go +++ b/pkg/handlers/backup.go @@ -206,7 +206,7 @@ func (h *Handler) CreateInstanceBackup(w http.ResponseWriter, r *http.Request) { } c := clusters[0] - backup, err := snapshot.CreateInstanceBackup(context.TODO(), c, false) + backupName, err := snapshot.CreateInstanceBackup(context.TODO(), c, false) if err != nil { logger.Error(errors.Wrap(err, "failed to create instance snapshot")) createInstanceBackupResponse.Error = "failed to create instance backup" @@ -215,7 +215,7 @@ func (h *Handler) CreateInstanceBackup(w http.ResponseWriter, r *http.Request) { } createInstanceBackupResponse.Success = true - createInstanceBackupResponse.BackupName = backup.ObjectMeta.Name + createInstanceBackupResponse.BackupName = backupName JSON(w, http.StatusOK, createInstanceBackupResponse) } diff --git a/pkg/handlers/restore.go b/pkg/handlers/restore.go index 322539282b..47bbd7919f 100644 --- a/pkg/handlers/restore.go +++ b/pkg/handlers/restore.go @@ -47,6 +47,14 @@ func (h *Handler) CreateApplicationRestore(w http.ResponseWriter, r *http.Reques return } + if snapshot.IsInstanceBackup(*backup) && snapshot.GetInstanceBackupType(*backup) != snapshottypes.InstanceBackupTypeLegacy { + err := errors.New("only legacy type instance backups are restorable") + logger.Error(err) + createRestoreResponse.Error = err.Error() + JSON(w, http.StatusInternalServerError, createRestoreResponse) + return + } + appID := backup.Annotations["kots.io/app-id"] sequence, err := strconv.ParseInt(backup.Annotations["kots.io/app-sequence"], 10, 64) if err != nil { @@ -149,7 +157,7 @@ func (h *Handler) RestoreApps(w http.ResponseWriter, r *http.Request) { return } - if backup.Annotations["kots.io/instance"] != "true" { + if backup.Annotations[snapshottypes.InstanceBackupAnnotation] != "true" { err := errors.Errorf("backup %s is not an instance backup", backup.ObjectMeta.Name) logger.Error(err) restoreResponse.Error = err.Error() @@ -244,7 +252,7 @@ func (h *Handler) GetRestoreAppsStatus(w http.ResponseWriter, r *http.Request) { return } - if backup.Annotations["kots.io/instance"] != "true" { + if backup.Annotations[snapshottypes.InstanceBackupAnnotation] != "true" { err := errors.Errorf("backup %s is not an instance backup", backup.ObjectMeta.Name) logger.Error(err) response.Error = err.Error() diff --git a/pkg/k8sutil/kotsadm.go b/pkg/k8sutil/kotsadm.go index a3b625bf50..606918ae11 100644 --- a/pkg/k8sutil/kotsadm.go +++ b/pkg/k8sutil/kotsadm.go @@ -21,12 +21,7 @@ const ( KotsadmIDConfigMapName = "kotsadm-id" ) -func FindKotsadmImage(namespace string) (string, error) { - clientset, err := GetClientset() - if err != nil { - return "", errors.Wrap(err, "failed to get k8s client set") - } - +func FindKotsadmImage(clientset kubernetes.Interface, namespace string) (string, error) { var containers []corev1.Container if os.Getenv("POD_OWNER_KIND") == "deployment" { kotsadmDeployment, err := clientset.AppsV1().Deployments(namespace).Get(context.TODO(), "kotsadm", metav1.GetOptions{}) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 4361ffaca1..dbe8e98e6a 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -2,15 +2,17 @@ package snapshot import ( "context" + "crypto/sha256" "encoding/json" "fmt" - "io/ioutil" "math" "os" "strconv" + "strings" "time" units "github.com/docker/go-units" + "github.com/google/uuid" "github.com/pkg/errors" embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types" @@ -35,8 +37,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/kubernetes" - kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/utils/ptr" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ) func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled bool) (*velerov1.Backup, error) { @@ -111,6 +115,8 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b return nil, errors.Errorf("application %s does not have a backup spec", a.Slug) } + // We have to render the backup spec as older versions of kots stored the unrendered spec in + // the database. renderedBackup, err := helper.RenderAppFile(a, nil, []byte(backupSpec), kotsKinds, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to render backup") @@ -191,254 +197,514 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b return backup, nil } -func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstream, isScheduled bool) (*velerov1.Backup, error) { - logger.Debug("creating instance backup") +type instanceBackupMetadata struct { + backupName string + backupReqestedAt time.Time + kotsadmNamespace string + backupStorageLocationNamespace string + apps map[string]appInstanceBackupMetadata + isScheduled bool + snapshotTTL time.Duration + ec *ecInstanceBackupMetadata +} + +type appInstanceBackupMetadata struct { + app *apptypes.App + kotsKinds *kotsutil.KotsKinds + parentSequence int64 +} + +type ecInstanceBackupMetadata struct { + installation embeddedclusterv1beta1.Installation + seaweedFSS3ServiceIP string +} + +func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstream, isScheduled bool) (string, error) { + logger.Info("Creating instance backup") cfg, err := k8sutil.GetClusterConfig() if err != nil { - return nil, errors.Wrap(err, "failed to get cluster config") + return "", errors.Wrap(err, "failed to get cluster config") } - clientset, err := kubernetes.NewForConfig(cfg) + k8sClient, err := kubernetes.NewForConfig(cfg) if err != nil { - return nil, errors.Wrap(err, "failed to create clientset") + return "", errors.Wrap(err, "failed to create clientset") } - isKurl, err := kurl.IsKurl(clientset) + ctrlClient, err := k8sutil.GetKubeClient(ctx) if err != nil { - return nil, errors.Wrap(err, "failed to check if cluster is kurl") + return "", fmt.Errorf("failed to get kubeclient: %w", err) } - kotsadmNamespace := util.PodNamespace - appsSequences := map[string]int64{} - appVersions := map[string]string{} - includedNamespaces := []string{kotsadmNamespace} - excludedNamespaces := []string{} - backupAnnotations := map[string]string{} - backupOrderedResources := map[string]string{} - backupHooks := velerov1.BackupHooks{ - Resources: []velerov1.BackupResourceHookSpec{}, - } - // non-supported fields that are intentionally left out cuz they might break full snapshots: - // - includedResources - // - excludedResources - // - labelSelector + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return "", errors.Wrap(err, "failed to create velero clientset") + } - appNamespace := kotsadmNamespace - if os.Getenv("KOTSADM_TARGET_NAMESPACE") != "" { - appNamespace = os.Getenv("KOTSADM_TARGET_NAMESPACE") + metadata, err := getInstanceBackupMetadata(ctx, k8sClient, ctrlClient, veleroClient, cluster, isScheduled) + if err != nil { + return "", errors.Wrap(err, "failed to get instance backup metadata") } - if appNamespace != kotsadmNamespace { - includedNamespaces = append(includedNamespaces, appNamespace) + + appVeleroBackup, err := getAppInstanceBackupSpec(k8sClient, metadata) + if err != nil { + return "", errors.Wrap(err, "failed to get app instance backup spec") } - if isKurl { - includedNamespaces = append(includedNamespaces, "kurl") + veleroBackup, err := getInfrastructureInstanceBackupSpec(ctx, k8sClient, metadata, appVeleroBackup != nil) + if err != nil { + return "", errors.Wrap(err, "failed to get instance backup specs") + } + + err = excludeShutdownPodsFromBackup(ctx, k8sClient, veleroBackup) + if err != nil { + logger.Errorf("Failed to exclude shutdown pods from backup: %v", err) } + if appVeleroBackup != nil { + err = excludeShutdownPodsFromBackup(ctx, k8sClient, appVeleroBackup) + if err != nil { + logger.Errorf("Failed to exclude shutdown pods from application backup: %v", err) + } + } + + logger.Infof("Creating instance backup CR %s", veleroBackup.Name) + backup, err := veleroClient.Backups(metadata.backupStorageLocationNamespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) + if err != nil { + return "", errors.Wrap(err, "failed to create velero backup") + } + + if appVeleroBackup != nil { + logger.Infof("Creating instance app backup CR %s", appVeleroBackup.Name) + _, err := veleroClient.Backups(metadata.backupStorageLocationNamespace).Create(ctx, appVeleroBackup, metav1.CreateOptions{}) + if err != nil { + return "", errors.Wrap(err, "failed to create application velero backup") + } + } + + return backup.Name, nil // TODO(improveddr): return metadata.BackupName +} + +// GetBackupName returns the name of the backup from the velero backup object label. +func GetBackupName(veleroBackup velerov1.Backup) string { + if val, ok := veleroBackup.GetLabels()[types.InstanceBackupNameLabel]; ok { + return val + } + return veleroBackup.GetName() +} + +// IsInstanceBackup returns true if the backup is an instance backup. +func IsInstanceBackup(veleroBackup velerov1.Backup) bool { + if val, ok := veleroBackup.GetAnnotations()[types.InstanceBackupAnnotation]; ok { + return val == "true" + } + return false +} + +// GetInstanceBackupType returns the type of the backup from the velero backup object annotation. +func GetInstanceBackupType(veleroBackup velerov1.Backup) string { + if val, ok := veleroBackup.GetAnnotations()[types.InstanceBackupTypeAnnotation]; ok { + return val + } + return types.InstanceBackupTypeLegacy +} + +// GetInstanceBackupCount returns the expected number of backups from the velero backup object +// annotation. +func GetInstanceBackupCount(veleroBackup velerov1.Backup) int { + if val, ok := veleroBackup.GetAnnotations()[types.InstanceBackupCountAnnotation]; ok { + num, _ := strconv.Atoi(val) + if num > 0 { + return num + } + } + return 1 +} + +// getInstanceBackupMetadata returns metadata about the instance backup for use in creating an +// instance backup. +func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interface, ctrlClient ctrlclient.Client, veleroClient veleroclientv1.VeleroV1Interface, cluster *downstreamtypes.Downstream, isScheduled bool) (instanceBackupMetadata, error) { + metadata := instanceBackupMetadata{ + backupName: getBackupNameFromPrefix("instance"), + backupReqestedAt: time.Now().UTC(), + kotsadmNamespace: util.PodNamespace, + apps: make(map[string]appInstanceBackupMetadata, 0), + isScheduled: isScheduled, + } + + if cluster.SnapshotTTL != "" { + snapshotTTL, err := time.ParseDuration(cluster.SnapshotTTL) + if err != nil { + return metadata, errors.Wrap(err, "failed to parse snapshot ttl") + } + metadata.snapshotTTL = snapshotTTL + } + + kotsadmVeleroBackendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, k8sClient, veleroClient, metadata.kotsadmNamespace) + if err != nil { + return metadata, errors.Wrap(err, "failed to find backupstoragelocations") + } else if kotsadmVeleroBackendStorageLocation == nil { + return metadata, errors.New("no backup store location found") + } + metadata.backupStorageLocationNamespace = kotsadmVeleroBackendStorageLocation.Namespace + apps, err := store.GetStore().ListInstalledApps() if err != nil { - return nil, errors.Wrap(err, "failed to list installed apps") + return metadata, errors.Wrap(err, "failed to list installed apps") } - for _, a := range apps { - downstreams, err := store.GetStore().ListDownstreamsForApp(a.ID) + for _, app := range apps { + downstreams, err := store.GetStore().ListDownstreamsForApp(app.ID) if err != nil { - return nil, errors.Wrapf(err, "failed to list downstreams for app %s", a.Slug) + return metadata, errors.Wrapf(err, "failed to list downstreams for app %s", app.Slug) } if len(downstreams) == 0 { - logger.Error(errors.Wrapf(err, "no downstreams found for app %s", a.Slug)) + logger.Errorf("No downstreams found for app %s", app.Slug) continue } - parentSequence, err := store.GetStore().GetCurrentParentSequence(a.ID, downstreams[0].ClusterID) + parentSequence, err := store.GetStore().GetCurrentParentSequence(app.ID, downstreams[0].ClusterID) if err != nil { - return nil, errors.Wrapf(err, "failed to get current downstream parent sequence for app %s", a.Slug) + return metadata, errors.Wrapf(err, "failed to get current downstream parent sequence for app %s", app.Slug) } if parentSequence == -1 { // no version is deployed for this app yet continue } - archiveDir, err := ioutil.TempDir("", "kotsadm") + archiveDir, err := os.MkdirTemp("", "kotsadm") if err != nil { - return nil, errors.Wrapf(err, "failed to create temp dir for app %s", a.Slug) + return metadata, errors.Wrapf(err, "failed to create temp dir for app %s", app.Slug) } - defer os.RemoveAll(archiveDir) + defer func() { + _ = os.RemoveAll(archiveDir) + }() - err = store.GetStore().GetAppVersionArchive(a.ID, parentSequence, archiveDir) + err = store.GetStore().GetAppVersionArchive(app.ID, parentSequence, archiveDir) if err != nil { - return nil, errors.Wrapf(err, "failed to get app version archive for app %s", a.Slug) + return metadata, errors.Wrapf(err, "failed to get app version archive for app %s", app.Slug) } kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { - return nil, errors.Wrap(err, "failed to load kots kinds from path") + return metadata, errors.Wrap(err, "failed to load kots kinds from path") } - backupSpec, err := kotsKinds.Marshal("velero.io", "v1", "Backup") - if err != nil { - return nil, errors.Wrap(err, "failed to get backup spec from kotskinds") + metadata.apps[app.Slug] = appInstanceBackupMetadata{ + app: app, + kotsKinds: kotsKinds, + parentSequence: parentSequence, } - if backupSpec == "" { - continue + // if there's only one app, use the slug as the backup name + if len(apps) == 1 && len(metadata.apps) == 1 { + metadata.backupName = getBackupNameFromPrefix(app.Slug) } - appsSequences[a.Slug] = parentSequence - appVersions[a.Slug] = kotsKinds.Installation.Spec.VersionLabel + // optimization as we no longer need the archive dir + _ = os.RemoveAll(archiveDir) + } - renderedBackup, err := helper.RenderAppFile(a, nil, []byte(backupSpec), kotsKinds, kotsadmNamespace) - if err != nil { - return nil, errors.Wrap(err, "failed to render backup") - } - veleroBackup, err := kotsutil.LoadBackupFromContents(renderedBackup) - if err != nil { - return nil, errors.Wrap(err, "failed to load backup from contents") - } + metadata.ec, err = getECInstanceBackupMetadata(ctx, ctrlClient) + if err != nil { + return metadata, errors.Wrap(err, "failed to get embedded cluster metadata") + } - // ** merge app backup info ** // - // included namespaces - includedNamespaces = append(includedNamespaces, veleroBackup.Spec.IncludedNamespaces...) - includedNamespaces = append(includedNamespaces, kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) + return metadata, nil +} - // excluded namespaces - excludedNamespaces = append(excludedNamespaces, veleroBackup.Spec.ExcludedNamespaces...) +// getECInstanceBackupMetadata returns metadata about the embedded cluster for use in creating an +// instance backup. +func getECInstanceBackupMetadata(ctx context.Context, ctrlClient ctrlclient.Client) (*ecInstanceBackupMetadata, error) { + if !util.IsEmbeddedCluster() { + return nil, nil + } - // annotations - for k, v := range veleroBackup.Annotations { - backupAnnotations[k] = v - } + installation, err := embeddedcluster.GetCurrentInstallation(ctx, ctrlClient) + if err != nil { + return nil, fmt.Errorf("failed to get current installation: %w", err) + } - // ordered resources - for k, v := range veleroBackup.Spec.OrderedResources { - backupOrderedResources[k] = v - } + seaweedFSS3ServiceIP, err := embeddedcluster.GetSeaweedFSS3ServiceIP(ctx, ctrlClient) + if err != nil { + return nil, fmt.Errorf("failed to get seaweedfs s3 service ip: %w", err) + } - // backup hooks - backupHooks.Resources = append(backupHooks.Resources, veleroBackup.Spec.Hooks.Resources...) + return &ecInstanceBackupMetadata{ + installation: *installation, + seaweedFSS3ServiceIP: seaweedFSS3ServiceIP, + }, nil +} + +// getInfrastructureInstanceBackupSpec returns the velero backup spec for the instance backup. This +// is either the kotsadm backup or the legacy backup if this is not using improved DR. +func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata, hasAppBackup bool) (*velerov1.Backup, error) { + // veleroBackup is the kotsadm backup or legacy backup if usesImprovedDR is false + veleroBackup := &velerov1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{}, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{metadata.kotsadmNamespace}, + ExcludedNamespaces: []string{}, + IncludeClusterResources: ptr.To(true), + OrLabelSelectors: instanceBackupLabelSelectors(metadata.ec != nil), + OrderedResources: map[string]string{}, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{}, + }, + }, } - veleroClient, err := veleroclientv1.NewForConfig(cfg) - if err != nil { - return nil, errors.Wrap(err, "failed to create velero clientset") + if util.AppNamespace() != metadata.kotsadmNamespace { + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, util.AppNamespace()) } - kotsadmVeleroBackendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) + isKurl, err := kurl.IsKurl(k8sClient) if err != nil { - return nil, errors.Wrap(err, "failed to find backupstoragelocations") + return nil, errors.Wrap(err, "failed to check if cluster is kurl") } - - if kotsadmVeleroBackendStorageLocation == nil { - return nil, errors.New("no backup store location found") + if isKurl { + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, "kurl") } - isKotsadmClusterScoped := k8sutil.IsKotsadmClusterScoped(ctx, clientset, kotsadmNamespace) + isKotsadmClusterScoped := k8sutil.IsKotsadmClusterScoped(ctx, k8sClient, metadata.kotsadmNamespace) if !isKotsadmClusterScoped { // in minimal rbac, a kotsadm role and rolebinding will exist in the velero namespace to give kotsadm access to velero. // we backup and restore those so that restoring to a new cluster won't require that the user provide those permissions again. - includedNamespaces = append(includedNamespaces, kotsadmVeleroBackendStorageLocation.Namespace) + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, metadata.backupStorageLocationNamespace) + } + + for _, appMeta := range metadata.apps { + // Don't merge the backup spec if we are using the new improved DR. + if !hasAppBackup { + err := mergeAppBackupSpec(veleroBackup, appMeta, metadata.kotsadmNamespace, metadata.ec != nil) + if err != nil { + return nil, errors.Wrap(err, "failed to merge app backup spec") + } + } } - kotsadmImage, err := k8sutil.FindKotsadmImage(kotsadmNamespace) + veleroBackup.Annotations, err = appendCommonAnnotations(k8sClient, veleroBackup.Annotations, metadata, hasAppBackup) if err != nil { - return nil, errors.Wrap(err, "failed to find kotsadm image") + return nil, errors.Wrap(err, "failed to add annotations to backup") + } + if hasAppBackup { + // Only add improved disaster recovery annotations and labels if we have an app backup + if veleroBackup.Labels == nil { + veleroBackup.Labels = map[string]string{} + } + veleroBackup.Labels[types.InstanceBackupNameLabel] = metadata.backupName + veleroBackup.Annotations[types.InstanceBackupTypeAnnotation] = types.InstanceBackupTypeInfra + veleroBackup.Annotations[types.InstanceBackupCountAnnotation] = strconv.Itoa(2) } - snapshotTrigger := "manual" - if isScheduled { - snapshotTrigger = "schedule" + if metadata.ec != nil { + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, ecIncludedNamespaces(metadata.ec.installation)...) } - // marshal apps sequences map - b, err := json.Marshal(appsSequences) - if err != nil { - return nil, errors.Wrap(err, "failed to marshal apps sequences") + if metadata.snapshotTTL > 0 { + veleroBackup.Spec.TTL = metav1.Duration{ + Duration: metadata.snapshotTTL, + } } - marshalledAppsSequences := string(b) - // marshal apps versions map - b, err = json.Marshal(appVersions) - if err != nil { - return nil, errors.Wrap(err, "failed to marshal apps versions") + veleroBackup.Spec.IncludedNamespaces = prepareIncludedNamespaces(veleroBackup.Spec.IncludedNamespaces) + + return veleroBackup, nil +} + +var EnableImprovedDR = false + +// getAppInstanceBackup returns a backup spec only if this is Embedded Cluster and the vendor has +// defined both a backup and restore custom resource (improved DR). +func getAppInstanceBackupSpec(k8sClient kubernetes.Interface, metadata instanceBackupMetadata) (*velerov1.Backup, error) { + // TODO(improveddr): remove this once we have fully implemented the improved DR + if !EnableImprovedDR { + return nil, nil } - marshalledAppVersions := string(b) - // add kots annotations - backupAnnotations["kots.io/snapshot-trigger"] = snapshotTrigger - backupAnnotations["kots.io/snapshot-requested"] = time.Now().UTC().Format(time.RFC3339) - backupAnnotations["kots.io/instance"] = "true" - backupAnnotations["kots.io/kotsadm-image"] = kotsadmImage - backupAnnotations["kots.io/kotsadm-deploy-namespace"] = kotsadmNamespace - backupAnnotations["kots.io/apps-sequences"] = marshalledAppsSequences - backupAnnotations["kots.io/apps-versions"] = marshalledAppVersions - backupAnnotations["kots.io/is-airgap"] = strconv.FormatBool(kotsadm.IsAirgap()) - - if util.IsEmbeddedCluster() { - kbClient, err := k8sutil.GetKubeClient(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get kubeclient: %w", err) + if metadata.ec == nil { + return nil, nil + } + + var appVeleroBackup *velerov1.Backup + + for _, appMeta := range metadata.apps { + // if there is both a backup and a restore spec this is using the new improved DR + if appMeta.kotsKinds.Backup == nil || appMeta.kotsKinds.Restore == nil { + continue } - installation, err := embeddedcluster.GetCurrentInstallation(ctx, kbClient) - if err != nil { - return nil, fmt.Errorf("failed to get current installation: %w", err) + + if len(metadata.apps) > 1 { + return nil, errors.New("cannot create backup for Embedded Cluster with multiple apps") } - ecAnnotations, err := ecBackupAnnotations(ctx, kbClient, installation) - if err != nil { - return nil, fmt.Errorf("failed to get embedded cluster backup annotations: %w", err) + + if appMeta.kotsKinds.Backup == nil { + return nil, errors.New("backup spec is empty, this is unexpected") } - for k, v := range ecAnnotations { - backupAnnotations[k] = v + + appVeleroBackup = appMeta.kotsKinds.Backup.DeepCopy() + + appVeleroBackup.Name = "" + appVeleroBackup.GenerateName = "application-" + + break + } + + if appVeleroBackup == nil { + return nil, nil + } + + var err error + appVeleroBackup.Annotations, err = appendCommonAnnotations(k8sClient, appVeleroBackup.Annotations, metadata, true) + if err != nil { + return nil, errors.Wrap(err, "failed to add annotations to application backup") + } + // Add improved disaster recovery annotations and labels + if appVeleroBackup.Labels == nil { + appVeleroBackup.Labels = map[string]string{} + } + appVeleroBackup.Labels[types.InstanceBackupNameLabel] = metadata.backupName + appVeleroBackup.Annotations[types.InstanceBackupTypeAnnotation] = types.InstanceBackupTypeApp + appVeleroBackup.Annotations[types.InstanceBackupCountAnnotation] = strconv.Itoa(2) + + appVeleroBackup.Spec.StorageLocation = "default" + + if metadata.snapshotTTL > 0 { + appVeleroBackup.Spec.TTL = metav1.Duration{ + Duration: metadata.snapshotTTL, } - includedNamespaces = append(includedNamespaces, ecIncludedNamespaces(installation)...) } - includeClusterResources := true - veleroBackup := &velerov1.Backup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "", - GenerateName: "instance-", - Namespace: kotsadmVeleroBackendStorageLocation.Namespace, - Annotations: backupAnnotations, - }, - Spec: velerov1.BackupSpec{ - StorageLocation: "default", - IncludedNamespaces: prepareIncludedNamespaces(includedNamespaces), - ExcludedNamespaces: excludedNamespaces, - IncludeClusterResources: &includeClusterResources, - OrLabelSelectors: instanceBackupLabelSelectors(util.IsEmbeddedCluster()), - OrderedResources: backupOrderedResources, - Hooks: backupHooks, - }, + return appVeleroBackup, nil +} + +// mergeAppBackupSpec merges the app backup spec into the velero backup spec when improved DR is +// disabled. Unsupported fields that are intentionally left out because they might break full +// snapshots: +// - includedResources +// - excludedResources +// - labelSelector +func mergeAppBackupSpec(backup *velerov1.Backup, appMeta appInstanceBackupMetadata, kotsadmNamespace string, isEC bool) error { + backupSpec, err := appMeta.kotsKinds.Marshal("velero.io", "v1", "Backup") + if err != nil { + return errors.Wrap(err, "failed to get backup spec from kotskinds") } - embeddedRegistryHost, _, _ := kotsutil.GetEmbeddedRegistryCreds(clientset) - if embeddedRegistryHost != "" { - veleroBackup.ObjectMeta.Annotations["kots.io/embedded-registry"] = embeddedRegistryHost + if backupSpec == "" { + // If this is Embedded Cluster, backups are always enabled and we must include the + // namespace. + if isEC { + backup.Spec.IncludedNamespaces = append(backup.Spec.IncludedNamespaces, appMeta.kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) + } + return nil } - if cluster.SnapshotTTL != "" { - ttlDuration, err := time.ParseDuration(cluster.SnapshotTTL) - if err != nil { - return nil, errors.Wrap(err, "failed to parse cluster snapshot ttl value as duration") + // We have to render the backup spec as older versions of kots stored the unrendered spec in + // the database. + renderedBackup, err := helper.RenderAppFile(appMeta.app, nil, []byte(backupSpec), appMeta.kotsKinds, kotsadmNamespace) + if err != nil { + return errors.Wrap(err, "failed to render backup") + } + kotskindsBackup, err := kotsutil.LoadBackupFromContents(renderedBackup) + if err != nil { + return errors.Wrap(err, "failed to load backup from contents") + } + + // included namespaces + backup.Spec.IncludedNamespaces = append(backup.Spec.IncludedNamespaces, appMeta.kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) + backup.Spec.IncludedNamespaces = append(backup.Spec.IncludedNamespaces, kotskindsBackup.Spec.IncludedNamespaces...) + + // excluded namespaces + backup.Spec.ExcludedNamespaces = append(backup.Spec.ExcludedNamespaces, kotskindsBackup.Spec.ExcludedNamespaces...) + + // annotations + if len(kotskindsBackup.ObjectMeta.Annotations) > 0 { + if backup.Annotations == nil { + backup.Annotations = map[string]string{} } - veleroBackup.Spec.TTL = metav1.Duration{ - Duration: ttlDuration, + for k, v := range kotskindsBackup.ObjectMeta.Annotations { + backup.Annotations[k] = v } } - err = excludeShutdownPodsFromBackup(ctx, clientset, veleroBackup) + // ordered resources + if len(kotskindsBackup.Spec.OrderedResources) > 0 { + if backup.Spec.OrderedResources == nil { + backup.Spec.OrderedResources = map[string]string{} + } + for k, v := range kotskindsBackup.Spec.OrderedResources { + backup.Spec.OrderedResources[k] = v + } + } + + // backup hooks + backup.Spec.Hooks.Resources = append(backup.Spec.Hooks.Resources, kotskindsBackup.Spec.Hooks.Resources...) + + return nil +} + +// appendCommonAnnotations appends common annotations to the backup annotations +func appendCommonAnnotations(k8sClient kubernetes.Interface, annotations map[string]string, metadata instanceBackupMetadata, hasAppBackup bool) (map[string]string, error) { + kotsadmImage, err := k8sutil.FindKotsadmImage(k8sClient, metadata.kotsadmNamespace) if err != nil { - logger.Error(errors.Wrap(err, "failed to exclude shutdown pods from backup")) + return nil, errors.Wrap(err, "failed to find kotsadm image") } - backup, err := veleroClient.Backups(kotsadmVeleroBackendStorageLocation.Namespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) + snapshotTrigger := "manual" + if metadata.isScheduled { + snapshotTrigger = "schedule" + } + + appSequences := map[string]int64{} + appVersions := map[string]string{} + + for slug, appMeta := range metadata.apps { + appSequences[slug] = appMeta.parentSequence + appVersions[slug] = appMeta.kotsKinds.Installation.Spec.VersionLabel + } + + // marshal apps sequences map + b, err := json.Marshal(appSequences) if err != nil { - return nil, errors.Wrap(err, "failed to create velero backup") + return nil, errors.Wrap(err, "failed to marshal app sequences") } + marshalledAppSequences := string(b) - return backup, nil + // marshal apps versions map + b, err = json.Marshal(appVersions) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal app versions") + } + marshalledAppVersions := string(b) + + if annotations == nil { + annotations = make(map[string]string, 0) + } + annotations["kots.io/snapshot-trigger"] = snapshotTrigger + annotations["kots.io/snapshot-requested"] = metadata.backupReqestedAt.Format(time.RFC3339) + annotations[types.InstanceBackupAnnotation] = "true" + annotations["kots.io/kotsadm-image"] = kotsadmImage + annotations["kots.io/kotsadm-deploy-namespace"] = metadata.kotsadmNamespace + annotations["kots.io/apps-sequences"] = marshalledAppSequences + annotations["kots.io/apps-versions"] = marshalledAppVersions + annotations["kots.io/is-airgap"] = strconv.FormatBool(kotsadm.IsAirgap()) + embeddedRegistryHost, _, _ := kotsutil.GetEmbeddedRegistryCreds(k8sClient) + if embeddedRegistryHost != "" { + annotations["kots.io/embedded-registry"] = embeddedRegistryHost + } + + if metadata.ec != nil { + annotations = appendECAnnotations(annotations, *metadata.ec) + } + + return annotations, nil } func ListBackupsForApp(ctx context.Context, kotsadmNamespace string, appID string) ([]*types.Backup, error) { @@ -608,7 +874,7 @@ func ListInstanceBackups(ctx context.Context, kotsadmNamespace string) ([]*types for _, veleroBackup := range veleroBackups.Items { // TODO: Enforce version? - if veleroBackup.Annotations["kots.io/instance"] != "true" { + if !IsInstanceBackup(veleroBackup) { continue } @@ -751,7 +1017,7 @@ func getSnapshotVolumeSummary(ctx context.Context, veleroBackup *velerov1.Backup return &volumeSummary, nil } -func GetBackup(ctx context.Context, kotsadmNamespace string, snapshotName string) (*velerov1.Backup, error) { +func GetBackup(ctx context.Context, kotsadmNamespace string, backupID string) (*velerov1.Backup, error) { cfg, err := k8sutil.GetClusterConfig() if err != nil { return nil, errors.Wrap(err, "failed to get cluster config") @@ -777,8 +1043,7 @@ func GetBackup(ctx context.Context, kotsadmNamespace string, snapshotName string veleroNamespace := bsl.Namespace - // get the backup - backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, snapshotName, metav1.GetOptions{}) + backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, backupID, metav1.GetOptions{}) if err != nil { return nil, errors.Wrap(err, "failed to get backup") } @@ -786,7 +1051,19 @@ func GetBackup(ctx context.Context, kotsadmNamespace string, snapshotName string return backup, nil } -func DeleteBackup(ctx context.Context, kotsadmNamespace string, snapshotName string) error { +func getBackupNameFromPrefix(appSlug string) string { + randStr := fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%d-%s", + time.Now().UnixNano(), + strings.Replace(uuid.New().String(), "-", "", 4), + ))))[:8] + backupName := appSlug + if len(backupName)+9 > validation.DNS1035LabelMaxLength { + backupName = backupName[:validation.DNS1035LabelMaxLength-9] + } + return fmt.Sprintf("%s-%s", backupName, randStr) +} + +func DeleteBackup(ctx context.Context, kotsadmNamespace string, backupID string) error { cfg, err := k8sutil.GetClusterConfig() if err != nil { return errors.Wrap(err, "failed to get cluster config") @@ -813,11 +1090,11 @@ func DeleteBackup(ctx context.Context, kotsadmNamespace string, snapshotName str veleroNamespace := bsl.Namespace veleroDeleteBackupRequest := &velerov1.DeleteBackupRequest{ ObjectMeta: metav1.ObjectMeta{ - Name: snapshotName, + Name: backupID, Namespace: veleroNamespace, }, Spec: velerov1.DeleteBackupRequestSpec{ - BackupName: snapshotName, + BackupName: backupID, }, } @@ -859,7 +1136,7 @@ func HasUnfinishedInstanceBackup(ctx context.Context, kotsadmNamespace string) ( return false, nil } -func GetBackupDetail(ctx context.Context, kotsadmNamespace string, backupName string) (*types.BackupDetail, error) { +func GetBackupDetail(ctx context.Context, kotsadmNamespace string, backupID string) (*types.BackupDetail, error) { cfg, err := k8sutil.GetClusterConfig() if err != nil { return nil, errors.Wrap(err, "failed to get cluster config") @@ -882,13 +1159,13 @@ func GetBackupDetail(ctx context.Context, kotsadmNamespace string, backupName st veleroNamespace := backendStorageLocation.Namespace - backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, backupName, metav1.GetOptions{}) + backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, backupID, metav1.GetOptions{}) if err != nil { return nil, errors.Wrap(err, "failed to get backup") } backupVolumes, err := veleroClient.PodVolumeBackups(veleroNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: fmt.Sprintf("velero.io/backup-name=%s", velerolabel.GetValidName(backupName)), + LabelSelector: fmt.Sprintf("velero.io/backup-name=%s", velerolabel.GetValidName(backupID)), }) if err != nil { return nil, errors.Wrap(err, "failed to list volumes") @@ -908,7 +1185,7 @@ func GetBackupDetail(ctx context.Context, kotsadmNamespace string, backupName st result.VolumeSizeHuman = units.HumanSize(float64(totalBytesDone)) // TODO: should this be TotalBytes rather than BytesDone? if backup.Status.Phase == velerov1.BackupPhaseCompleted || backup.Status.Phase == velerov1.BackupPhasePartiallyFailed || backup.Status.Phase == velerov1.BackupPhaseFailed { - errs, warnings, execs, err := downloadBackupLogs(ctx, veleroNamespace, backupName) + errs, warnings, execs, err := downloadBackupLogs(ctx, veleroNamespace, backupID) result.Errors = errs result.Warnings = warnings result.Hooks = execs @@ -956,8 +1233,8 @@ func listBackupVolumes(backupVolumes []velerov1.PodVolumeBackup) []types.Snapsho return volumes } -func downloadBackupLogs(ctx context.Context, veleroNamespace, backupName string) ([]types.SnapshotError, []types.SnapshotError, []*types.SnapshotHook, error) { - gzipReader, err := DownloadRequest(ctx, veleroNamespace, velerov1.DownloadTargetKindBackupLog, backupName) +func downloadBackupLogs(ctx context.Context, veleroNamespace, backupID string) ([]types.SnapshotError, []types.SnapshotError, []*types.SnapshotHook, error) { + gzipReader, err := DownloadRequest(ctx, veleroNamespace, velerov1.DownloadTargetKindBackupLog, backupID) if err != nil { return nil, nil, nil, errors.Wrap(err, "failed to download backup log") } @@ -980,36 +1257,34 @@ func mergeLabelSelector(kots metav1.LabelSelector, app metav1.LabelSelector) met return kots } -// ecBackupAnnotations returns the annotations that should be added to an embedded cluster backup -func ecBackupAnnotations(ctx context.Context, kbClient kbclient.Client, in *embeddedclusterv1beta1.Installation) (map[string]string, error) { - annotations := map[string]string{} - - seaweedFSS3ServiceIP, err := embeddedcluster.GetSeaweedFSS3ServiceIP(ctx, kbClient) - if err != nil { - return nil, fmt.Errorf("failed to get seaweedfs s3 service ip: %w", err) +// appendECAnnotations appends annotations that should be added to an embedded cluster backup +func appendECAnnotations(annotations map[string]string, ecMeta ecInstanceBackupMetadata) map[string]string { + if annotations == nil { + annotations = make(map[string]string, 0) } - if seaweedFSS3ServiceIP != "" { - annotations["kots.io/embedded-cluster-seaweedfs-s3-ip"] = seaweedFSS3ServiceIP + + if ecMeta.seaweedFSS3ServiceIP != "" { + annotations["kots.io/embedded-cluster-seaweedfs-s3-ip"] = ecMeta.seaweedFSS3ServiceIP } annotations["kots.io/embedded-cluster"] = "true" annotations["kots.io/embedded-cluster-id"] = util.EmbeddedClusterID() annotations["kots.io/embedded-cluster-version"] = util.EmbeddedClusterVersion() - annotations["kots.io/embedded-cluster-is-ha"] = strconv.FormatBool(in.Spec.HighAvailability) + annotations["kots.io/embedded-cluster-is-ha"] = strconv.FormatBool(ecMeta.installation.Spec.HighAvailability) - if in.Spec.Network != nil { - annotations["kots.io/embedded-cluster-pod-cidr"] = in.Spec.Network.PodCIDR - annotations["kots.io/embedded-cluster-service-cidr"] = in.Spec.Network.ServiceCIDR + if ecMeta.installation.Spec.Network != nil { + annotations["kots.io/embedded-cluster-pod-cidr"] = ecMeta.installation.Spec.Network.PodCIDR + annotations["kots.io/embedded-cluster-service-cidr"] = ecMeta.installation.Spec.Network.ServiceCIDR } - if in.Spec.RuntimeConfig != nil { - rcAnnotations := ecRuntimeConfigToBackupAnnotations(in.Spec.RuntimeConfig) + if ecMeta.installation.Spec.RuntimeConfig != nil { + rcAnnotations := ecRuntimeConfigToBackupAnnotations(ecMeta.installation.Spec.RuntimeConfig) for k, v := range rcAnnotations { annotations[k] = v } } - return annotations, nil + return annotations } func ecRuntimeConfigToBackupAnnotations(runtimeConfig *embeddedclusterv1beta1.RuntimeConfigSpec) map[string]string { @@ -1029,7 +1304,7 @@ func ecRuntimeConfigToBackupAnnotations(runtimeConfig *embeddedclusterv1beta1.Ru } // ecIncludedNamespaces returns the namespaces that should be included in an embedded cluster backup -func ecIncludedNamespaces(in *embeddedclusterv1beta1.Installation) []string { +func ecIncludedNamespaces(in embeddedclusterv1beta1.Installation) []string { includedNamespaces := []string{"embedded-cluster", "kube-system", "openebs"} if in.Spec.AirGap { includedNamespaces = append(includedNamespaces, "registry") @@ -1084,17 +1359,24 @@ func excludeShutdownPodsFromBackup(ctx context.Context, clientset kubernetes.Int labelSets = append(labelSets, orLabelSet...) } - for _, labelSet := range labelSets { + for _, namespace := range veleroBackup.Spec.IncludedNamespaces { + if namespace == "*" { + namespace = "" // specifying an empty ("") namespace in client-go retrieves resources from all namespaces + } + podListOption := metav1.ListOptions{ - LabelSelector: labelSet, FieldSelector: fields.SelectorFromSet(selectorMap).String(), } - for _, namespace := range veleroBackup.Spec.IncludedNamespaces { - if namespace == "*" { - namespace = "" // specifying an empty ("") namespace in client-go retrieves resources from all namespaces - } + if len(labelSets) > 0 { + for _, labelSet := range labelSets { + podListOption.LabelSelector = labelSet + if err := excludeShutdownPodsFromBackupInNamespace(ctx, clientset, namespace, podListOption); err != nil { + return errors.Wrap(err, "failed to exclude shutdown pods from backup") + } + } + } else { if err := excludeShutdownPodsFromBackupInNamespace(ctx, clientset, namespace, podListOption); err != nil { return errors.Wrap(err, "failed to exclude shutdown pods from backup") } diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 0e3af9aa5f..f289067aa7 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -2,25 +2,41 @@ package snapshot import ( "context" + "os" + "path/filepath" "testing" + "time" + gomock "github.com/golang/mock/gomock" embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" - "github.com/replicatedhq/kots/pkg/embeddedcluster" + downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types" + apptypes "github.com/replicatedhq/kots/pkg/app/types" kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" + "github.com/replicatedhq/kots/pkg/kotsutil" + registrytypes "github.com/replicatedhq/kots/pkg/registry/types" + "github.com/replicatedhq/kots/pkg/store" + mock_store "github.com/replicatedhq/kots/pkg/store/mock" + "github.com/replicatedhq/kots/pkg/util" + kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerofake "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake" + veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" coretest "k8s.io/client-go/testing" - kbclient "sigs.k8s.io/controller-runtime/pkg/client" - fakekbclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + ctrlclientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestPrepareIncludedNamespaces(t *testing.T) { @@ -178,6 +194,17 @@ func mockK8sClientWithShutdownPods() kubernetes.Interface { Reason: "Shutdown", }, }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shutdown-no-label", + Namespace: "test", + Labels: map[string]string{}, + }, + Status: corev1.PodStatus{ + Phase: "Failed", + Reason: "Shutdown", + }, + }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup-shutdown", @@ -205,6 +232,16 @@ func mockK8sClientWithShutdownPods() kubernetes.Interface { Phase: "Running", }, }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-running-no-label", + Namespace: "test", + Labels: map[string]string{}, + }, + Status: corev1.PodStatus{ + Phase: "Running", + }, + }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup-running", @@ -276,7 +313,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { { name: "expect error when k8s client list pod returns error", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockGetPodsInANamespaceErrorClient(), namespace: "test", failedPodListOptions: kotsadmPodListOption, @@ -286,7 +323,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { { name: "expect error when k8s client update shutdown pod returns error", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockUpdateShutdownPodErrorClient(), namespace: "test", failedPodListOptions: kotsadmPodListOption, @@ -296,7 +333,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { { name: "expect no error when no shutdown pods are found", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockGetRunningPodsClient(), namespace: "test", failedPodListOptions: kotsadmPodListOption, @@ -307,7 +344,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for kotsadm backup label", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), namespace: "test", failedPodListOptions: kotsadmPodListOption, @@ -315,10 +352,23 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { wantErr: false, wantNumOfPodsWithExcludeAnnotation: 1, }, + { + name: "expect no error when shutdown pods are found and updated for no label selector", + args: args{ + ctx: context.Background(), + clientset: mockK8sClientWithShutdownPods(), + namespace: "test", + failedPodListOptions: metav1.ListOptions{ + FieldSelector: fields.SelectorFromSet(selectorMap).String(), + }, + }, + wantErr: false, + wantNumOfPodsWithExcludeAnnotation: 2, + }, { name: "expect no error when shutdown pods are found and updated for app slug label", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), namespace: "test-2", failedPodListOptions: appSlugPodListOption, @@ -329,7 +379,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for app slug label with all namespaces", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), namespace: "", failedPodListOptions: appSlugPodListOption, @@ -340,7 +390,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for kotsadm backup label with all namespaces", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), namespace: "", failedPodListOptions: kotsadmPodListOption, @@ -358,7 +408,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { foundNumofPodsWithExcludeAnnotation := 0 if !tt.wantErr { // get pods in test namespace and check if they have the velero exclude annotation for Shutdown pods - pods, err := tt.args.clientset.CoreV1().Pods(tt.args.namespace).List(context.TODO(), tt.args.failedPodListOptions) + pods, err := tt.args.clientset.CoreV1().Pods(tt.args.namespace).List(context.Background(), tt.args.failedPodListOptions) if err != nil { t.Errorf("excludeShutdownPodsFromBackupInNamespace() error = %v, wantErr %v", err, tt.wantErr) } @@ -400,7 +450,7 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { { name: "expect no error when namespaces are empty", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), veleroBackup: &velerov1.Backup{ Spec: velerov1.BackupSpec{ @@ -414,7 +464,7 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { { name: "expect no error when pods are running", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockGetRunningPodsClient(), veleroBackup: &velerov1.Backup{ Spec: velerov1.BackupSpec{ @@ -428,7 +478,7 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { { name: "expect error when k8s client list pods returns error", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockGetPodsInANamespaceErrorClient(), veleroBackup: &velerov1.Backup{ Spec: velerov1.BackupSpec{ @@ -442,7 +492,7 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for app slug label", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), veleroBackup: &velerov1.Backup{ Spec: velerov1.BackupSpec{ @@ -456,7 +506,7 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for kotsadm backup label and namespace is *", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), veleroBackup: &velerov1.Backup{ Spec: velerov1.BackupSpec{ @@ -470,7 +520,7 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for app slug match expression", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), veleroBackup: &velerov1.Backup{ Spec: velerov1.BackupSpec{ @@ -481,6 +531,19 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { }, wantErr: false, }, + { + name: "expect no error when shutdown pods are found and updated for app slug label and no label selector", + args: args{ + ctx: context.Background(), + clientset: mockK8sClientWithShutdownPods(), + veleroBackup: &velerov1.Backup{ + Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"test"}, + }, + }, + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -606,11 +669,11 @@ func Test_excludeShutdownPodsFromBackup_check(t *testing.T) { req := require.New(t) mockClient := fake.NewSimpleClientset(tt.resources...) - err := excludeShutdownPodsFromBackup(context.TODO(), mockClient, tt.args.veleroBackup) + err := excludeShutdownPodsFromBackup(context.Background(), mockClient, tt.args.veleroBackup) req.NoError(err) // count the number of pods with exclude annotation - testPods, err := mockClient.CoreV1().Pods("test").List(context.TODO(), metav1.ListOptions{}) + testPods, err := mockClient.CoreV1().Pods("test").List(context.Background(), metav1.ListOptions{}) req.NoError(err) foundExcluded := []string{} @@ -682,27 +745,32 @@ func Test_instanceBackupLabelSelectors(t *testing.T) { } } -func Test_ecBackupAnnotations(t *testing.T) { +func Test_appendECAnnotations(t *testing.T) { scheme := runtime.NewScheme() corev1.AddToScheme(scheme) embeddedclusterv1beta1.AddToScheme(scheme) tests := []struct { - name string - kbClient kbclient.Client - in *embeddedclusterv1beta1.Installation - env map[string]string - want map[string]string + name string + prev map[string]string + in embeddedclusterv1beta1.Installation + seaweedFSS3ServiceIP string + env map[string]string + want map[string]string }{ { - name: "basic", - kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).Build(), - in: &embeddedclusterv1beta1.Installation{}, + name: "basic", + prev: map[string]string{ + "prev-key": "prev-value", + }, + in: embeddedclusterv1beta1.Installation{}, + seaweedFSS3ServiceIP: "", env: map[string]string{ "EMBEDDED_CLUSTER_ID": "embedded-cluster-id", "EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version", }, want: map[string]string{ + "prev-key": "prev-value", "kots.io/embedded-cluster": "true", "kots.io/embedded-cluster-id": "embedded-cluster-id", "kots.io/embedded-cluster-version": "embedded-cluster-version", @@ -710,13 +778,13 @@ func Test_ecBackupAnnotations(t *testing.T) { }, }, { - name: "online ha", - kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).Build(), - in: &embeddedclusterv1beta1.Installation{ + name: "online ha", + in: embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ HighAvailability: true, }, }, + seaweedFSS3ServiceIP: "", env: map[string]string{ "EMBEDDED_CLUSTER_ID": "embedded-cluster-id", "EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version", @@ -730,23 +798,13 @@ func Test_ecBackupAnnotations(t *testing.T) { }, { name: "airgap ha", - kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).WithObjects( - &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: embeddedcluster.SeaweedfsS3SVCName, - Namespace: embeddedcluster.SeaweedfsNamespace, - }, - Spec: corev1.ServiceSpec{ - ClusterIP: "10.96.0.10", - }, - }, - ).Build(), - in: &embeddedclusterv1beta1.Installation{ + in: embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ HighAvailability: true, AirGap: true, }, }, + seaweedFSS3ServiceIP: "10.96.0.10", env: map[string]string{ "EMBEDDED_CLUSTER_ID": "embedded-cluster-id", "EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version", @@ -760,9 +818,8 @@ func Test_ecBackupAnnotations(t *testing.T) { }, }, { - name: "with pod and service cidrs", - kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).Build(), - in: &embeddedclusterv1beta1.Installation{ + name: "with pod and service cidrs", + in: embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ Network: &embeddedclusterv1beta1.NetworkSpec{ PodCIDR: "10.128.0.0/20", @@ -770,6 +827,7 @@ func Test_ecBackupAnnotations(t *testing.T) { }, }, }, + seaweedFSS3ServiceIP: "", env: map[string]string{ "EMBEDDED_CLUSTER_ID": "embedded-cluster-id", "EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version", @@ -790,8 +848,11 @@ func Test_ecBackupAnnotations(t *testing.T) { for k, v := range tt.env { t.Setenv(k, v) } - got, err := ecBackupAnnotations(context.TODO(), tt.kbClient, tt.in) - req.NoError(err) + ecMeta := ecInstanceBackupMetadata{ + installation: tt.in, + seaweedFSS3ServiceIP: tt.seaweedFSS3ServiceIP, + } + got := appendECAnnotations(tt.prev, ecMeta) req.Equal(tt.want, got) }) } @@ -800,12 +861,12 @@ func Test_ecBackupAnnotations(t *testing.T) { func Test_ecIncludedNamespaces(t *testing.T) { tests := []struct { name string - in *embeddedclusterv1beta1.Installation + in embeddedclusterv1beta1.Installation want []string }{ { name: "online", - in: &embeddedclusterv1beta1.Installation{}, + in: embeddedclusterv1beta1.Installation{}, want: []string{ "embedded-cluster", "kube-system", @@ -814,7 +875,7 @@ func Test_ecIncludedNamespaces(t *testing.T) { }, { name: "online ha", - in: &embeddedclusterv1beta1.Installation{ + in: embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ HighAvailability: true, }, @@ -827,7 +888,7 @@ func Test_ecIncludedNamespaces(t *testing.T) { }, { name: "airgap", - in: &embeddedclusterv1beta1.Installation{ + in: embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ AirGap: true, }, @@ -841,7 +902,7 @@ func Test_ecIncludedNamespaces(t *testing.T) { }, { name: "airgap ha", - in: &embeddedclusterv1beta1.Installation{ + in: embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ HighAvailability: true, AirGap: true, @@ -864,3 +925,2043 @@ func Test_ecIncludedNamespaces(t *testing.T) { }) } } + +func Test_appendCommonAnnotations(t *testing.T) { + kotsadmSts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm", + Namespace: "kotsadm", + }, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kotsadm", + Image: "kotsadm/kotsadm:1.0.0", + }, + }, + }, + }, + }, + } + registryCredsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "registry-creds", + Namespace: "kotsadm", + }, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + ".dockerconfigjson": []byte(`{"auths":{"host":{"username":"kurl","password":"password"}}}`), + }, + } + + type args struct { + k8sClient kubernetes.Interface + annotations map[string]string + metadata instanceBackupMetadata + hasAppBackup bool + } + tests := []struct { + name string + setup func(t *testing.T) + args args + want map[string]string + wantErr bool + }{ + { + name: "cli install, airgap, multiple apps, not scheduled, has ttl", + setup: func(t *testing.T) { + t.Setenv("DISABLE_OUTBOUND_CONNECTIONS", "true") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + annotations: map[string]string{}, + metadata: instanceBackupMetadata{ + backupName: "instance-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: &apptypes.App{}, + kotsKinds: &kotsutil.KotsKinds{ + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + }, + parentSequence: 1, + }, + "app-2": { + app: &apptypes.App{}, + kotsKinds: &kotsutil.KotsKinds{ + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.2", + }, + }, + }, + parentSequence: 2, + }, + }, + isScheduled: false, + snapshotTTL: 24 * time.Hour, + ec: nil, + }, + hasAppBackup: false, + }, + want: map[string]string{ + "kots.io/apps-sequences": "{\"app-1\":1,\"app-2\":2}", + "kots.io/apps-versions": "{\"app-1\":\"1.0.1\",\"app-2\":\"1.0.2\"}", + "kots.io/embedded-registry": "host", + "kots.io/instance": "true", + "kots.io/is-airgap": "true", + "kots.io/kotsadm-deploy-namespace": "kotsadm", + "kots.io/kotsadm-image": "kotsadm/kotsadm:1.0.0", + "kots.io/snapshot-requested": "2024-01-01T00:00:00Z", + "kots.io/snapshot-trigger": "manual", + }, + }, + { + name: "ec install, scheduled, no ttl, improved dr", + setup: func(t *testing.T) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + annotations: map[string]string{}, + metadata: instanceBackupMetadata{ + backupName: "instance-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: &apptypes.App{}, + kotsKinds: &kotsutil.KotsKinds{ + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + }, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: &ecInstanceBackupMetadata{ + installation: embeddedclusterv1beta1.Installation{ + Spec: embeddedclusterv1beta1.InstallationSpec{ + HighAvailability: true, + Network: &embeddedclusterv1beta1.NetworkSpec{ + PodCIDR: "10.128.0.0/20", + ServiceCIDR: "10.129.0.0/20", + }, + RuntimeConfig: &embeddedclusterv1beta1.RuntimeConfigSpec{ + DataDir: "/var/lib/ec", + AdminConsole: embeddedclusterv1beta1.AdminConsoleSpec{ + Port: 30001, + }, + LocalArtifactMirror: embeddedclusterv1beta1.LocalArtifactMirrorSpec{ + Port: 50001, + }, + }, + }, + }, + seaweedFSS3ServiceIP: "10.96.0.10", + }, + }, + hasAppBackup: true, + }, + want: map[string]string{ + "kots.io/apps-sequences": "{\"app-1\":1}", + "kots.io/apps-versions": "{\"app-1\":\"1.0.1\"}", + "kots.io/embedded-registry": "host", + "kots.io/instance": "true", + "kots.io/is-airgap": "false", + "kots.io/kotsadm-deploy-namespace": "kotsadm", + "kots.io/kotsadm-image": "kotsadm/kotsadm:1.0.0", + "kots.io/snapshot-requested": "2024-01-01T00:00:00Z", + "kots.io/snapshot-trigger": "schedule", + "kots.io/embedded-cluster": "true", + "kots.io/embedded-cluster-id": "embedded-cluster-id", + "kots.io/embedded-cluster-version": "embedded-cluster-version", + "kots.io/embedded-cluster-is-ha": "true", + "kots.io/embedded-cluster-pod-cidr": "10.128.0.0/20", + "kots.io/embedded-cluster-service-cidr": "10.129.0.0/20", + "kots.io/embedded-cluster-seaweedfs-s3-ip": "10.96.0.10", + "kots.io/embedded-cluster-admin-console-port": "30001", + "kots.io/embedded-cluster-local-artifact-mirror-port": "50001", + "kots.io/embedded-cluster-data-dir": "/var/lib/ec", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setup != nil { + tt.setup(t) + } + got, err := appendCommonAnnotations(tt.args.k8sClient, tt.args.annotations, tt.args.metadata, tt.args.hasAppBackup) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_mergeAppBackupSpec(t *testing.T) { + mockStoreExpectApp1 := func(mockStore *mock_store.MockStore) { + mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ + Hostname: "hostname", + Username: "username", + Password: "password", + Namespace: "namespace", + IsReadOnly: true, + }, nil) + } + + type args struct { + backup *velerov1.Backup + appMeta appInstanceBackupMetadata + kotsadmNamespace string + isEC bool + } + tests := []struct { + name string + setup func(t *testing.T, mockStore *mock_store.MockStore) + args args + want *velerov1.Backup + wantErr bool + }{ + { + name: "no backup spec", + args: args{ + backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm"}, + }, + }, + appMeta: appInstanceBackupMetadata{ + app: &apptypes.App{ + ID: "1", + Slug: "app-1", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + Spec: kotsv1beta1.ApplicationSpec{ + AdditionalNamespaces: []string{"another-namespace-1", "another-namespace-2"}, + }, + }, + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + }, + parentSequence: 1, + }, + kotsadmNamespace: "kotsadm", + isEC: false, + }, + want: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm"}, + }, + }, + }, + { + name: "has backup spec", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + mockStoreExpectApp1(mockStore) + }, + args: args{ + backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm"}, + }, + }, + appMeta: appInstanceBackupMetadata{ + app: &apptypes.App{ + ID: "1", + Slug: "app-1", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + Spec: kotsv1beta1.ApplicationSpec{ + AdditionalNamespaces: []string{"another-namespace-1", "another-namespace-2"}, + }, + }, + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + Backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "annotation-1": "true", + "annotation-2": "false", + }, + }, + Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"include-namespace-1", "include-namespace-2", "template-isairgap-{{repl IsAirgap }}"}, + ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, + OrderedResources: map[string]string{ + "resource-1": "true", + "resource-2": "false", + }, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{ + { + Name: "hook-1", + }, + { + Name: "hook-2", + }, + }, + }, + }, + }, + }, + parentSequence: 1, + }, + kotsadmNamespace: "kotsadm", + isEC: false, + }, + want: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + "annotation-1": "true", + "annotation-2": "false", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm", "another-namespace-1", "another-namespace-2", "include-namespace-1", "include-namespace-2", "template-isairgap-true"}, + ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, + OrderedResources: map[string]string{ + "resource-1": "true", + "resource-2": "false", + }, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{ + { + Name: "hook-1", + }, + { + Name: "hook-2", + }, + }, + }, + }, + }, + }, + { + name: "ec, no backup spec", + args: args{ + backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm"}, + }, + }, + appMeta: appInstanceBackupMetadata{ + app: &apptypes.App{ + ID: "1", + Slug: "app-1", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + Spec: kotsv1beta1.ApplicationSpec{ + AdditionalNamespaces: []string{"another-namespace-1", "another-namespace-2"}, + }, + }, + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + }, + parentSequence: 1, + }, + kotsadmNamespace: "kotsadm", + isEC: true, + }, + want: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm", "another-namespace-1", "another-namespace-2"}, + }, + }, + }, + { + name: "ec, has backup spec", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + mockStoreExpectApp1(mockStore) + }, + args: args{ + backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm"}, + }, + }, + appMeta: appInstanceBackupMetadata{ + app: &apptypes.App{ + ID: "1", + Slug: "app-1", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + Spec: kotsv1beta1.ApplicationSpec{ + AdditionalNamespaces: []string{"another-namespace-1", "another-namespace-2"}, + }, + }, + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + Backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "annotation-1": "true", + "annotation-2": "false", + }, + }, + Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"include-namespace-1", "include-namespace-2", "template-isairgap-{{repl IsAirgap }}"}, + ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, + OrderedResources: map[string]string{ + "resource-1": "true", + "resource-2": "false", + }, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{ + { + Name: "hook-1", + }, + { + Name: "hook-2", + }, + }, + }, + }, + }, + }, + parentSequence: 1, + }, + kotsadmNamespace: "kotsadm", + isEC: true, + }, + want: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + "annotation-1": "true", + "annotation-2": "false", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm", "another-namespace-1", "another-namespace-2", "include-namespace-1", "include-namespace-2", "template-isairgap-true"}, + ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, + OrderedResources: map[string]string{ + "resource-1": "true", + "resource-2": "false", + }, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{ + { + Name: "hook-1", + }, + { + Name: "hook-2", + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStore := mock_store.NewMockStore(ctrl) + store.SetStore(mockStore) + + t.Cleanup(func() { + store.SetStore(nil) + }) + + if tt.setup != nil { + tt.setup(t, mockStore) + } + err := mergeAppBackupSpec(tt.args.backup, tt.args.appMeta, tt.args.kotsadmNamespace, tt.args.isEC) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tt.want, tt.args.backup) + }) + } +} + +func Test_getAppInstanceBackupSpec(t *testing.T) { + EnableImprovedDR = true + t.Cleanup(func() { + EnableImprovedDR = false + }) + + kotsadmSts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm", + Namespace: "kotsadm", + }, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kotsadm", + Image: "kotsadm/kotsadm:1.0.0", + }, + }, + }, + }, + }, + } + registryCredsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "registry-creds", + Namespace: "kotsadm", + }, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + ".dockerconfigjson": []byte(`{"auths":{"host":{"username":"kurl","password":"password"}}}`), + }, + } + + app1 := &apptypes.App{ + ID: "1", + Slug: "app-1", + IsAirgap: true, + } + + app2 := &apptypes.App{ + ID: "2", + Slug: "app-2", + IsAirgap: true, + } + + kotsKinds := &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + Spec: kotsv1beta1.ApplicationSpec{ + AdditionalNamespaces: []string{"another-namespace-1", "another-namespace-2"}, + }, + }, + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + Backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-backup", + Annotations: map[string]string{ + "annotation-1": "true", + "annotation-2": "false", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "blah", + TTL: metav1.Duration{Duration: 1 * time.Hour}, + IncludedNamespaces: []string{"include-namespace-1", "include-namespace-2"}, + ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, + OrderedResources: map[string]string{ + "resource-1": "true", + "resource-2": "false", + }, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{ + { + Name: "hook-1", + }, + { + Name: "hook-2", + }, + }, + }, + }, + }, + Restore: &velerov1.Restore{}, + } + + ecMeta := &ecInstanceBackupMetadata{ + installation: embeddedclusterv1beta1.Installation{ + Spec: embeddedclusterv1beta1.InstallationSpec{ + HighAvailability: true, + Network: &embeddedclusterv1beta1.NetworkSpec{ + PodCIDR: "10.128.0.0/20", + ServiceCIDR: "10.129.0.0/20", + }, + RuntimeConfig: &embeddedclusterv1beta1.RuntimeConfigSpec{ + DataDir: "/var/lib/ec", + AdminConsole: embeddedclusterv1beta1.AdminConsoleSpec{ + Port: 30001, + }, + LocalArtifactMirror: embeddedclusterv1beta1.LocalArtifactMirrorSpec{ + Port: 50001, + }, + }, + }, + }, + seaweedFSS3ServiceIP: "10.96.0.10", + } + + type args struct { + k8sClient kubernetes.Interface + metadata instanceBackupMetadata + } + tests := []struct { + name string + setup func(t *testing.T, mockStore *mock_store.MockStore) + args args + assert func(t *testing.T, got *velerov1.Backup, err error) + }{ + { + name: "not ec with backup and restore spec should return nil", + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: &kotsutil.KotsKinds{ + Backup: &velerov1.Backup{}, + Restore: &velerov1.Restore{}, + }, + parentSequence: 1, + }, + }, + ec: nil, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Nil(t, got) + }, + }, + { + name: "ec wihtout restore spec should return nil", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "instance-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: &kotsutil.KotsKinds{ + Backup: &velerov1.Backup{}, + }, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Nil(t, got) + }, + }, + { + name: "ec with backup and restore spec and multiple apps should return error", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "instance-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + "app-2": { + app: app2, + kotsKinds: kotsKinds, + parentSequence: 2, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.Error(t, err) + assert.Nil(t, got) + }, + }, + { + name: "not ec with backup and restore spec and multiple apps should not return error", + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: &kotsutil.KotsKinds{ + Backup: &velerov1.Backup{}, + Restore: &velerov1.Restore{}, + }, + parentSequence: 1, + }, + "app-2": { + app: app2, + kotsKinds: &kotsutil.KotsKinds{ + Backup: &velerov1.Backup{}, + Restore: &velerov1.Restore{}, + }, + parentSequence: 2, + }, + }, + ec: nil, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Nil(t, got) + }, + }, + { + name: "ec with backup and restore spec should override name", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Equal(t, "", got.Name) + assert.Equal(t, "application-", got.GenerateName) + }, + }, + { + name: "ec with backup and restore spec should append backup name label", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + if assert.Contains(t, got.Labels, "replicated.com/backup-name") { + assert.Equal(t, "app-1-17332487841234", got.Labels["replicated.com/backup-name"]) + } + }, + }, + { + name: "ec with backup and restore spec should append common annotations", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + if assert.Contains(t, got.Annotations, "replicated.com/backup-type") { + assert.Equal(t, "app", got.Annotations["replicated.com/backup-type"]) + } + if assert.Contains(t, got.Annotations, "replicated.com/backup-count") { + assert.Equal(t, "2", got.Annotations["replicated.com/backup-count"]) + } + }, + }, + { + name: "ec with backup and restore spec overrides storage location", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "instance-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Equal(t, "default", got.Spec.StorageLocation) + }, + }, + { + name: "ec with backup and restore spec overrides snapshot ttl", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "instance-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + snapshotTTL: 24 * time.Hour, + ec: ecMeta, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Equal(t, metav1.Duration{Duration: 24 * time.Hour}, got.Spec.TTL) + }, + }, + { + name: "ec with backup and restore spec does not override snapshot ttl if unset", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "instance-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Equal(t, metav1.Duration{Duration: 1 * time.Hour}, got.Spec.TTL) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStore := mock_store.NewMockStore(ctrl) + store.SetStore(mockStore) + + t.Cleanup(func() { + store.SetStore(nil) + }) + + if tt.setup != nil { + tt.setup(t, mockStore) + } + got, err := getAppInstanceBackupSpec(tt.args.k8sClient, tt.args.metadata) + tt.assert(t, got, err) + }) + } +} + +func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { + kotsadmSts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm", + Namespace: "kotsadm", + }, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kotsadm", + Image: "kotsadm/kotsadm:1.0.0", + }, + }, + }, + }, + }, + } + registryCredsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "registry-creds", + Namespace: "kotsadm", + }, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + ".dockerconfigjson": []byte(`{"auths":{"host":{"username":"kurl","password":"password"}}}`), + }, + } + + app1 := &apptypes.App{ + ID: "1", + Slug: "app-1", + IsAirgap: true, + } + + kotsKinds := &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + Spec: kotsv1beta1.ApplicationSpec{ + AdditionalNamespaces: []string{"another-namespace-1", "another-namespace-2", "duplicate-namespace"}, + }, + }, + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + Backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-backup", + Annotations: map[string]string{ + "annotation-1": "true", + "annotation-2": "false", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "blah", + TTL: metav1.Duration{Duration: 1 * time.Hour}, + IncludedNamespaces: []string{"include-namespace-1", "include-namespace-2", "template-isairgap-{{repl IsAirgap }}", "duplicate-namespace"}, + ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, + OrderedResources: map[string]string{ + "resource-1": "true", + "resource-2": "false", + }, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{ + { + Name: "hook-1", + }, + { + Name: "hook-2", + }, + }, + }, + }, + }, + Restore: &velerov1.Restore{}, + } + + ecMeta := &ecInstanceBackupMetadata{ + installation: embeddedclusterv1beta1.Installation{ + Spec: embeddedclusterv1beta1.InstallationSpec{ + HighAvailability: true, + Network: &embeddedclusterv1beta1.NetworkSpec{ + PodCIDR: "10.128.0.0/20", + ServiceCIDR: "10.129.0.0/20", + }, + RuntimeConfig: &embeddedclusterv1beta1.RuntimeConfigSpec{ + DataDir: "/var/lib/ec", + AdminConsole: embeddedclusterv1beta1.AdminConsoleSpec{ + Port: 30001, + }, + LocalArtifactMirror: embeddedclusterv1beta1.LocalArtifactMirrorSpec{ + Port: 50001, + }, + }, + }, + }, + seaweedFSS3ServiceIP: "10.96.0.10", + } + + mockStoreExpectApp1 := func(mockStore *mock_store.MockStore) { + mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ + Hostname: "hostname", + Username: "username", + Password: "password", + Namespace: "namespace", + IsReadOnly: true, + }, nil) + } + + type args struct { + k8sClient kubernetes.Interface + metadata instanceBackupMetadata + hasAppBackup bool + } + tests := []struct { + name string + setup func(t *testing.T, mockStore *mock_store.MockStore) + args args + assert func(t *testing.T, got *velerov1.Backup, err error) + }{ + { + name: "KOTSADM_TARGET_NAMESPACE should be added to includedNamespaces", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + util.KotsadmTargetNamespace = "kotsadm-target" + t.Cleanup(func() { + util.KotsadmTargetNamespace = "" + }) + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: nil, + }, + hasAppBackup: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "kotsadm") + assert.Contains(t, got.Spec.IncludedNamespaces, "kotsadm-target") + }, + }, + { + name: "if kurl should be added to includedNamespaces", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kurl-config", + Namespace: "kube-system", + }, + }), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: nil, + }, + hasAppBackup: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "kurl") + }, + }, + { + name: "not cluster scoped should include backup storage location namespace", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: nil, + }, + hasAppBackup: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "kotsadm-backups") + }, + }, + { + name: "cluster scoped should not include backup storage location namespace", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret, &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm-rolebinding", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "kotsadm", + Namespace: "kotsadm", + }, + }, + }), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: nil, + }, + hasAppBackup: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.NotContains(t, got.Spec.IncludedNamespaces, "kotsadm-backups") + }, + }, + { + name: "should merge backup spec when not using improved dr", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppBackup: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "include-namespace-1") + }, + }, + { + name: "should not merge backup spec when using improved dr", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppBackup: true, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.NotContains(t, got.Spec.IncludedNamespaces, "include-namespace-1") + }, + }, + { + name: "should not add improved dr metadata when not using improved dr", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppBackup: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.NotContains(t, got.Labels, "replicated.com/backup-name") + assert.NotContains(t, got.Annotations, "replicated.com/backup-type") + assert.NotContains(t, got.Annotations, "replicated.com/backup-count") + }, + }, + { + name: "should add improved dr metadata when not using improved dr", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppBackup: true, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + if assert.Contains(t, got.Labels, "replicated.com/backup-name") { + assert.Equal(t, "app-1-17332487841234", got.Labels["replicated.com/backup-name"]) + } + if assert.Contains(t, got.Annotations, "replicated.com/backup-type") { + assert.Equal(t, "infra", got.Annotations["replicated.com/backup-type"]) + } + if assert.Contains(t, got.Annotations, "replicated.com/backup-count") { + assert.Equal(t, "2", got.Annotations["replicated.com/backup-count"]) + } + }, + }, + { + name: "should add ec namespaces to includedNamespaces if ec", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppBackup: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "embedded-cluster") + }, + }, + { + name: "should add ec namespaces to includedNamespaces if ec", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppBackup: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "embedded-cluster") + }, + }, + { + name: "should override snapshot ttl if set", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + snapshotTTL: 24 * time.Hour, + ec: ecMeta, + }, + hasAppBackup: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Equal(t, metav1.Duration{Duration: 24 * time.Hour}, got.Spec.TTL) + }, + }, + { + name: "should not override snapshot ttl if unset", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppBackup: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Zero(t, got.Spec.TTL) + }, + }, + { + name: "should deduplicate includedNamespaces", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppBackup: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + count := 0 + for _, ns := range got.Spec.IncludedNamespaces { + if ns == "duplicate-namespace" { + count++ + } + } + assert.Equal(t, 1, count, "Duplicate namespace should be removed") + }, + }, + { + name: "should render app backup spec", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppBackup: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "template-isairgap-true") + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStore := mock_store.NewMockStore(ctrl) + store.SetStore(mockStore) + + t.Cleanup(func() { + store.SetStore(nil) + }) + + if tt.setup != nil { + tt.setup(t, mockStore) + } + got, err := getInfrastructureInstanceBackupSpec(context.Background(), tt.args.k8sClient, tt.args.metadata, tt.args.hasAppBackup) + tt.assert(t, got, err) + }) + } +} + +func Test_getInstanceBackupMetadata(t *testing.T) { + scheme := runtime.NewScheme() + corev1.AddToScheme(scheme) + embeddedclusterv1beta1.AddToScheme(scheme) + + testBsl := &velerov1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "velero", + }, + Spec: velerov1.BackupStorageLocationSpec{ + Provider: "aws", + Default: true, + }, + } + veleroNamespaceConfigmap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm-velero-namespace", + }, + Data: map[string]string{ + "veleroNamespace": "velero", + }, + } + veleroDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "velero", + Namespace: "velero", + }, + } + + installation := embeddedclusterv1beta1.Installation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "20060102150405", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + BinaryName: "my-app", + }, + } + seaweedFSS3Service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ec-seaweedfs-s3", + Namespace: "seaweedfs", + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.96.0.10", + }, + } + + type args struct { + k8sClient kubernetes.Interface + ctrlClient ctrlclient.Client + veleroClient veleroclientv1.VeleroV1Interface + cluster *downstreamtypes.Downstream + isScheduled bool + } + tests := []struct { + name string + setup func(t *testing.T, mockStore *mock_store.MockStore) + args args + want instanceBackupMetadata + wantErr bool + }{ + { + name: "cli install", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + util.PodNamespace = "test" + t.Cleanup(func() { + util.PodNamespace = "" + }) + + mockStore.EXPECT().ListInstalledApps().Times(1).Return([]*apptypes.App{ + { + ID: "1", + Name: "App 1", + Slug: "app-1", + IsAirgap: true, + }, + { + ID: "2", + Name: "App 2", + Slug: "app-2", + IsAirgap: true, + }, + }, nil) + mockStore.EXPECT().ListDownstreamsForApp(gomock.Any()).Times(2).Return([]downstreamtypes.Downstream{ + { + ClusterID: "cluster-id", + ClusterSlug: "cluster-slug", + Name: "cluster-name", + CurrentSequence: 1, + SnapshotSchedule: "manual", + SnapshotTTL: "24h", + }, + }, nil) + mockStore.EXPECT().GetCurrentParentSequence("1", "cluster-id").Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetCurrentParentSequence("2", "cluster-id").Times(1).Return(int64(2), nil) + mockStore.EXPECT().GetAppVersionArchive("1", int64(1), gomock.Any()).Times(1).DoAndReturn(func(appID string, sequence int64, archiveDir string) error { + err := setupArchiveDirectoriesAndFiles(archiveDir, map[string]string{ + "upstream/app.yaml": ` +apiVersion: kots.io/v1beta1 +kind: Application +metadata: + name: app-1 +spec: + title: My App 1`, + }) + require.NoError(t, err) + return nil + }) + mockStore.EXPECT().GetAppVersionArchive("2", int64(2), gomock.Any()).Times(1).DoAndReturn(func(appID string, sequence int64, archiveDir string) error { + err := setupArchiveDirectoriesAndFiles(archiveDir, map[string]string{ + "upstream/app.yaml": ` +apiVersion: kots.io/v1beta1 +kind: Application +metadata: + name: app-2 +spec: + title: My App 2`, + }) + require.NoError(t, err) + return nil + }) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), + ctrlClient: ctrlclientfake.NewClientBuilder().WithScheme(scheme).WithObjects().Build(), + veleroClient: velerofake.NewSimpleClientset(testBsl).VeleroV1(), + cluster: &downstreamtypes.Downstream{ + SnapshotTTL: "24h", + }, + isScheduled: true, + }, + want: instanceBackupMetadata{ + backupName: "instance-", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "test", + backupStorageLocationNamespace: "velero", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: &apptypes.App{ + ID: "1", + Name: "App 1", + Slug: "app-1", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kots.io/v1beta1", + Kind: "Application", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "app-1", + }, + Spec: kotsv1beta1.ApplicationSpec{ + Title: "My App 1", + }, + }, + Installation: kotsv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kots.io/v1beta1", + Kind: "Installation", + }, + }, + }, + parentSequence: 1, + }, + "app-2": { + app: &apptypes.App{ + ID: "2", + Name: "App 2", + Slug: "app-2", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kots.io/v1beta1", + Kind: "Application", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "app-2", + }, + Spec: kotsv1beta1.ApplicationSpec{ + Title: "My App 2", + }, + }, + Installation: kotsv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kots.io/v1beta1", + Kind: "Installation", + }, + }, + }, + parentSequence: 2, + }, + }, + isScheduled: true, + snapshotTTL: 24 * time.Hour, + ec: nil, + }, + }, + { + name: "ec install", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + + util.PodNamespace = "test" + t.Cleanup(func() { + util.PodNamespace = "" + }) + + mockStore.EXPECT().ListInstalledApps().Times(1).Return([]*apptypes.App{ + { + ID: "1", + Name: "App 1", + Slug: "app-1", + IsAirgap: true, + }, + }, nil) + mockStore.EXPECT().ListDownstreamsForApp(gomock.Any()).Times(1).Return([]downstreamtypes.Downstream{ + { + ClusterID: "cluster-id", + ClusterSlug: "cluster-slug", + Name: "cluster-name", + CurrentSequence: 1, + SnapshotSchedule: "manual", + SnapshotTTL: "24h", + }, + }, nil) + mockStore.EXPECT().GetCurrentParentSequence("1", "cluster-id").Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetAppVersionArchive("1", int64(1), gomock.Any()).Times(1).DoAndReturn(func(appID string, sequence int64, archiveDir string) error { + err := setupArchiveDirectoriesAndFiles(archiveDir, map[string]string{ + "upstream/app.yaml": ` +apiVersion: kots.io/v1beta1 +kind: Application +metadata: + name: app-1 +spec: + title: My App 1`, + }) + require.NoError(t, err) + return nil + }) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), + ctrlClient: ctrlclientfake.NewClientBuilder().WithScheme(scheme).WithObjects( + &installation, + seaweedFSS3Service, + ).Build(), + veleroClient: velerofake.NewSimpleClientset(testBsl).VeleroV1(), + cluster: &downstreamtypes.Downstream{ + SnapshotTTL: "24h", + }, + isScheduled: true, + }, + want: instanceBackupMetadata{ + backupName: "app-1-", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "test", + backupStorageLocationNamespace: "velero", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: &apptypes.App{ + ID: "1", + Name: "App 1", + Slug: "app-1", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kots.io/v1beta1", + Kind: "Application", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "app-1", + }, + Spec: kotsv1beta1.ApplicationSpec{ + Title: "My App 1", + }, + }, + Installation: kotsv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kots.io/v1beta1", + Kind: "Installation", + }, + }, + }, + parentSequence: 1, + }, + }, + isScheduled: true, + snapshotTTL: 24 * time.Hour, + ec: &ecInstanceBackupMetadata{ + installation: installation, + seaweedFSS3ServiceIP: "10.96.0.10", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStore := mock_store.NewMockStore(ctrl) + store.SetStore(mockStore) + + t.Cleanup(func() { + store.SetStore(nil) + }) + + if tt.setup != nil { + tt.setup(t, mockStore) + } + + got, err := getInstanceBackupMetadata(context.Background(), tt.args.k8sClient, tt.args.ctrlClient, tt.args.veleroClient, tt.args.cluster, tt.args.isScheduled) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + assert.Regexp(t, "^"+tt.want.backupName, got.backupName) + assert.NotZero(t, got.backupReqestedAt) + tt.want.backupName = got.backupName + tt.want.backupReqestedAt = got.backupReqestedAt + + assert.Equal(t, tt.want, got) + }) + } +} + +func setupArchiveDirectoriesAndFiles(archiveDir string, files map[string]string) error { + for path, content := range files { + dir := filepath.Dir(path) + if err := os.MkdirAll(filepath.Join(archiveDir, dir), 0744); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(archiveDir, path), []byte(content), 0644); err != nil { + return err + } + } + return nil +} + +func Test_getBackupNameFromPrefix(t *testing.T) { + type args struct { + appSlug string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "basic", + args: args{ + appSlug: "test", + }, + want: `^test-[a-f0-9]{8}$`, + }, + { + name: "truncate", + args: args{ + appSlug: "test-truncate-this-string-to-a-valid-backup-name-length", + }, + want: `^test-truncate-this-string-to-a-valid-backup-name-lengt-[a-f0-9]{8}$`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getBackupNameFromPrefix(tt.args.appSlug) + assert.Regexp(t, tt.want, got) + assert.LessOrEqual(t, len(got), validation.DNS1035LabelMaxLength) + }) + } +} diff --git a/pkg/kotsadmsnapshot/restore.go b/pkg/kotsadmsnapshot/restore.go index 97c1d20f0c..6fe82be98e 100644 --- a/pkg/kotsadmsnapshot/restore.go +++ b/pkg/kotsadmsnapshot/restore.go @@ -16,14 +16,13 @@ import ( velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" velerolabel "github.com/vmware-tanzu/velero/pkg/label" - "go.uber.org/zap" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/utils/pointer" ) -func GetRestore(ctx context.Context, kotsadmNamespace string, snapshotName string) (*velerov1.Restore, error) { +func GetRestore(ctx context.Context, kotsadmNamespace string, restoreID string) (*velerov1.Restore, error) { cfg, err := k8sutil.GetClusterConfig() if err != nil { return nil, errors.Wrap(err, "failed to get cluster config") @@ -49,7 +48,7 @@ func GetRestore(ctx context.Context, kotsadmNamespace string, snapshotName strin veleroNamespace := bsl.Namespace - restore, err := veleroClient.Restores(veleroNamespace).Get(ctx, snapshotName, metav1.GetOptions{}) + restore, err := veleroClient.Restores(veleroNamespace).Get(ctx, restoreID, metav1.GetOptions{}) if err != nil { if kuberneteserrors.IsNotFound(err) { return nil, nil @@ -60,11 +59,10 @@ func GetRestore(ctx context.Context, kotsadmNamespace string, snapshotName strin return restore, nil } -func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snapshotName string, appSlug string) error { +func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, backupID string, appSlug string) error { // Reference https://github.com/vmware-tanzu/velero/blob/42b612645863c2b3e451b447f9bf798295dd7dba/pkg/cmd/cli/restore/create.go#L222 - logger.Debug("creating restore", - zap.String("snapshotName", snapshotName)) + logger.Debugf("Creating restore for backup %s", backupID) cfg, err := k8sutil.GetClusterConfig() if err != nil { @@ -92,7 +90,7 @@ func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snap veleroNamespace := bsl.Namespace // get the backup - backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, snapshotName, metav1.GetOptions{}) + backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, backupID, metav1.GetOptions{}) if err != nil { return errors.Wrap(err, "failed to find backup") } @@ -100,20 +98,24 @@ func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snap restore := &velerov1.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, - Name: snapshotName, // restore name same as snapshot name + Name: backupID, // restore name same as snapshot name }, Spec: velerov1.RestoreSpec{ - BackupName: snapshotName, + BackupName: backupID, RestorePVs: pointer.Bool(true), IncludeClusterResources: pointer.Bool(true), }, } - if backup.Annotations["kots.io/instance"] == "true" { + if IsInstanceBackup(*backup) { + if GetInstanceBackupType(*backup) != types.InstanceBackupTypeLegacy { + return errors.New("only legacy type instance backups are restorable") + } + // only restore app-specific objects - restore.ObjectMeta.Name = fmt.Sprintf("%s.%s", snapshotName, appSlug) + restore.ObjectMeta.Name = fmt.Sprintf("%s.%s", backupID, appSlug) restore.ObjectMeta.Annotations = map[string]string{ - "kots.io/instance": "true", + types.InstanceBackupAnnotation: "true", } restore.Spec.LabelSelector = &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -130,7 +132,7 @@ func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snap return nil } -func DeleteRestore(ctx context.Context, kotsadmNamespace string, snapshotName string) error { +func DeleteRestore(ctx context.Context, kotsadmNamespace string, restoreID string) error { cfg, err := k8sutil.GetClusterConfig() if err != nil { return errors.Wrap(err, "failed to get cluster config") @@ -153,15 +155,15 @@ func DeleteRestore(ctx context.Context, kotsadmNamespace string, snapshotName st veleroNamespace := bsl.Namespace - err = veleroClient.Restores(veleroNamespace).Delete(ctx, snapshotName, metav1.DeleteOptions{}) + err = veleroClient.Restores(veleroNamespace).Delete(ctx, restoreID, metav1.DeleteOptions{}) if err != nil && !strings.Contains(err.Error(), "not found") { - return errors.Wrapf(err, "failed to delete restore %s", snapshotName) + return errors.Wrapf(err, "failed to delete restore %s", restoreID) } return nil } -func GetRestoreDetails(ctx context.Context, kotsadmNamespace string, restoreName string) (*types.RestoreDetail, error) { +func GetRestoreDetails(ctx context.Context, kotsadmNamespace string, restoreID string) (*types.RestoreDetail, error) { cfg, err := k8sutil.GetClusterConfig() if err != nil { return nil, errors.Wrap(err, "failed to get cluster config") @@ -187,7 +189,7 @@ func GetRestoreDetails(ctx context.Context, kotsadmNamespace string, restoreName veleroNamespace := backendStorageLocation.Namespace - restore, err := veleroClient.Restores(veleroNamespace).Get(ctx, restoreName, metav1.GetOptions{}) + restore, err := veleroClient.Restores(veleroNamespace).Get(ctx, restoreID, metav1.GetOptions{}) if err != nil { return nil, errors.Wrap(err, "failed to get restore") } diff --git a/pkg/kotsadmsnapshot/types/types.go b/pkg/kotsadmsnapshot/types/types.go index d23cc379d6..7e67ed15ae 100644 --- a/pkg/kotsadmsnapshot/types/types.go +++ b/pkg/kotsadmsnapshot/types/types.go @@ -6,6 +6,29 @@ import ( velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) +const ( + // InstanceBackupNameLabel is the label used to store the name of the backup for an instance + // backup. + InstanceBackupNameLabel = "replicated.com/backup-name" + // InstanceBackupTypeAnnotation is the annotation used to store the type of backup for an + // instance backup. + InstanceBackupTypeAnnotation = "replicated.com/backup-type" + // InstanceBackupCountAnnotation is the annotation used to store the expected number of backups + // for an instance backup. + InstanceBackupCountAnnotation = "replicated.com/backup-count" + + // InstanceBackupTypeInfra indicates that the backup is of type infrastructure. + InstanceBackupTypeInfra = "infra" + // InstanceBackupTypeApp indicates that the backup is of type application. + InstanceBackupTypeApp = "app" + // InstanceBackupTypeLegacy indicates that the backup is of type legacy (infra + app). + InstanceBackupTypeLegacy = "legacy" + + // InstanceBackupAnnotation is the annotation used to indicate that a backup is an instance + // backup. + InstanceBackupAnnotation = "kots.io/instance" +) + type App struct { Slug string `json:"slug"` Sequence int64 `json:"sequence"` diff --git a/pkg/kotsutil/kots.go b/pkg/kotsutil/kots.go index 67b26214a1..4b69b7f5a8 100644 --- a/pkg/kotsutil/kots.go +++ b/pkg/kotsutil/kots.go @@ -107,6 +107,7 @@ type KotsKinds struct { IdentityConfig *kotsv1beta1.IdentityConfig Backup *velerov1.Backup + Restore *velerov1.Restore Installer *kurlv1beta1.Installer LintConfig *kotsv1beta1.LintConfig @@ -115,7 +116,7 @@ type KotsKinds struct { } func IsKotsKind(apiVersion string, kind string) bool { - if apiVersion == "velero.io/v1" && kind == "Backup" { + if apiVersion == "velero.io/v1" && (kind == "Backup" || kind == "Restore") { return true } if apiVersion == "kots.io/v1beta1" { @@ -437,30 +438,25 @@ func (o KotsKinds) Marshal(g string, v string, k string) (string, error) { if g == "velero.io" { if v == "v1" { - if k == "Backup" { + switch k { + case "Backup": if o.Backup == nil { - if util.IsEmbeddedCluster() { - // return the default backup object - backup := &velerov1.Backup{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "velero.io/v1", - Kind: "Backup", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "backup", - }, - } - o.Backup = backup - } else { - return "", nil - } - + return "", nil } var b bytes.Buffer if err := s.Encode(o.Backup, &b); err != nil { return "", errors.Wrap(err, "failed to encode backup") } return string(b.Bytes()), nil + case "Restore": + if o.Restore == nil { + return "", nil + } + var b bytes.Buffer + if err := s.Encode(o.Restore, &b); err != nil { + return "", errors.Wrap(err, "failed to encode restore") + } + return string(b.Bytes()), nil } } } @@ -567,6 +563,8 @@ func (k *KotsKinds) addKotsKinds(content []byte) error { k.HostPreflight = decoded.(*troubleshootv1beta2.HostPreflight) case "velero.io/v1, Kind=Backup": k.Backup = decoded.(*velerov1.Backup) + case "velero.io/v1, Kind=Restore": + k.Restore = decoded.(*velerov1.Restore) case "kurl.sh/v1beta1, Kind=Installer", "cluster.kurl.sh/v1beta1, Kind=Installer": k.Installer = decoded.(*kurlv1beta1.Installer) case "app.k8s.io/v1beta1, Kind=Application": @@ -1085,6 +1083,21 @@ func LoadBackupFromContents(content []byte) (*velerov1.Backup, error) { return obj.(*velerov1.Backup), nil } +func LoadRestoreFromContents(content []byte) (*velerov1.Restore, error) { + decode := scheme.Codecs.UniversalDeserializer().Decode + + obj, gvk, err := decode(content, nil, nil) + if err != nil { + return nil, errors.Wrapf(err, "failed to decode: %v", string(content)) + } + + if gvk.String() != "velero.io/v1, Kind=Restore" { + return nil, errors.Errorf("unexpected gvk: %s", gvk.String()) + } + + return obj.(*velerov1.Restore), nil +} + func LoadApplicationFromContents(content []byte) (*applicationv1beta1.Application, error) { decode := scheme.Codecs.UniversalDeserializer().Decode diff --git a/pkg/kotsutil/kots_test.go b/pkg/kotsutil/kots_test.go index db580d103f..44e5fe512e 100644 --- a/pkg/kotsutil/kots_test.go +++ b/pkg/kotsutil/kots_test.go @@ -1051,18 +1051,7 @@ status: {} v: "v1", k: "Backup", }, - want: `apiVersion: velero.io/v1 -kind: Backup -metadata: - creationTimestamp: null - name: backup -spec: - csiSnapshotTimeout: 0s - hooks: {} - metadata: {} - ttl: 0s -status: {} -`, + want: "", }, { name: "backup exists, EC", diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 2e750b5ef8..0d414d3321 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -603,7 +603,7 @@ func (o *Operator) handleUndeployCompleted(a *apptypes.App) error { if err != nil { return errors.Wrap(err, "failed to get backup") } - if backup.Annotations["kots.io/instance"] == "true" { + if snapshot.IsInstanceBackup(*backup) { restoreName = fmt.Sprintf("%s.%s", snapshotName, a.Slug) } @@ -643,7 +643,7 @@ func (o *Operator) checkRestoreComplete(a *apptypes.App, restore *velerov1.Resto } var sequence int64 = 0 - if backupAnnotations["kots.io/instance"] == "true" { + if snapshot.IsInstanceBackup(*backup) { b, ok := backupAnnotations["kots.io/apps-sequences"] if !ok || b == "" { return errors.New("instance backup is missing apps sequences annotation") diff --git a/pkg/snapshot/backup.go b/pkg/snapshot/backup.go index 4404f4fb73..26f47e91b5 100644 --- a/pkg/snapshot/backup.go +++ b/pkg/snapshot/backup.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "os" "time" @@ -12,6 +12,7 @@ import ( "github.com/pkg/errors" "github.com/replicatedhq/kots/pkg/auth" "github.com/replicatedhq/kots/pkg/k8sutil" + snapshottypes "github.com/replicatedhq/kots/pkg/kotsadmsnapshot/types" "github.com/replicatedhq/kots/pkg/logger" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" @@ -104,7 +105,7 @@ func CreateInstanceBackup(ctx context.Context, options CreateInstanceBackupOptio } defer resp.Body.Close() - respBody, err := ioutil.ReadAll(resp.Body) + respBody, err := io.ReadAll(resp.Body) if err != nil { log.FinishSpinnerWithError() return nil, errors.Wrap(err, "failed to read server response") @@ -166,7 +167,6 @@ func CreateInstanceBackup(ctx context.Context, options CreateInstanceBackupOptio } func ListInstanceBackups(ctx context.Context, options ListInstanceBackupsOptions) ([]velerov1.Backup, error) { - b, err := ListAllBackups(ctx, options) if err != nil { return nil, errors.Wrap(err, "failed to get backup list") @@ -175,7 +175,7 @@ func ListInstanceBackups(ctx context.Context, options ListInstanceBackupsOptions backups := []velerov1.Backup{} for _, backup := range b { - if backup.Annotations["kots.io/instance"] != "true" { + if backup.Annotations[snapshottypes.InstanceBackupAnnotation] != "true" { continue } diff --git a/pkg/snapshot/restore.go b/pkg/snapshot/restore.go index af3c438f3b..bb61fed4b9 100644 --- a/pkg/snapshot/restore.go +++ b/pkg/snapshot/restore.go @@ -72,10 +72,14 @@ func RestoreInstanceBackup(ctx context.Context, options RestoreInstanceBackupOpt } // make sure this is an instance backup - if backup.Annotations["kots.io/instance"] != "true" { + if backup.Annotations[snapshottypes.InstanceBackupAnnotation] != "true" { return errors.Wrap(err, "backup provided is not an instance backup") } + if getInstanceBackupType(*backup) != snapshottypes.InstanceBackupTypeLegacy { + return errors.New("only legacy type instance backups are restorable") + } + kotsadmImage, ok := backup.Annotations["kots.io/kotsadm-image"] if !ok { return errors.Wrap(err, "failed to find kotsadm image annotation") @@ -123,15 +127,15 @@ func RestoreInstanceBackup(ctx context.Context, options RestoreInstanceBackupOpt restore := &velerov1.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, - Name: fmt.Sprintf("%s.kotsadm", options.BackupName), + Name: fmt.Sprintf("%s.kotsadm", backup.Name), Annotations: map[string]string{ - "kots.io/instance": "true", - "kots.io/kotsadm-image": kotsadmImage, - "kots.io/kotsadm-deploy-namespace": kotsadmNamespace, + snapshottypes.InstanceBackupAnnotation: "true", + "kots.io/kotsadm-image": kotsadmImage, + "kots.io/kotsadm-deploy-namespace": kotsadmNamespace, }, }, Spec: velerov1.RestoreSpec{ - BackupName: options.BackupName, + BackupName: backup.Name, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ kotsadmtypes.KotsadmKey: kotsadmtypes.KotsadmLabelValue, // restoring applications is in a separate step after kotsadm spins up @@ -200,7 +204,7 @@ func RestoreInstanceBackup(ctx context.Context, options RestoreInstanceBackupOpt } // initiate kotsadm applications restore - err = initiateKotsadmApplicationsRestore(options.BackupName, kotsadmNamespace, kotsadmPodName, log) + err = initiateKotsadmApplicationsRestore(backup.Name, kotsadmNamespace, kotsadmPodName, log) if err != nil { log.FinishSpinnerWithError() return errors.Wrap(err, "failed to restore kotsadm applications") @@ -208,7 +212,7 @@ func RestoreInstanceBackup(ctx context.Context, options RestoreInstanceBackupOpt if options.WaitForApps { // wait for applications restore to finish - err = waitForKotsadmApplicationsRestore(options.BackupName, kotsadmNamespace, kotsadmPodName, log) + err = waitForKotsadmApplicationsRestore(backup.Name, kotsadmNamespace, kotsadmPodName, log) if err != nil { if _, ok := errors.Cause(err).(*kotsadmtypes.ErrorAppsRestore); ok { log.FinishSpinnerWithError() @@ -282,7 +286,7 @@ func ListInstanceRestores(ctx context.Context, options ListInstanceRestoresOptio restores := []velerov1.Restore{} for _, restore := range r.Items { - if restore.Annotations["kots.io/instance"] != "true" { + if restore.Annotations[snapshottypes.InstanceBackupAnnotation] != "true" { continue } @@ -328,7 +332,7 @@ func waitForVeleroRestoreCompleted(ctx context.Context, veleroNamespace string, } } -func initiateKotsadmApplicationsRestore(backupName string, kotsadmNamespace string, kotsadmPodName string, log *logger.CLILogger) error { +func initiateKotsadmApplicationsRestore(backupID string, kotsadmNamespace string, kotsadmPodName string, log *logger.CLILogger) error { getPodName := func() (string, error) { return kotsadmPodName, nil } @@ -361,7 +365,7 @@ func initiateKotsadmApplicationsRestore(backupName string, kotsadmNamespace stri return errors.Wrap(err, "failed to get kotsadm auth slug") } - url := fmt.Sprintf("http://localhost:%d/api/v1/snapshot/%s/restore-apps", localPort, backupName) + url := fmt.Sprintf("http://localhost:%d/api/v1/snapshot/%s/restore-apps", localPort, backupID) requestPayload := map[string]interface{}{ "restoreAll": true, @@ -390,7 +394,7 @@ func initiateKotsadmApplicationsRestore(backupName string, kotsadmNamespace stri return nil } -func waitForKotsadmApplicationsRestore(backupName string, kotsadmNamespace string, kotsadmPodName string, log *logger.CLILogger) error { +func waitForKotsadmApplicationsRestore(backupID string, kotsadmNamespace string, kotsadmPodName string, log *logger.CLILogger) error { getPodName := func() (string, error) { return kotsadmPodName, nil } @@ -423,7 +427,7 @@ func waitForKotsadmApplicationsRestore(backupName string, kotsadmNamespace strin return errors.Wrap(err, "failed to get kotsadm auth slug") } - url := fmt.Sprintf("http://localhost:%d/api/v1/snapshot/%s/apps-restore-status", localPort, backupName) + url := fmt.Sprintf("http://localhost:%d/api/v1/snapshot/%s/apps-restore-status", localPort, backupID) for { requestPayload := map[string]interface{}{ @@ -501,3 +505,11 @@ func waitForKotsadmApplicationsRestore(backupName string, kotsadmNamespace strin time.Sleep(time.Second * 2) } } + +// getInstanceBackupType returns the type of the backup from the velero backup object annotation. +func getInstanceBackupType(veleroBackup velerov1.Backup) string { + if val, ok := veleroBackup.GetAnnotations()[snapshottypes.InstanceBackupTypeAnnotation]; ok { + return val + } + return snapshottypes.InstanceBackupTypeLegacy +} diff --git a/pkg/snapshotscheduler/snapshotscheduler.go b/pkg/snapshotscheduler/snapshotscheduler.go index f518aacf20..9f79552c57 100644 --- a/pkg/snapshotscheduler/snapshotscheduler.go +++ b/pkg/snapshotscheduler/snapshotscheduler.go @@ -203,15 +203,15 @@ func handleCluster(c *downstreamtypes.Downstream) error { return nil } - backup, err := snapshot.CreateInstanceBackup(context.Background(), c, true) + backupName, err := snapshot.CreateInstanceBackup(context.Background(), c, true) if err != nil { return errors.Wrap(err, "failed to create instance backup") } - if err := store.GetStore().UpdateScheduledInstanceSnapshot(next.ID, backup.ObjectMeta.Name); err != nil { + if err := store.GetStore().UpdateScheduledInstanceSnapshot(next.ID, backupName); err != nil { return errors.Wrap(err, "failed to update scheduled instance snapshot") } - logger.Infof("Created instance backup %s from scheduled instance snapshot %s", backup.ObjectMeta.Name, next.ID) + logger.Infof("Created instance backup %s from scheduled instance snapshot %s", backupName, next.ID) if len(pending) > 1 { err := store.GetStore().DeletePendingScheduledInstanceSnapshots(c.ClusterID) diff --git a/pkg/store/kotsstore/version_store.go b/pkg/store/kotsstore/version_store.go index cc6f41ab02..eca2791be0 100644 --- a/pkg/store/kotsstore/version_store.go +++ b/pkg/store/kotsstore/version_store.go @@ -614,6 +614,10 @@ func (s *KOTSStore) upsertAppVersionRecordStatements(appID string, sequence int6 if err != nil { return nil, errors.Wrap(err, "failed to marshal backup spec") } + restoreSpec, err := kotsKinds.Marshal("velero.io", "v1", "Restore") + if err != nil { + return nil, errors.Wrap(err, "failed to marshal restore spec") + } identitySpec, err := kotsKinds.Marshal("kots.io", "v1beta1", "Identity") if err != nil { return nil, errors.Wrap(err, "failed to marshal identity spec") @@ -644,8 +648,8 @@ func (s *KOTSStore) upsertAppVersionRecordStatements(appID string, sequence int6 } query := `insert into app_version (app_id, sequence, created_at, version_label, is_required, release_notes, update_cursor, channel_id, channel_name, upstream_released_at, encryption_key, - supportbundle_spec, analyzer_spec, preflight_spec, app_spec, kots_app_spec, kots_installation_spec, kots_license, config_spec, config_values, backup_spec, identity_spec, branding_archive, embeddedcluster_config) - values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + supportbundle_spec, analyzer_spec, preflight_spec, app_spec, kots_app_spec, kots_installation_spec, kots_license, config_spec, config_values, backup_spec, restore_spec, identity_spec, branding_archive, embeddedcluster_config) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(app_id, sequence) DO UPDATE SET created_at = EXCLUDED.created_at, version_label = EXCLUDED.version_label, @@ -666,6 +670,7 @@ func (s *KOTSStore) upsertAppVersionRecordStatements(appID string, sequence int6 config_spec = EXCLUDED.config_spec, config_values = EXCLUDED.config_values, backup_spec = EXCLUDED.backup_spec, + restore_spec = EXCLUDED.restore_spec, identity_spec = EXCLUDED.identity_spec, branding_archive = EXCLUDED.branding_archive, embeddedcluster_config = EXCLUDED.embeddedcluster_config` @@ -694,6 +699,7 @@ func (s *KOTSStore) upsertAppVersionRecordStatements(appID string, sequence int6 configSpec, configValuesSpec, backupSpec, + restoreSpec, identitySpec, base64.StdEncoding.EncodeToString(brandingArchive), embeddedClusterConfig,