From 7289482478c344090d65956d03d2e0a5e588781f Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Thu, 21 Nov 2024 12:22:26 -0800 Subject: [PATCH 01/44] feat(ec): improved disaster recovery --- pkg/handlers/backup.go | 4 +- pkg/kotsadmsnapshot/backup.go | 275 +++++++++++++++------ pkg/kotsadmsnapshot/restore.go | 3 + pkg/kotsutil/kots.go | 37 +-- pkg/snapshotscheduler/snapshotscheduler.go | 6 +- 5 files changed, 224 insertions(+), 101 deletions(-) diff --git a/pkg/handlers/backup.go b/pkg/handlers/backup.go index 2d850f2e1c..ba3713d36a 100644 --- a/pkg/handlers/backup.go +++ b/pkg/handlers/backup.go @@ -206,7 +206,7 @@ func (h *Handler) CreateInstanceBackup(w http.ResponseWriter, r *http.Request) { } c := clusters[0] - backup, err := snapshot.CreateInstanceBackup(context.TODO(), c, false) + backupName, err := snapshot.CreateInstanceBackup(context.TODO(), c, false) if err != nil { logger.Error(errors.Wrap(err, "failed to create instance snapshot")) createInstanceBackupResponse.Error = "failed to create instance backup" @@ -215,7 +215,7 @@ func (h *Handler) CreateInstanceBackup(w http.ResponseWriter, r *http.Request) { } createInstanceBackupResponse.Success = true - createInstanceBackupResponse.BackupName = backup.ObjectMeta.Name + createInstanceBackupResponse.BackupName = backupName JSON(w, http.StatusOK, createInstanceBackupResponse) } diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 4361ffaca1..6a234dd3fa 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" "math" "os" "strconv" @@ -39,6 +38,16 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + InstanceBackupNameAnnotation = "replicated.com/backup-name" + InstanceBackupTypeAnnotation = "replicated.com/backup-type" + InstanceBackupsExpectedAnnotation = "replicated.com/backups-expected" + + InstanceBackupTypeKotsadm = "kotsadm" + InstanceBackupTypeApp = "app" + InstanceBackupTypeCombined = "combined" +) + func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled bool) (*velerov1.Backup, error) { downstreams, err := store.GetStore().ListDownstreamsForApp(a.ID) if err != nil { @@ -191,25 +200,38 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b return backup, nil } -func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstream, isScheduled bool) (*velerov1.Backup, error) { +func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstream, isScheduled bool) (string, error) { logger.Debug("creating instance backup") cfg, err := k8sutil.GetClusterConfig() if err != nil { - return nil, errors.Wrap(err, "failed to get cluster config") + return "", errors.Wrap(err, "failed to get cluster config") } clientset, err := kubernetes.NewForConfig(cfg) if err != nil { - return nil, errors.Wrap(err, "failed to create clientset") + return "", errors.Wrap(err, "failed to create clientset") } - isKurl, err := kurl.IsKurl(clientset) + veleroClient, err := veleroclientv1.NewForConfig(cfg) if err != nil { - return nil, errors.Wrap(err, "failed to check if cluster is kurl") + return "", errors.Wrap(err, "failed to create velero clientset") } kotsadmNamespace := util.PodNamespace + + kotsadmVeleroBackendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) + if err != nil { + return "", errors.Wrap(err, "failed to find backupstoragelocations") + } + + if kotsadmVeleroBackendStorageLocation == nil { + return "", errors.New("no backup store location found") + } + + // appVeleroBackup is only set if usesImprovedDR is true + var appVeleroBackup *velerov1.Backup + appsSequences := map[string]int64{} appVersions := map[string]string{} includedNamespaces := []string{kotsadmNamespace} @@ -232,19 +254,23 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre includedNamespaces = append(includedNamespaces, appNamespace) } + isKurl, err := kurl.IsKurl(clientset) + if err != nil { + return "", errors.Wrap(err, "failed to check if cluster is kurl") + } if isKurl { includedNamespaces = append(includedNamespaces, "kurl") } apps, err := store.GetStore().ListInstalledApps() if err != nil { - return nil, errors.Wrap(err, "failed to list installed apps") + return "", errors.Wrap(err, "failed to list installed apps") } for _, a := range apps { downstreams, err := store.GetStore().ListDownstreamsForApp(a.ID) if err != nil { - return nil, errors.Wrapf(err, "failed to list downstreams for app %s", a.Slug) + return "", errors.Wrapf(err, "failed to list downstreams for app %s", a.Slug) } if len(downstreams) == 0 { @@ -254,84 +280,90 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre parentSequence, err := store.GetStore().GetCurrentParentSequence(a.ID, downstreams[0].ClusterID) if err != nil { - return nil, errors.Wrapf(err, "failed to get current downstream parent sequence for app %s", a.Slug) + return "", errors.Wrapf(err, "failed to get current downstream parent sequence for app %s", a.Slug) } if parentSequence == -1 { // no version is deployed for this app yet continue } - archiveDir, err := ioutil.TempDir("", "kotsadm") + archiveDir, err := os.MkdirTemp("", "kotsadm") if err != nil { - return nil, errors.Wrapf(err, "failed to create temp dir for app %s", a.Slug) + return "", errors.Wrapf(err, "failed to create temp dir for app %s", a.Slug) } defer os.RemoveAll(archiveDir) err = store.GetStore().GetAppVersionArchive(a.ID, parentSequence, archiveDir) if err != nil { - return nil, errors.Wrapf(err, "failed to get app version archive for app %s", a.Slug) + return "", errors.Wrapf(err, "failed to get app version archive for app %s", a.Slug) } kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { - return nil, errors.Wrap(err, "failed to load kots kinds from path") + return "", errors.Wrap(err, "failed to load kots kinds from path") } backupSpec, err := kotsKinds.Marshal("velero.io", "v1", "Backup") if err != nil { - return nil, errors.Wrap(err, "failed to get backup spec from kotskinds") + return "", errors.Wrap(err, "failed to get backup spec from kotskinds") } + var veleroBackup *velerov1.Backup if backupSpec == "" { - continue - } - - appsSequences[a.Slug] = parentSequence - appVersions[a.Slug] = kotsKinds.Installation.Spec.VersionLabel - - renderedBackup, err := helper.RenderAppFile(a, nil, []byte(backupSpec), kotsKinds, kotsadmNamespace) - if err != nil { - return nil, errors.Wrap(err, "failed to render backup") - } - veleroBackup, err := kotsutil.LoadBackupFromContents(renderedBackup) - if err != nil { - return nil, errors.Wrap(err, "failed to load backup from contents") - } - - // ** merge app backup info ** // - // included namespaces - includedNamespaces = append(includedNamespaces, veleroBackup.Spec.IncludedNamespaces...) - includedNamespaces = append(includedNamespaces, kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) + // If this is Embedded Cluster, backups are always enabled and we will use a default + // minimal backup spec + if util.IsEmbeddedCluster() { + veleroBackup = getDefaultEmbeddedClusterBackupSpec() + } else { + continue + } + } else { + renderedBackup, err := helper.RenderAppFile(a, nil, []byte(backupSpec), kotsKinds, kotsadmNamespace) + if err != nil { + return "", errors.Wrap(err, "failed to render backup") + } + veleroBackup, err = kotsutil.LoadBackupFromContents(renderedBackup) + if err != nil { + return "", errors.Wrap(err, "failed to load backup from contents") + } - // excluded namespaces - excludedNamespaces = append(excludedNamespaces, veleroBackup.Spec.ExcludedNamespaces...) + // If this is EC and it is using the new improved DR (there is a backup and restore custom + // resource), dont merge the backup spec. + if usesImprovedDR(kotsKinds) { + if len(apps) > 0 { + return "", errors.New("cannot create backup for Embedded Cluster with multiple apps") + } + appVeleroBackup = veleroBackup + appVeleroBackup.Name = "" + appVeleroBackup.GenerateName = a.Slug + "-" + appVeleroBackup.Namespace = kotsadmVeleroBackendStorageLocation.Namespace + } else { + + // ** merge app backup info ** // + // included namespaces + includedNamespaces = append(includedNamespaces, veleroBackup.Spec.IncludedNamespaces...) + includedNamespaces = append(includedNamespaces, kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) + + // excluded namespaces + excludedNamespaces = append(excludedNamespaces, veleroBackup.Spec.ExcludedNamespaces...) + + // annotations + for k, v := range veleroBackup.Annotations { + backupAnnotations[k] = v + } - // annotations - for k, v := range veleroBackup.Annotations { - backupAnnotations[k] = v - } + // ordered resources + for k, v := range veleroBackup.Spec.OrderedResources { + backupOrderedResources[k] = v + } - // ordered resources - for k, v := range veleroBackup.Spec.OrderedResources { - backupOrderedResources[k] = v + // backup hooks + backupHooks.Resources = append(backupHooks.Resources, veleroBackup.Spec.Hooks.Resources...) + } } - // backup hooks - backupHooks.Resources = append(backupHooks.Resources, veleroBackup.Spec.Hooks.Resources...) - } - - veleroClient, err := veleroclientv1.NewForConfig(cfg) - if err != nil { - return nil, errors.Wrap(err, "failed to create velero clientset") - } - - kotsadmVeleroBackendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) - if err != nil { - return nil, errors.Wrap(err, "failed to find backupstoragelocations") - } - - if kotsadmVeleroBackendStorageLocation == nil { - return nil, errors.New("no backup store location found") + appsSequences[a.Slug] = parentSequence + appVersions[a.Slug] = kotsKinds.Installation.Spec.VersionLabel } isKotsadmClusterScoped := k8sutil.IsKotsadmClusterScoped(ctx, clientset, kotsadmNamespace) @@ -343,7 +375,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre kotsadmImage, err := k8sutil.FindKotsadmImage(kotsadmNamespace) if err != nil { - return nil, errors.Wrap(err, "failed to find kotsadm image") + return "", errors.Wrap(err, "failed to find kotsadm image") } snapshotTrigger := "manual" @@ -354,39 +386,64 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre // marshal apps sequences map b, err := json.Marshal(appsSequences) if err != nil { - return nil, errors.Wrap(err, "failed to marshal apps sequences") + return "", errors.Wrap(err, "failed to marshal apps sequences") } marshalledAppsSequences := string(b) // marshal apps versions map b, err = json.Marshal(appVersions) if err != nil { - return nil, errors.Wrap(err, "failed to marshal apps versions") + return "", errors.Wrap(err, "failed to marshal apps versions") } marshalledAppVersions := string(b) - // add kots annotations - backupAnnotations["kots.io/snapshot-trigger"] = snapshotTrigger - backupAnnotations["kots.io/snapshot-requested"] = time.Now().UTC().Format(time.RFC3339) - backupAnnotations["kots.io/instance"] = "true" - backupAnnotations["kots.io/kotsadm-image"] = kotsadmImage - backupAnnotations["kots.io/kotsadm-deploy-namespace"] = kotsadmNamespace - backupAnnotations["kots.io/apps-sequences"] = marshalledAppsSequences - backupAnnotations["kots.io/apps-versions"] = marshalledAppVersions - backupAnnotations["kots.io/is-airgap"] = strconv.FormatBool(kotsadm.IsAirgap()) + now := time.Now() + backupName := fmt.Sprintf("backup-%d", now.UnixNano()) + + addCommonAnnotations := func(annotations map[string]string) map[string]string { + if annotations == nil { + annotations = make(map[string]string, 0) + } + annotations["kots.io/snapshot-trigger"] = snapshotTrigger + annotations["kots.io/snapshot-requested"] = now.UTC().Format(time.RFC3339) + annotations["kots.io/instance"] = "true" + annotations["kots.io/kotsadm-image"] = kotsadmImage + annotations["kots.io/kotsadm-deploy-namespace"] = kotsadmNamespace + annotations["kots.io/apps-sequences"] = marshalledAppsSequences + annotations["kots.io/apps-versions"] = marshalledAppVersions + annotations["kots.io/is-airgap"] = strconv.FormatBool(kotsadm.IsAirgap()) + + // Add improved disaster recovery annotation labels + annotations[InstanceBackupNameAnnotation] = backupName + if appVeleroBackup != nil { + annotations[InstanceBackupsExpectedAnnotation] = strconv.Itoa(2) + } else { + annotations[InstanceBackupsExpectedAnnotation] = strconv.Itoa(1) + } + return annotations + } + + backupAnnotations = addCommonAnnotations(backupAnnotations) + if appVeleroBackup != nil { + backupAnnotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeKotsadm + appVeleroBackup.Annotations = addCommonAnnotations(appVeleroBackup.Annotations) + appVeleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeApp + } else { + backupAnnotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeCombined + } if util.IsEmbeddedCluster() { kbClient, err := k8sutil.GetKubeClient(ctx) if err != nil { - return nil, fmt.Errorf("failed to get kubeclient: %w", err) + return "", fmt.Errorf("failed to get kubeclient: %w", err) } installation, err := embeddedcluster.GetCurrentInstallation(ctx, kbClient) if err != nil { - return nil, fmt.Errorf("failed to get current installation: %w", err) + return "", fmt.Errorf("failed to get current installation: %w", err) } ecAnnotations, err := ecBackupAnnotations(ctx, kbClient, installation) if err != nil { - return nil, fmt.Errorf("failed to get embedded cluster backup annotations: %w", err) + return "", fmt.Errorf("failed to get embedded cluster backup annotations: %w", err) } for k, v := range ecAnnotations { backupAnnotations[k] = v @@ -421,7 +478,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre if cluster.SnapshotTTL != "" { ttlDuration, err := time.ParseDuration(cluster.SnapshotTTL) if err != nil { - return nil, errors.Wrap(err, "failed to parse cluster snapshot ttl value as duration") + return "", errors.Wrap(err, "failed to parse cluster snapshot ttl value as duration") } veleroBackup.Spec.TTL = metav1.Duration{ Duration: ttlDuration, @@ -433,12 +490,67 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre logger.Error(errors.Wrap(err, "failed to exclude shutdown pods from backup")) } - backup, err := veleroClient.Backups(kotsadmVeleroBackendStorageLocation.Namespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) + _, err = veleroClient.Backups(kotsadmVeleroBackendStorageLocation.Namespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) if err != nil { - return nil, errors.Wrap(err, "failed to create velero backup") + return "", errors.Wrap(err, "failed to create velero backup") } - return backup, nil + if appVeleroBackup != nil { + _, err := veleroClient.Backups(kotsadmVeleroBackendStorageLocation.Namespace).Create(ctx, appVeleroBackup, metav1.CreateOptions{}) + if err != nil { + return "", errors.Wrap(err, "failed to create application velero backup") + } + } + + return backupName, nil +} + +func GetBackupName(veleroBackup velerov1.Backup) string { + if val, ok := veleroBackup.GetAnnotations()[InstanceBackupNameAnnotation]; ok { + return val + } + return veleroBackup.GetName() +} + +func GetInstanceBackupType(veleroBackup velerov1.Backup) string { + if val, ok := veleroBackup.GetAnnotations()[InstanceBackupTypeAnnotation]; ok { + return val + } + return "" +} + +func GetInstanceBackupsExpected(veleroBackup velerov1.Backup) int { + if val, ok := veleroBackup.GetAnnotations()[InstanceBackupsExpectedAnnotation]; ok { + num, _ := strconv.Atoi(val) + if num > 0 { + return num + } + } + return 1 +} + +// usesImprovedDR returns true if this is Embedded Cluster and the vendor has defined both a backup +// and restore custom resource. +func usesImprovedDR(kotsKinds *kotsutil.KotsKinds) bool { + if !util.IsEmbeddedCluster() { + return false + } + + return kotsKinds.Backup != nil && kotsKinds.Restore != nil +} + +// getDefaultEmbeddedClusterBackupSpec returns a default velero backup spec for an embedded +// cluster using the old DR (no backup and restore custom resource). +func getDefaultEmbeddedClusterBackupSpec() *velerov1.Backup { + return &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "backup", + }, + } } func ListBackupsForApp(ctx context.Context, kotsadmNamespace string, appID string) ([]*types.Backup, error) { @@ -612,8 +724,13 @@ func ListInstanceBackups(ctx context.Context, kotsadmNamespace string) ([]*types continue } + // TODO: support for improved DR in UI + if GetInstanceBackupType(veleroBackup) == InstanceBackupTypeApp { + continue + } + backup := types.Backup{ - Name: veleroBackup.Name, + Name: GetBackupName(veleroBackup), Status: string(veleroBackup.Status.Phase), IncludedApps: make([]types.App, 0), } @@ -895,7 +1012,7 @@ func GetBackupDetail(ctx context.Context, kotsadmNamespace string, backupName st } result := &types.BackupDetail{ - Name: backup.Name, + Name: GetBackupName(*backup), Status: string(backup.Status.Phase), Namespaces: backup.Spec.IncludedNamespaces, Volumes: listBackupVolumes(backupVolumes.Items), diff --git a/pkg/kotsadmsnapshot/restore.go b/pkg/kotsadmsnapshot/restore.go index 97c1d20f0c..71192d0d62 100644 --- a/pkg/kotsadmsnapshot/restore.go +++ b/pkg/kotsadmsnapshot/restore.go @@ -127,6 +127,9 @@ func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snap return errors.Wrap(err, "failed to create restore") } + // create the kotsKinds.Restore included with the yaml from the vendor + // Add the EC annotation + return nil } diff --git a/pkg/kotsutil/kots.go b/pkg/kotsutil/kots.go index 67b26214a1..5bb7e6adbe 100644 --- a/pkg/kotsutil/kots.go +++ b/pkg/kotsutil/kots.go @@ -107,6 +107,7 @@ type KotsKinds struct { IdentityConfig *kotsv1beta1.IdentityConfig Backup *velerov1.Backup + Restore *velerov1.Restore Installer *kurlv1beta1.Installer LintConfig *kotsv1beta1.LintConfig @@ -115,7 +116,7 @@ type KotsKinds struct { } func IsKotsKind(apiVersion string, kind string) bool { - if apiVersion == "velero.io/v1" && kind == "Backup" { + if apiVersion == "velero.io/v1" && (kind == "Backup" || kind == "Restore") { return true } if apiVersion == "kots.io/v1beta1" { @@ -439,22 +440,7 @@ func (o KotsKinds) Marshal(g string, v string, k string) (string, error) { if v == "v1" { if k == "Backup" { if o.Backup == nil { - if util.IsEmbeddedCluster() { - // return the default backup object - backup := &velerov1.Backup{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "velero.io/v1", - Kind: "Backup", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "backup", - }, - } - o.Backup = backup - } else { - return "", nil - } - + return "", nil } var b bytes.Buffer if err := s.Encode(o.Backup, &b); err != nil { @@ -567,6 +553,8 @@ func (k *KotsKinds) addKotsKinds(content []byte) error { k.HostPreflight = decoded.(*troubleshootv1beta2.HostPreflight) case "velero.io/v1, Kind=Backup": k.Backup = decoded.(*velerov1.Backup) + case "velero.io/v1, Kind=Restore": + k.Restore = decoded.(*velerov1.Restore) case "kurl.sh/v1beta1, Kind=Installer", "cluster.kurl.sh/v1beta1, Kind=Installer": k.Installer = decoded.(*kurlv1beta1.Installer) case "app.k8s.io/v1beta1, Kind=Application": @@ -1085,6 +1073,21 @@ func LoadBackupFromContents(content []byte) (*velerov1.Backup, error) { return obj.(*velerov1.Backup), nil } +func LoadRestoreFromContents(content []byte) (*velerov1.Restore, error) { + decode := scheme.Codecs.UniversalDeserializer().Decode + + obj, gvk, err := decode(content, nil, nil) + if err != nil { + return nil, errors.Wrapf(err, "failed to decode: %v", string(content)) + } + + if gvk.String() != "velero.io/v1, Kind=Restore" { + return nil, errors.Errorf("unexpected gvk: %s", gvk.String()) + } + + return obj.(*velerov1.Restore), nil +} + func LoadApplicationFromContents(content []byte) (*applicationv1beta1.Application, error) { decode := scheme.Codecs.UniversalDeserializer().Decode diff --git a/pkg/snapshotscheduler/snapshotscheduler.go b/pkg/snapshotscheduler/snapshotscheduler.go index f518aacf20..9f79552c57 100644 --- a/pkg/snapshotscheduler/snapshotscheduler.go +++ b/pkg/snapshotscheduler/snapshotscheduler.go @@ -203,15 +203,15 @@ func handleCluster(c *downstreamtypes.Downstream) error { return nil } - backup, err := snapshot.CreateInstanceBackup(context.Background(), c, true) + backupName, err := snapshot.CreateInstanceBackup(context.Background(), c, true) if err != nil { return errors.Wrap(err, "failed to create instance backup") } - if err := store.GetStore().UpdateScheduledInstanceSnapshot(next.ID, backup.ObjectMeta.Name); err != nil { + if err := store.GetStore().UpdateScheduledInstanceSnapshot(next.ID, backupName); err != nil { return errors.Wrap(err, "failed to update scheduled instance snapshot") } - logger.Infof("Created instance backup %s from scheduled instance snapshot %s", backup.ObjectMeta.Name, next.ID) + logger.Infof("Created instance backup %s from scheduled instance snapshot %s", backupName, next.ID) if len(pending) > 1 { err := store.GetStore().DeletePendingScheduledInstanceSnapshots(c.ClusterID) From 09ae90b0f2aeb1dba9019d75f40bee6081b729ca Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 2 Dec 2024 13:27:51 -0800 Subject: [PATCH 02/44] app version record restore spec --- deploy/assets/postgres/tables/app_version.yaml | 2 ++ pkg/store/kotsstore/version_store.go | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/deploy/assets/postgres/tables/app_version.yaml b/deploy/assets/postgres/tables/app_version.yaml index 32c488180a..a3f95182fb 100644 --- a/deploy/assets/postgres/tables/app_version.yaml +++ b/deploy/assets/postgres/tables/app_version.yaml @@ -66,6 +66,8 @@ spec: type: text - name: backup_spec type: text + - name: restore_spec + type: text - name: identity_spec type: text - name: branding_archive diff --git a/pkg/store/kotsstore/version_store.go b/pkg/store/kotsstore/version_store.go index cc6f41ab02..ea16f66cc2 100644 --- a/pkg/store/kotsstore/version_store.go +++ b/pkg/store/kotsstore/version_store.go @@ -614,6 +614,10 @@ func (s *KOTSStore) upsertAppVersionRecordStatements(appID string, sequence int6 if err != nil { return nil, errors.Wrap(err, "failed to marshal backup spec") } + restoreSpec, err := kotsKinds.Marshal("velero.io", "v1", "Restore") + if err != nil { + return nil, errors.Wrap(err, "failed to marshal backup spec") + } identitySpec, err := kotsKinds.Marshal("kots.io", "v1beta1", "Identity") if err != nil { return nil, errors.Wrap(err, "failed to marshal identity spec") @@ -644,8 +648,8 @@ func (s *KOTSStore) upsertAppVersionRecordStatements(appID string, sequence int6 } query := `insert into app_version (app_id, sequence, created_at, version_label, is_required, release_notes, update_cursor, channel_id, channel_name, upstream_released_at, encryption_key, - supportbundle_spec, analyzer_spec, preflight_spec, app_spec, kots_app_spec, kots_installation_spec, kots_license, config_spec, config_values, backup_spec, identity_spec, branding_archive, embeddedcluster_config) - values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + supportbundle_spec, analyzer_spec, preflight_spec, app_spec, kots_app_spec, kots_installation_spec, kots_license, config_spec, config_values, backup_spec, restore_spec, identity_spec, branding_archive, embeddedcluster_config) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(app_id, sequence) DO UPDATE SET created_at = EXCLUDED.created_at, version_label = EXCLUDED.version_label, @@ -666,6 +670,7 @@ func (s *KOTSStore) upsertAppVersionRecordStatements(appID string, sequence int6 config_spec = EXCLUDED.config_spec, config_values = EXCLUDED.config_values, backup_spec = EXCLUDED.backup_spec, + restore_spec = EXCLUDED.restore_spec, identity_spec = EXCLUDED.identity_spec, branding_archive = EXCLUDED.branding_archive, embeddedcluster_config = EXCLUDED.embeddedcluster_config` @@ -694,6 +699,7 @@ func (s *KOTSStore) upsertAppVersionRecordStatements(appID string, sequence int6 configSpec, configValuesSpec, backupSpec, + restoreSpec, identitySpec, base64.StdEncoding.EncodeToString(brandingArchive), embeddedClusterConfig, From 2fd3812e202c656aac6b51ef06e0265487537621 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 2 Dec 2024 13:28:27 -0800 Subject: [PATCH 03/44] fix tests --- pkg/kotsutil/kots_test.go | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/pkg/kotsutil/kots_test.go b/pkg/kotsutil/kots_test.go index db580d103f..44e5fe512e 100644 --- a/pkg/kotsutil/kots_test.go +++ b/pkg/kotsutil/kots_test.go @@ -1051,18 +1051,7 @@ status: {} v: "v1", k: "Backup", }, - want: `apiVersion: velero.io/v1 -kind: Backup -metadata: - creationTimestamp: null - name: backup -spec: - csiSnapshotTimeout: 0s - hooks: {} - metadata: {} - ttl: 0s -status: {} -`, + want: "", }, { name: "backup exists, EC", From 7e0b853350a967cdad3cf7b7afb618a2ea885732 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 2 Dec 2024 13:41:19 -0800 Subject: [PATCH 04/44] refactor --- pkg/kotsadmsnapshot/backup.go | 94 ++++++++++++++++------------------- 1 file changed, 44 insertions(+), 50 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 6a234dd3fa..b01eaaefea 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -229,29 +230,42 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return "", errors.New("no backup store location found") } - // appVeleroBackup is only set if usesImprovedDR is true - var appVeleroBackup *velerov1.Backup - - appsSequences := map[string]int64{} - appVersions := map[string]string{} - includedNamespaces := []string{kotsadmNamespace} - excludedNamespaces := []string{} - backupAnnotations := map[string]string{} - backupOrderedResources := map[string]string{} - backupHooks := velerov1.BackupHooks{ - Resources: []velerov1.BackupResourceHookSpec{}, + // veleroBackup is the kotsadm backup or combined backup if usesImprovedDR is false + veleroBackup := &velerov1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{}, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{}, + ExcludedNamespaces: []string{}, + IncludeClusterResources: ptr.To(true), + OrLabelSelectors: instanceBackupLabelSelectors(util.IsEmbeddedCluster()), + OrderedResources: map[string]string{}, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{}, + }, + }, } // non-supported fields that are intentionally left out cuz they might break full snapshots: // - includedResources // - excludedResources // - labelSelector + // appVeleroBackup is only set if usesImprovedDR is true + var appVeleroBackup *velerov1.Backup + + appsSequences := map[string]int64{} + appVersions := map[string]string{} + appNamespace := kotsadmNamespace if os.Getenv("KOTSADM_TARGET_NAMESPACE") != "" { appNamespace = os.Getenv("KOTSADM_TARGET_NAMESPACE") } if appNamespace != kotsadmNamespace { - includedNamespaces = append(includedNamespaces, appNamespace) + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, appNamespace) } isKurl, err := kurl.IsKurl(clientset) @@ -259,7 +273,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return "", errors.Wrap(err, "failed to check if cluster is kurl") } if isKurl { - includedNamespaces = append(includedNamespaces, "kurl") + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, "kurl") } apps, err := store.GetStore().ListInstalledApps() @@ -308,7 +322,6 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return "", errors.Wrap(err, "failed to get backup spec from kotskinds") } - var veleroBackup *velerov1.Backup if backupSpec == "" { // If this is Embedded Cluster, backups are always enabled and we will use a default // minimal backup spec @@ -322,7 +335,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre if err != nil { return "", errors.Wrap(err, "failed to render backup") } - veleroBackup, err = kotsutil.LoadBackupFromContents(renderedBackup) + kotskindsBackup, err := kotsutil.LoadBackupFromContents(renderedBackup) if err != nil { return "", errors.Wrap(err, "failed to load backup from contents") } @@ -333,32 +346,30 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre if len(apps) > 0 { return "", errors.New("cannot create backup for Embedded Cluster with multiple apps") } - appVeleroBackup = veleroBackup + appVeleroBackup = kotskindsBackup appVeleroBackup.Name = "" appVeleroBackup.GenerateName = a.Slug + "-" - appVeleroBackup.Namespace = kotsadmVeleroBackendStorageLocation.Namespace } else { - // ** merge app backup info ** // // included namespaces - includedNamespaces = append(includedNamespaces, veleroBackup.Spec.IncludedNamespaces...) - includedNamespaces = append(includedNamespaces, kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, kotskindsBackup.Spec.IncludedNamespaces...) + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) // excluded namespaces - excludedNamespaces = append(excludedNamespaces, veleroBackup.Spec.ExcludedNamespaces...) + veleroBackup.Spec.ExcludedNamespaces = append(veleroBackup.Spec.ExcludedNamespaces, kotskindsBackup.Spec.ExcludedNamespaces...) // annotations - for k, v := range veleroBackup.Annotations { - backupAnnotations[k] = v + for k, v := range kotskindsBackup.Annotations { + veleroBackup.Annotations[k] = v } // ordered resources - for k, v := range veleroBackup.Spec.OrderedResources { - backupOrderedResources[k] = v + for k, v := range kotskindsBackup.Spec.OrderedResources { + veleroBackup.Spec.OrderedResources[k] = v } // backup hooks - backupHooks.Resources = append(backupHooks.Resources, veleroBackup.Spec.Hooks.Resources...) + veleroBackup.Spec.Hooks.Resources = append(veleroBackup.Spec.Hooks.Resources, kotskindsBackup.Spec.Hooks.Resources...) } } @@ -370,7 +381,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre if !isKotsadmClusterScoped { // in minimal rbac, a kotsadm role and rolebinding will exist in the velero namespace to give kotsadm access to velero. // we backup and restore those so that restoring to a new cluster won't require that the user provide those permissions again. - includedNamespaces = append(includedNamespaces, kotsadmVeleroBackendStorageLocation.Namespace) + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, kotsadmVeleroBackendStorageLocation.Namespace) } kotsadmImage, err := k8sutil.FindKotsadmImage(kotsadmNamespace) @@ -423,13 +434,13 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return annotations } - backupAnnotations = addCommonAnnotations(backupAnnotations) + veleroBackup.Annotations = addCommonAnnotations(veleroBackup.Annotations) if appVeleroBackup != nil { - backupAnnotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeKotsadm + veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeKotsadm appVeleroBackup.Annotations = addCommonAnnotations(appVeleroBackup.Annotations) appVeleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeApp } else { - backupAnnotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeCombined + veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeCombined } if util.IsEmbeddedCluster() { @@ -446,29 +457,12 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return "", fmt.Errorf("failed to get embedded cluster backup annotations: %w", err) } for k, v := range ecAnnotations { - backupAnnotations[k] = v + veleroBackup.Annotations[k] = v } - includedNamespaces = append(includedNamespaces, ecIncludedNamespaces(installation)...) + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, ecIncludedNamespaces(installation)...) } - includeClusterResources := true - veleroBackup := &velerov1.Backup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "", - GenerateName: "instance-", - Namespace: kotsadmVeleroBackendStorageLocation.Namespace, - Annotations: backupAnnotations, - }, - Spec: velerov1.BackupSpec{ - StorageLocation: "default", - IncludedNamespaces: prepareIncludedNamespaces(includedNamespaces), - ExcludedNamespaces: excludedNamespaces, - IncludeClusterResources: &includeClusterResources, - OrLabelSelectors: instanceBackupLabelSelectors(util.IsEmbeddedCluster()), - OrderedResources: backupOrderedResources, - Hooks: backupHooks, - }, - } + veleroBackup.Spec.IncludedNamespaces = prepareIncludedNamespaces(veleroBackup.Spec.IncludedNamespaces) embeddedRegistryHost, _, _ := kotsutil.GetEmbeddedRegistryCreds(clientset) if embeddedRegistryHost != "" { From c92d9ca0e64e7aa0668ab3d0fb93b0ab1870164c Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 2 Dec 2024 13:47:36 -0800 Subject: [PATCH 05/44] refactor --- pkg/kotsadmsnapshot/backup.go | 50 ++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index b01eaaefea..0f7574259e 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -249,10 +249,6 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre }, }, } - // non-supported fields that are intentionally left out cuz they might break full snapshots: - // - includedResources - // - excludedResources - // - labelSelector // appVeleroBackup is only set if usesImprovedDR is true var appVeleroBackup *velerov1.Backup @@ -351,25 +347,8 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre appVeleroBackup.GenerateName = a.Slug + "-" } else { // ** merge app backup info ** // - // included namespaces - veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, kotskindsBackup.Spec.IncludedNamespaces...) veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) - - // excluded namespaces - veleroBackup.Spec.ExcludedNamespaces = append(veleroBackup.Spec.ExcludedNamespaces, kotskindsBackup.Spec.ExcludedNamespaces...) - - // annotations - for k, v := range kotskindsBackup.Annotations { - veleroBackup.Annotations[k] = v - } - - // ordered resources - for k, v := range kotskindsBackup.Spec.OrderedResources { - veleroBackup.Spec.OrderedResources[k] = v - } - - // backup hooks - veleroBackup.Spec.Hooks.Resources = append(veleroBackup.Spec.Hooks.Resources, kotskindsBackup.Spec.Hooks.Resources...) + mergeAppBackupSpec(veleroBackup, kotskindsBackup) } } @@ -547,6 +526,33 @@ func getDefaultEmbeddedClusterBackupSpec() *velerov1.Backup { } } +// mergeAppBackupSpec merges the app backup spec into the velero backup spec when improved DR is +// disabled. Unsupported fields that are intentionally left out because they might break full +// snapshots: +// - includedResources +// - excludedResources +// - labelSelector +func mergeAppBackupSpec(backup, appBackup *velerov1.Backup) { + // included namespaces + backup.Spec.IncludedNamespaces = append(backup.Spec.IncludedNamespaces, appBackup.Spec.IncludedNamespaces...) + + // excluded namespaces + backup.Spec.ExcludedNamespaces = append(backup.Spec.ExcludedNamespaces, appBackup.Spec.ExcludedNamespaces...) + + // annotations + for k, v := range appBackup.Annotations { + backup.Annotations[k] = v + } + + // ordered resources + for k, v := range appBackup.Spec.OrderedResources { + backup.Spec.OrderedResources[k] = v + } + + // backup hooks + backup.Spec.Hooks.Resources = append(backup.Spec.Hooks.Resources, appBackup.Spec.Hooks.Resources...) +} + func ListBackupsForApp(ctx context.Context, kotsadmNamespace string, appID string) ([]*types.Backup, error) { cfg, err := k8sutil.GetClusterConfig() if err != nil { From 96625f88468df2b728eec4a1689025f55c4a0b0b Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 2 Dec 2024 14:26:02 -0800 Subject: [PATCH 06/44] refactor --- pkg/kotsadmsnapshot/backup.go | 91 +++++++++++++++++++++-------------- 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 0f7574259e..e10fc5ff89 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -253,7 +253,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre // appVeleroBackup is only set if usesImprovedDR is true var appVeleroBackup *velerov1.Backup - appsSequences := map[string]int64{} + appSequences := map[string]int64{} appVersions := map[string]string{} appNamespace := kotsadmNamespace @@ -272,6 +272,13 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, "kurl") } + isKotsadmClusterScoped := k8sutil.IsKotsadmClusterScoped(ctx, clientset, kotsadmNamespace) + if !isKotsadmClusterScoped { + // in minimal rbac, a kotsadm role and rolebinding will exist in the velero namespace to give kotsadm access to velero. + // we backup and restore those so that restoring to a new cluster won't require that the user provide those permissions again. + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, kotsadmVeleroBackendStorageLocation.Namespace) + } + apps, err := store.GetStore().ListInstalledApps() if err != nil { return "", errors.Wrap(err, "failed to list installed apps") @@ -352,17 +359,10 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre } } - appsSequences[a.Slug] = parentSequence + appSequences[a.Slug] = parentSequence appVersions[a.Slug] = kotsKinds.Installation.Spec.VersionLabel } - isKotsadmClusterScoped := k8sutil.IsKotsadmClusterScoped(ctx, clientset, kotsadmNamespace) - if !isKotsadmClusterScoped { - // in minimal rbac, a kotsadm role and rolebinding will exist in the velero namespace to give kotsadm access to velero. - // we backup and restore those so that restoring to a new cluster won't require that the user provide those permissions again. - veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, kotsadmVeleroBackendStorageLocation.Namespace) - } - kotsadmImage, err := k8sutil.FindKotsadmImage(kotsadmNamespace) if err != nil { return "", errors.Wrap(err, "failed to find kotsadm image") @@ -374,11 +374,11 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre } // marshal apps sequences map - b, err := json.Marshal(appsSequences) + b, err := json.Marshal(appSequences) if err != nil { return "", errors.Wrap(err, "failed to marshal apps sequences") } - marshalledAppsSequences := string(b) + marshalledAppSequences := string(b) // marshal apps versions map b, err = json.Marshal(appVersions) @@ -390,7 +390,12 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre now := time.Now() backupName := fmt.Sprintf("backup-%d", now.UnixNano()) - addCommonAnnotations := func(annotations map[string]string) map[string]string { + numBackups := 1 + if appVeleroBackup != nil { + numBackups = 2 + } + + appendCommonAnnotations := func(annotations map[string]string) map[string]string { if annotations == nil { annotations = make(map[string]string, 0) } @@ -399,24 +404,25 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre annotations["kots.io/instance"] = "true" annotations["kots.io/kotsadm-image"] = kotsadmImage annotations["kots.io/kotsadm-deploy-namespace"] = kotsadmNamespace - annotations["kots.io/apps-sequences"] = marshalledAppsSequences + annotations["kots.io/apps-sequences"] = marshalledAppSequences annotations["kots.io/apps-versions"] = marshalledAppVersions annotations["kots.io/is-airgap"] = strconv.FormatBool(kotsadm.IsAirgap()) + embeddedRegistryHost, _, _ := kotsutil.GetEmbeddedRegistryCreds(clientset) + if embeddedRegistryHost != "" { + annotations["kots.io/embedded-registry"] = embeddedRegistryHost + } // Add improved disaster recovery annotation labels annotations[InstanceBackupNameAnnotation] = backupName - if appVeleroBackup != nil { - annotations[InstanceBackupsExpectedAnnotation] = strconv.Itoa(2) - } else { - annotations[InstanceBackupsExpectedAnnotation] = strconv.Itoa(1) - } + annotations[InstanceBackupsExpectedAnnotation] = strconv.Itoa(numBackups) + return annotations } - veleroBackup.Annotations = addCommonAnnotations(veleroBackup.Annotations) + veleroBackup.Annotations = appendCommonAnnotations(veleroBackup.Annotations) if appVeleroBackup != nil { veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeKotsadm - appVeleroBackup.Annotations = addCommonAnnotations(appVeleroBackup.Annotations) + appVeleroBackup.Annotations = appendCommonAnnotations(appVeleroBackup.Annotations) appVeleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeApp } else { veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeCombined @@ -431,21 +437,19 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre if err != nil { return "", fmt.Errorf("failed to get current installation: %w", err) } - ecAnnotations, err := ecBackupAnnotations(ctx, kbClient, installation) + + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, ecIncludedNamespaces(installation)...) + + appVeleroBackup.Annotations, err = appendECAnnotations(ctx, appVeleroBackup.Annotations, kbClient, installation) if err != nil { - return "", fmt.Errorf("failed to get embedded cluster backup annotations: %w", err) + return "", errors.Wrap(err, "failed to add annotations to backup for embedded cluster") } - for k, v := range ecAnnotations { - veleroBackup.Annotations[k] = v + if appVeleroBackup != nil { + appVeleroBackup.Annotations, err = appendECAnnotations(ctx, appVeleroBackup.Annotations, kbClient, installation) + if err != nil { + return "", errors.Wrap(err, "failed to add annotations to application backup for embedded cluster") + } } - veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, ecIncludedNamespaces(installation)...) - } - - veleroBackup.Spec.IncludedNamespaces = prepareIncludedNamespaces(veleroBackup.Spec.IncludedNamespaces) - - embeddedRegistryHost, _, _ := kotsutil.GetEmbeddedRegistryCreds(clientset) - if embeddedRegistryHost != "" { - veleroBackup.ObjectMeta.Annotations["kots.io/embedded-registry"] = embeddedRegistryHost } if cluster.SnapshotTTL != "" { @@ -456,12 +460,25 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre veleroBackup.Spec.TTL = metav1.Duration{ Duration: ttlDuration, } + if appVeleroBackup != nil { + appVeleroBackup.Spec.TTL = metav1.Duration{ + Duration: ttlDuration, + } + } } err = excludeShutdownPodsFromBackup(ctx, clientset, veleroBackup) if err != nil { logger.Error(errors.Wrap(err, "failed to exclude shutdown pods from backup")) } + if appVeleroBackup != nil { + err = excludeShutdownPodsFromBackup(ctx, clientset, appVeleroBackup) + if err != nil { + logger.Error(errors.Wrap(err, "failed to exclude shutdown pods from application backup")) + } + } + + veleroBackup.Spec.IncludedNamespaces = prepareIncludedNamespaces(veleroBackup.Spec.IncludedNamespaces) _, err = veleroClient.Backups(kotsadmVeleroBackendStorageLocation.Namespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) if err != nil { @@ -1097,13 +1114,15 @@ func mergeLabelSelector(kots metav1.LabelSelector, app metav1.LabelSelector) met return kots } -// ecBackupAnnotations returns the annotations that should be added to an embedded cluster backup -func ecBackupAnnotations(ctx context.Context, kbClient kbclient.Client, in *embeddedclusterv1beta1.Installation) (map[string]string, error) { - annotations := map[string]string{} +// appendECAnnotations appends annotations that should be added to an embedded cluster backup +func appendECAnnotations(ctx context.Context, annotations map[string]string, kbClient kbclient.Client, in *embeddedclusterv1beta1.Installation) (map[string]string, error) { + if annotations == nil { + annotations = make(map[string]string, 0) + } seaweedFSS3ServiceIP, err := embeddedcluster.GetSeaweedFSS3ServiceIP(ctx, kbClient) if err != nil { - return nil, fmt.Errorf("failed to get seaweedfs s3 service ip: %w", err) + return annotations, fmt.Errorf("failed to get seaweedfs s3 service ip: %w", err) } if seaweedFSS3ServiceIP != "" { annotations["kots.io/embedded-cluster-seaweedfs-s3-ip"] = seaweedFSS3ServiceIP From c76341178a1624da430ce6cecfbeedb011270394 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 2 Dec 2024 14:28:47 -0800 Subject: [PATCH 07/44] refactor --- pkg/kotsadmsnapshot/backup.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index e10fc5ff89..e484ea7ee5 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -291,7 +291,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre } if len(downstreams) == 0 { - logger.Error(errors.Wrapf(err, "no downstreams found for app %s", a.Slug)) + logger.Errorf("No downstreams found for app %s", a.Slug) continue } @@ -469,12 +469,12 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre err = excludeShutdownPodsFromBackup(ctx, clientset, veleroBackup) if err != nil { - logger.Error(errors.Wrap(err, "failed to exclude shutdown pods from backup")) + logger.Errorf("Failed to exclude shutdown pods from backup: %v", err) } if appVeleroBackup != nil { err = excludeShutdownPodsFromBackup(ctx, clientset, appVeleroBackup) if err != nil { - logger.Error(errors.Wrap(err, "failed to exclude shutdown pods from application backup")) + logger.Errorf("Failed to exclude shutdown pods from application backup: %v", err) } } From 83f1002266ada90213d682eaacdd34eee923f7ea Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 2 Dec 2024 14:51:50 -0800 Subject: [PATCH 08/44] f --- pkg/kotsadmsnapshot/backup_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 0e3af9aa5f..cddc709231 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -682,20 +682,24 @@ func Test_instanceBackupLabelSelectors(t *testing.T) { } } -func Test_ecBackupAnnotations(t *testing.T) { +func Test_appendECAnnotations(t *testing.T) { scheme := runtime.NewScheme() corev1.AddToScheme(scheme) embeddedclusterv1beta1.AddToScheme(scheme) tests := []struct { name string + prev map[string]string kbClient kbclient.Client in *embeddedclusterv1beta1.Installation env map[string]string want map[string]string }{ { - name: "basic", + name: "basic", + prev: map[string]string{ + "prev-key": "prev-value", + }, kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).Build(), in: &embeddedclusterv1beta1.Installation{}, env: map[string]string{ @@ -703,6 +707,7 @@ func Test_ecBackupAnnotations(t *testing.T) { "EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version", }, want: map[string]string{ + "prev-key": "prev-value", "kots.io/embedded-cluster": "true", "kots.io/embedded-cluster-id": "embedded-cluster-id", "kots.io/embedded-cluster-version": "embedded-cluster-version", @@ -790,7 +795,7 @@ func Test_ecBackupAnnotations(t *testing.T) { for k, v := range tt.env { t.Setenv(k, v) } - got, err := ecBackupAnnotations(context.TODO(), tt.kbClient, tt.in) + got, err := appendECAnnotations(context.TODO(), tt.prev, tt.kbClient, tt.in) req.NoError(err) req.Equal(tt.want, got) }) From a0356b68632b59f7089375b60b2592c0116c9e72 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 2 Dec 2024 15:10:29 -0800 Subject: [PATCH 09/44] f --- pkg/kotsadmsnapshot/backup.go | 197 +++++++++++++++++++++++----------- 1 file changed, 135 insertions(+), 62 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index e484ea7ee5..babb3ae818 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -250,9 +250,6 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre }, } - // appVeleroBackup is only set if usesImprovedDR is true - var appVeleroBackup *velerov1.Backup - appSequences := map[string]int64{} appVersions := map[string]string{} @@ -284,6 +281,11 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return "", errors.Wrap(err, "failed to list installed apps") } + appVeleroBackup, err := getAppInstanceBackup(apps, kotsadmNamespace) + if err != nil { + return "", errors.Wrap(err, "failed to get app instance backup") + } + for _, a := range apps { downstreams, err := store.GetStore().ListDownstreamsForApp(a.ID) if err != nil { @@ -308,7 +310,9 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre if err != nil { return "", errors.Wrapf(err, "failed to create temp dir for app %s", a.Slug) } - defer os.RemoveAll(archiveDir) + defer func() { + _ = os.RemoveAll(archiveDir) + }() err = store.GetStore().GetAppVersionArchive(a.ID, parentSequence, archiveDir) if err != nil { @@ -320,47 +324,19 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return "", errors.Wrap(err, "failed to load kots kinds from path") } - backupSpec, err := kotsKinds.Marshal("velero.io", "v1", "Backup") - if err != nil { - return "", errors.Wrap(err, "failed to get backup spec from kotskinds") - } - - if backupSpec == "" { - // If this is Embedded Cluster, backups are always enabled and we will use a default - // minimal backup spec - if util.IsEmbeddedCluster() { - veleroBackup = getDefaultEmbeddedClusterBackupSpec() - } else { - continue - } - } else { - renderedBackup, err := helper.RenderAppFile(a, nil, []byte(backupSpec), kotsKinds, kotsadmNamespace) - if err != nil { - return "", errors.Wrap(err, "failed to render backup") - } - kotskindsBackup, err := kotsutil.LoadBackupFromContents(renderedBackup) + // Don't merge the backup spec if we are using the new improved DR. + if appVeleroBackup == nil { + err := mergeAppBackupSpec(veleroBackup, kotsKinds, kotsadmNamespace) if err != nil { - return "", errors.Wrap(err, "failed to load backup from contents") - } - - // If this is EC and it is using the new improved DR (there is a backup and restore custom - // resource), dont merge the backup spec. - if usesImprovedDR(kotsKinds) { - if len(apps) > 0 { - return "", errors.New("cannot create backup for Embedded Cluster with multiple apps") - } - appVeleroBackup = kotskindsBackup - appVeleroBackup.Name = "" - appVeleroBackup.GenerateName = a.Slug + "-" - } else { - // ** merge app backup info ** // - veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) - mergeAppBackupSpec(veleroBackup, kotskindsBackup) + return "", errors.Wrap(err, "failed to merge app backup spec") } } appSequences[a.Slug] = parentSequence appVersions[a.Slug] = kotsKinds.Installation.Spec.VersionLabel + + // optimization as we no longer need the archive dir + _ = os.RemoveAll(archiveDir) } kotsadmImage, err := k8sutil.FindKotsadmImage(kotsadmNamespace) @@ -519,28 +495,83 @@ func GetInstanceBackupsExpected(veleroBackup velerov1.Backup) int { return 1 } -// usesImprovedDR returns true if this is Embedded Cluster and the vendor has defined both a backup -// and restore custom resource. -func usesImprovedDR(kotsKinds *kotsutil.KotsKinds) bool { +// getAppInstanceBackup returns a backup spec only if this is Embedded Cluster and the vendor has +// defined both a backup and restore custom resource. +func getAppInstanceBackup(apps []*apptypes.App, kotsadmNamespace string) (*velerov1.Backup, error) { if !util.IsEmbeddedCluster() { - return false + return nil, nil } - return kotsKinds.Backup != nil && kotsKinds.Restore != nil -} + for _, a := range apps { + downstreams, err := store.GetStore().ListDownstreamsForApp(a.ID) + if err != nil { + return nil, errors.Wrapf(err, "failed to list downstreams for app %s", a.Slug) + } -// getDefaultEmbeddedClusterBackupSpec returns a default velero backup spec for an embedded -// cluster using the old DR (no backup and restore custom resource). -func getDefaultEmbeddedClusterBackupSpec() *velerov1.Backup { - return &velerov1.Backup{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "velero.io/v1", - Kind: "Backup", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "backup", - }, + if len(downstreams) == 0 { + // this is unexpected + continue + } + + parentSequence, err := store.GetStore().GetCurrentParentSequence(a.ID, downstreams[0].ClusterID) + if err != nil { + return nil, errors.Wrapf(err, "failed to get current downstream parent sequence for app %s", a.Slug) + } + if parentSequence == -1 { + // no version is deployed for this app yet + continue + } + + archiveDir, err := os.MkdirTemp("", "kotsadm") + if err != nil { + return nil, errors.Wrapf(err, "failed to create temp dir for app %s", a.Slug) + } + defer os.RemoveAll(archiveDir) + + err = store.GetStore().GetAppVersionArchive(a.ID, parentSequence, archiveDir) + if err != nil { + return nil, errors.Wrapf(err, "failed to get app version archive for app %s", a.Slug) + } + + kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) + if err != nil { + return nil, errors.Wrap(err, "failed to load kots kinds from path") + } + + // if there is both a backup and a restore spec this is using the new improved DR + if kotsKinds.Backup == nil || kotsKinds.Restore == nil { + continue + } + + if len(apps) > 0 { + return nil, errors.New("cannot create backup for Embedded Cluster with multiple apps") + } + + backupSpec, err := kotsKinds.Marshal("velero.io", "v1", "Backup") + if err != nil { + return nil, errors.Wrap(err, "failed to get backup spec from kotskinds") + } + + if backupSpec == "" { + return nil, errors.New("backup spec is empty, this is unexpected") + } + + renderedBackup, err := helper.RenderAppFile(a, nil, []byte(backupSpec), kotsKinds, kotsadmNamespace) + if err != nil { + return nil, errors.Wrap(err, "failed to render backup") + } + appVeleroBackup, err := kotsutil.LoadBackupFromContents(renderedBackup) + if err != nil { + return nil, errors.Wrap(err, "failed to load backup from contents") + } + + appVeleroBackup.Name = "" + appVeleroBackup.GenerateName = a.Slug + "-" + + return appVeleroBackup, nil } + + return nil, nil } // mergeAppBackupSpec merges the app backup spec into the velero backup spec when improved DR is @@ -549,25 +580,67 @@ func getDefaultEmbeddedClusterBackupSpec() *velerov1.Backup { // - includedResources // - excludedResources // - labelSelector -func mergeAppBackupSpec(backup, appBackup *velerov1.Backup) { +func mergeAppBackupSpec(backup *velerov1.Backup, kotsKinds *kotsutil.KotsKinds, kotsadmNamespace string) error { + backupSpec, err := kotsKinds.Marshal("velero.io", "v1", "Backup") + if err != nil { + return errors.Wrap(err, "failed to get backup spec from kotskinds") + } + + var kotskindsBackup *velerov1.Backup + if backupSpec == "" { + // If this is Embedded Cluster, backups are always enabled and we will use a default + // minimal backup spec + if util.IsEmbeddedCluster() { + kotskindsBackup = getDefaultEmbeddedClusterBackupSpec() + } else { + return nil + } + } else { + renderedBackup, err := helper.RenderAppFile(a, nil, []byte(backupSpec), kotsKinds, kotsadmNamespace) + if err != nil { + return errors.Wrap(err, "failed to render backup") + } + kotskindsBackup, err = kotsutil.LoadBackupFromContents(renderedBackup) + if err != nil { + return errors.Wrap(err, "failed to load backup from contents") + } + } + // included namespaces - backup.Spec.IncludedNamespaces = append(backup.Spec.IncludedNamespaces, appBackup.Spec.IncludedNamespaces...) + backup.Spec.IncludedNamespaces = append(backup.Spec.IncludedNamespaces, kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) + backup.Spec.IncludedNamespaces = append(backup.Spec.IncludedNamespaces, kotskindsBackup.Spec.IncludedNamespaces...) // excluded namespaces - backup.Spec.ExcludedNamespaces = append(backup.Spec.ExcludedNamespaces, appBackup.Spec.ExcludedNamespaces...) + backup.Spec.ExcludedNamespaces = append(backup.Spec.ExcludedNamespaces, kotskindsBackup.Spec.ExcludedNamespaces...) // annotations - for k, v := range appBackup.Annotations { + for k, v := range kotskindsBackup.Annotations { backup.Annotations[k] = v } // ordered resources - for k, v := range appBackup.Spec.OrderedResources { + for k, v := range kotskindsBackup.Spec.OrderedResources { backup.Spec.OrderedResources[k] = v } // backup hooks - backup.Spec.Hooks.Resources = append(backup.Spec.Hooks.Resources, appBackup.Spec.Hooks.Resources...) + backup.Spec.Hooks.Resources = append(backup.Spec.Hooks.Resources, kotskindsBackup.Spec.Hooks.Resources...) + + return nil +} + +// getDefaultEmbeddedClusterBackupSpec returns a default velero backup spec for an embedded +// cluster using the old DR (no backup and restore custom resource). +func getDefaultEmbeddedClusterBackupSpec() *velerov1.Backup { + return &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "backup", + }, + } } func ListBackupsForApp(ctx context.Context, kotsadmNamespace string, appID string) ([]*types.Backup, error) { From 87fc2db39c350f553043606306b7b71027278eab Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 2 Dec 2024 21:15:44 -0800 Subject: [PATCH 10/44] f --- pkg/kotsadmsnapshot/backup.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index babb3ae818..49cdc11ccf 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -326,7 +326,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre // Don't merge the backup spec if we are using the new improved DR. if appVeleroBackup == nil { - err := mergeAppBackupSpec(veleroBackup, kotsKinds, kotsadmNamespace) + err := mergeAppBackupSpec(veleroBackup, kotsKinds, a, kotsadmNamespace) if err != nil { return "", errors.Wrap(err, "failed to merge app backup spec") } @@ -580,7 +580,7 @@ func getAppInstanceBackup(apps []*apptypes.App, kotsadmNamespace string) (*veler // - includedResources // - excludedResources // - labelSelector -func mergeAppBackupSpec(backup *velerov1.Backup, kotsKinds *kotsutil.KotsKinds, kotsadmNamespace string) error { +func mergeAppBackupSpec(backup *velerov1.Backup, kotsKinds *kotsutil.KotsKinds, a *apptypes.App, kotsadmNamespace string) error { backupSpec, err := kotsKinds.Marshal("velero.io", "v1", "Backup") if err != nil { return errors.Wrap(err, "failed to get backup spec from kotskinds") From 7f615f6c5a413c65df0adb63a00e226d02d9cf1e Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Mon, 2 Dec 2024 21:16:05 -0800 Subject: [PATCH 11/44] f --- pkg/kotsadmsnapshot/backup.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 49cdc11ccf..333983f222 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -580,7 +580,7 @@ func getAppInstanceBackup(apps []*apptypes.App, kotsadmNamespace string) (*veler // - includedResources // - excludedResources // - labelSelector -func mergeAppBackupSpec(backup *velerov1.Backup, kotsKinds *kotsutil.KotsKinds, a *apptypes.App, kotsadmNamespace string) error { +func mergeAppBackupSpec(backup *velerov1.Backup, kotsKinds *kotsutil.KotsKinds, app *apptypes.App, kotsadmNamespace string) error { backupSpec, err := kotsKinds.Marshal("velero.io", "v1", "Backup") if err != nil { return errors.Wrap(err, "failed to get backup spec from kotskinds") @@ -596,7 +596,7 @@ func mergeAppBackupSpec(backup *velerov1.Backup, kotsKinds *kotsutil.KotsKinds, return nil } } else { - renderedBackup, err := helper.RenderAppFile(a, nil, []byte(backupSpec), kotsKinds, kotsadmNamespace) + renderedBackup, err := helper.RenderAppFile(app, nil, []byte(backupSpec), kotsKinds, kotsadmNamespace) if err != nil { return errors.Wrap(err, "failed to render backup") } From 4d8c645ef4b63df8f2c772714e923563b498e199 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Tue, 3 Dec 2024 06:59:18 -0800 Subject: [PATCH 12/44] f --- pkg/kotsadmsnapshot/backup.go | 301 ++++++++++++++++++---------------- 1 file changed, 156 insertions(+), 145 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 333983f222..64fcb76203 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -201,6 +201,21 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b return backup, nil } +type instanceBackupMetadata struct { + backupName string + kotsadmNamespace string + backupStorageLocationNamespace string + apps map[string]appInstanceBackupMetadata + isScheduled bool + snapshotTTL time.Duration +} + +type appInstanceBackupMetadata struct { + app *apptypes.App + kotsKinds *kotsutil.KotsKinds + parentSequence int64 +} + func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstream, isScheduled bool) (string, error) { logger.Debug("creating instance backup") @@ -209,7 +224,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return "", errors.Wrap(err, "failed to get cluster config") } - clientset, err := kubernetes.NewForConfig(cfg) + k8sClient, err := kubernetes.NewForConfig(cfg) if err != nil { return "", errors.Wrap(err, "failed to create clientset") } @@ -219,87 +234,73 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return "", errors.Wrap(err, "failed to create velero clientset") } - kotsadmNamespace := util.PodNamespace - - kotsadmVeleroBackendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) + metadata, err := getInstanceBackupMetadata(ctx, k8sClient, veleroClient, cluster, isScheduled) if err != nil { - return "", errors.Wrap(err, "failed to find backupstoragelocations") + return "", errors.Wrap(err, "failed to get instance backup metadata") } - if kotsadmVeleroBackendStorageLocation == nil { - return "", errors.New("no backup store location found") + veleroBackup, appVeleroBackup, err := getInstanceBackupSpecs(ctx, k8sClient, metadata) + if err != nil { + return "", errors.Wrap(err, "failed to get instance backup specs") } - // veleroBackup is the kotsadm backup or combined backup if usesImprovedDR is false - veleroBackup := &velerov1.Backup{ - ObjectMeta: metav1.ObjectMeta{ - Name: "", - GenerateName: "instance-", - Annotations: map[string]string{}, - }, - Spec: velerov1.BackupSpec{ - StorageLocation: "default", - IncludedNamespaces: []string{}, - ExcludedNamespaces: []string{}, - IncludeClusterResources: ptr.To(true), - OrLabelSelectors: instanceBackupLabelSelectors(util.IsEmbeddedCluster()), - OrderedResources: map[string]string{}, - Hooks: velerov1.BackupHooks{ - Resources: []velerov1.BackupResourceHookSpec{}, - }, - }, + _, err = veleroClient.Backups(metadata.backupStorageLocationNamespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) + if err != nil { + return "", errors.Wrap(err, "failed to create velero backup") } - appSequences := map[string]int64{} - appVersions := map[string]string{} - - appNamespace := kotsadmNamespace - if os.Getenv("KOTSADM_TARGET_NAMESPACE") != "" { - appNamespace = os.Getenv("KOTSADM_TARGET_NAMESPACE") - } - if appNamespace != kotsadmNamespace { - veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, appNamespace) + if appVeleroBackup != nil { + _, err := veleroClient.Backups(metadata.backupStorageLocationNamespace).Create(ctx, appVeleroBackup, metav1.CreateOptions{}) + if err != nil { + return "", errors.Wrap(err, "failed to create application velero backup") + } } - isKurl, err := kurl.IsKurl(clientset) - if err != nil { - return "", errors.Wrap(err, "failed to check if cluster is kurl") - } - if isKurl { - veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, "kurl") + return metadata.backupName, nil +} + +func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interface, veleroClient veleroclientv1.VeleroV1Interface, cluster *downstreamtypes.Downstream, isScheduled bool) (instanceBackupMetadata, error) { + metadata := instanceBackupMetadata{ + backupName: fmt.Sprintf("backup-%d", time.Now().UnixNano()), + kotsadmNamespace: util.PodNamespace, + isScheduled: isScheduled, } - isKotsadmClusterScoped := k8sutil.IsKotsadmClusterScoped(ctx, clientset, kotsadmNamespace) - if !isKotsadmClusterScoped { - // in minimal rbac, a kotsadm role and rolebinding will exist in the velero namespace to give kotsadm access to velero. - // we backup and restore those so that restoring to a new cluster won't require that the user provide those permissions again. - veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, kotsadmVeleroBackendStorageLocation.Namespace) + if cluster.SnapshotTTL != "" { + snapshotTTL, err := time.ParseDuration(cluster.SnapshotTTL) + if err != nil { + return metadata, errors.Wrap(err, "failed to parse snapshot ttl") + } + metadata.snapshotTTL = snapshotTTL } - apps, err := store.GetStore().ListInstalledApps() + kotsadmVeleroBackendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, k8sClient, veleroClient, metadata.kotsadmNamespace) if err != nil { - return "", errors.Wrap(err, "failed to list installed apps") + return metadata, errors.Wrap(err, "failed to find backupstoragelocations") + } else if kotsadmVeleroBackendStorageLocation == nil { + return metadata, errors.New("no backup store location found") } + metadata.backupStorageLocationNamespace = kotsadmVeleroBackendStorageLocation.Namespace - appVeleroBackup, err := getAppInstanceBackup(apps, kotsadmNamespace) + apps, err := store.GetStore().ListInstalledApps() if err != nil { - return "", errors.Wrap(err, "failed to get app instance backup") + return metadata, errors.Wrap(err, "failed to list installed apps") } - for _, a := range apps { - downstreams, err := store.GetStore().ListDownstreamsForApp(a.ID) + for _, app := range apps { + downstreams, err := store.GetStore().ListDownstreamsForApp(app.ID) if err != nil { - return "", errors.Wrapf(err, "failed to list downstreams for app %s", a.Slug) + return metadata, errors.Wrapf(err, "failed to list downstreams for app %s", app.Slug) } if len(downstreams) == 0 { - logger.Errorf("No downstreams found for app %s", a.Slug) + logger.Errorf("No downstreams found for app %s", app.Slug) continue } - parentSequence, err := store.GetStore().GetCurrentParentSequence(a.ID, downstreams[0].ClusterID) + parentSequence, err := store.GetStore().GetCurrentParentSequence(app.ID, downstreams[0].ClusterID) if err != nil { - return "", errors.Wrapf(err, "failed to get current downstream parent sequence for app %s", a.Slug) + return metadata, errors.Wrapf(err, "failed to get current downstream parent sequence for app %s", app.Slug) } if parentSequence == -1 { // no version is deployed for this app yet @@ -308,69 +309,130 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre archiveDir, err := os.MkdirTemp("", "kotsadm") if err != nil { - return "", errors.Wrapf(err, "failed to create temp dir for app %s", a.Slug) + return metadata, errors.Wrapf(err, "failed to create temp dir for app %s", app.Slug) } defer func() { _ = os.RemoveAll(archiveDir) }() - err = store.GetStore().GetAppVersionArchive(a.ID, parentSequence, archiveDir) + err = store.GetStore().GetAppVersionArchive(app.ID, parentSequence, archiveDir) if err != nil { - return "", errors.Wrapf(err, "failed to get app version archive for app %s", a.Slug) + return metadata, errors.Wrapf(err, "failed to get app version archive for app %s", app.Slug) } kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) if err != nil { - return "", errors.Wrap(err, "failed to load kots kinds from path") + return metadata, errors.Wrap(err, "failed to load kots kinds from path") + } + + metadata.apps[app.Slug] = appInstanceBackupMetadata{ + app: app, + kotsKinds: kotsKinds, + parentSequence: parentSequence, } + // optimization as we no longer need the archive dir + _ = os.RemoveAll(archiveDir) + } + + return metadata, nil +} + +func getInstanceBackupSpecs(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata) (veleroBackup *velerov1.Backup, appVeleroBackup *velerov1.Backup, err error) { + // veleroBackup is the kotsadm backup or combined backup if usesImprovedDR is false + veleroBackup = &velerov1.Backup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{}, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{}, + ExcludedNamespaces: []string{}, + IncludeClusterResources: ptr.To(true), + OrLabelSelectors: instanceBackupLabelSelectors(util.IsEmbeddedCluster()), + OrderedResources: map[string]string{}, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{}, + }, + }, + } + + appNamespace := metadata.kotsadmNamespace + if os.Getenv("KOTSADM_TARGET_NAMESPACE") != "" { + appNamespace = os.Getenv("KOTSADM_TARGET_NAMESPACE") + } + if appNamespace != metadata.kotsadmNamespace { + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, appNamespace) + } + + isKurl, err := kurl.IsKurl(k8sClient) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to check if cluster is kurl") + } + if isKurl { + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, "kurl") + } + + isKotsadmClusterScoped := k8sutil.IsKotsadmClusterScoped(ctx, k8sClient, metadata.kotsadmNamespace) + if !isKotsadmClusterScoped { + // in minimal rbac, a kotsadm role and rolebinding will exist in the velero namespace to give kotsadm access to velero. + // we backup and restore those so that restoring to a new cluster won't require that the user provide those permissions again. + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, metadata.backupStorageLocationNamespace) + } + + appVeleroBackup, err = getAppInstanceBackup(metadata) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to get app instance backup") + } + + appSequences := map[string]int64{} + appVersions := map[string]string{} + + for slug, appMeta := range metadata.apps { // Don't merge the backup spec if we are using the new improved DR. if appVeleroBackup == nil { - err := mergeAppBackupSpec(veleroBackup, kotsKinds, a, kotsadmNamespace) + err := mergeAppBackupSpec(veleroBackup, appMeta.kotsKinds, appMeta.app, metadata.kotsadmNamespace) if err != nil { - return "", errors.Wrap(err, "failed to merge app backup spec") + return nil, nil, errors.Wrap(err, "failed to merge app backup spec") } } - appSequences[a.Slug] = parentSequence - appVersions[a.Slug] = kotsKinds.Installation.Spec.VersionLabel - - // optimization as we no longer need the archive dir - _ = os.RemoveAll(archiveDir) + appSequences[slug] = appMeta.parentSequence + appVersions[slug] = appMeta.kotsKinds.Installation.Spec.VersionLabel } - kotsadmImage, err := k8sutil.FindKotsadmImage(kotsadmNamespace) + kotsadmImage, err := k8sutil.FindKotsadmImage(metadata.kotsadmNamespace) if err != nil { - return "", errors.Wrap(err, "failed to find kotsadm image") + return nil, nil, errors.Wrap(err, "failed to find kotsadm image") } snapshotTrigger := "manual" - if isScheduled { + if metadata.isScheduled { snapshotTrigger = "schedule" } // marshal apps sequences map b, err := json.Marshal(appSequences) if err != nil { - return "", errors.Wrap(err, "failed to marshal apps sequences") + return nil, nil, errors.Wrap(err, "failed to marshal apps sequences") } marshalledAppSequences := string(b) // marshal apps versions map b, err = json.Marshal(appVersions) if err != nil { - return "", errors.Wrap(err, "failed to marshal apps versions") + return nil, nil, errors.Wrap(err, "failed to marshal apps versions") } marshalledAppVersions := string(b) - now := time.Now() - backupName := fmt.Sprintf("backup-%d", now.UnixNano()) - numBackups := 1 if appVeleroBackup != nil { numBackups = 2 } + now := time.Now() appendCommonAnnotations := func(annotations map[string]string) map[string]string { if annotations == nil { annotations = make(map[string]string, 0) @@ -379,17 +441,17 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre annotations["kots.io/snapshot-requested"] = now.UTC().Format(time.RFC3339) annotations["kots.io/instance"] = "true" annotations["kots.io/kotsadm-image"] = kotsadmImage - annotations["kots.io/kotsadm-deploy-namespace"] = kotsadmNamespace + annotations["kots.io/kotsadm-deploy-namespace"] = metadata.kotsadmNamespace annotations["kots.io/apps-sequences"] = marshalledAppSequences annotations["kots.io/apps-versions"] = marshalledAppVersions annotations["kots.io/is-airgap"] = strconv.FormatBool(kotsadm.IsAirgap()) - embeddedRegistryHost, _, _ := kotsutil.GetEmbeddedRegistryCreds(clientset) + embeddedRegistryHost, _, _ := kotsutil.GetEmbeddedRegistryCreds(k8sClient) if embeddedRegistryHost != "" { annotations["kots.io/embedded-registry"] = embeddedRegistryHost } // Add improved disaster recovery annotation labels - annotations[InstanceBackupNameAnnotation] = backupName + annotations[InstanceBackupNameAnnotation] = metadata.backupName annotations[InstanceBackupsExpectedAnnotation] = strconv.Itoa(numBackups) return annotations @@ -407,48 +469,44 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre if util.IsEmbeddedCluster() { kbClient, err := k8sutil.GetKubeClient(ctx) if err != nil { - return "", fmt.Errorf("failed to get kubeclient: %w", err) + return nil, nil, fmt.Errorf("failed to get kubeclient: %w", err) } installation, err := embeddedcluster.GetCurrentInstallation(ctx, kbClient) if err != nil { - return "", fmt.Errorf("failed to get current installation: %w", err) + return nil, nil, fmt.Errorf("failed to get current installation: %w", err) } veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, ecIncludedNamespaces(installation)...) appVeleroBackup.Annotations, err = appendECAnnotations(ctx, appVeleroBackup.Annotations, kbClient, installation) if err != nil { - return "", errors.Wrap(err, "failed to add annotations to backup for embedded cluster") + return nil, nil, errors.Wrap(err, "failed to add annotations to backup for embedded cluster") } if appVeleroBackup != nil { appVeleroBackup.Annotations, err = appendECAnnotations(ctx, appVeleroBackup.Annotations, kbClient, installation) if err != nil { - return "", errors.Wrap(err, "failed to add annotations to application backup for embedded cluster") + return nil, nil, errors.Wrap(err, "failed to add annotations to application backup for embedded cluster") } } } - if cluster.SnapshotTTL != "" { - ttlDuration, err := time.ParseDuration(cluster.SnapshotTTL) - if err != nil { - return "", errors.Wrap(err, "failed to parse cluster snapshot ttl value as duration") - } + if metadata.snapshotTTL > 0 { veleroBackup.Spec.TTL = metav1.Duration{ - Duration: ttlDuration, + Duration: metadata.snapshotTTL, } if appVeleroBackup != nil { appVeleroBackup.Spec.TTL = metav1.Duration{ - Duration: ttlDuration, + Duration: metadata.snapshotTTL, } } } - err = excludeShutdownPodsFromBackup(ctx, clientset, veleroBackup) + err = excludeShutdownPodsFromBackup(ctx, k8sClient, veleroBackup) if err != nil { logger.Errorf("Failed to exclude shutdown pods from backup: %v", err) } if appVeleroBackup != nil { - err = excludeShutdownPodsFromBackup(ctx, clientset, appVeleroBackup) + err = excludeShutdownPodsFromBackup(ctx, k8sClient, appVeleroBackup) if err != nil { logger.Errorf("Failed to exclude shutdown pods from application backup: %v", err) } @@ -456,19 +514,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre veleroBackup.Spec.IncludedNamespaces = prepareIncludedNamespaces(veleroBackup.Spec.IncludedNamespaces) - _, err = veleroClient.Backups(kotsadmVeleroBackendStorageLocation.Namespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) - if err != nil { - return "", errors.Wrap(err, "failed to create velero backup") - } - - if appVeleroBackup != nil { - _, err := veleroClient.Backups(kotsadmVeleroBackendStorageLocation.Namespace).Create(ctx, appVeleroBackup, metav1.CreateOptions{}) - if err != nil { - return "", errors.Wrap(err, "failed to create application velero backup") - } - } - - return backupName, nil + return veleroBackup, appVeleroBackup, nil } func GetBackupName(veleroBackup velerov1.Backup) string { @@ -497,57 +543,22 @@ func GetInstanceBackupsExpected(veleroBackup velerov1.Backup) int { // getAppInstanceBackup returns a backup spec only if this is Embedded Cluster and the vendor has // defined both a backup and restore custom resource. -func getAppInstanceBackup(apps []*apptypes.App, kotsadmNamespace string) (*velerov1.Backup, error) { +func getAppInstanceBackup(metadata instanceBackupMetadata) (*velerov1.Backup, error) { if !util.IsEmbeddedCluster() { return nil, nil } - for _, a := range apps { - downstreams, err := store.GetStore().ListDownstreamsForApp(a.ID) - if err != nil { - return nil, errors.Wrapf(err, "failed to list downstreams for app %s", a.Slug) - } - - if len(downstreams) == 0 { - // this is unexpected - continue - } - - parentSequence, err := store.GetStore().GetCurrentParentSequence(a.ID, downstreams[0].ClusterID) - if err != nil { - return nil, errors.Wrapf(err, "failed to get current downstream parent sequence for app %s", a.Slug) - } - if parentSequence == -1 { - // no version is deployed for this app yet - continue - } - - archiveDir, err := os.MkdirTemp("", "kotsadm") - if err != nil { - return nil, errors.Wrapf(err, "failed to create temp dir for app %s", a.Slug) - } - defer os.RemoveAll(archiveDir) - - err = store.GetStore().GetAppVersionArchive(a.ID, parentSequence, archiveDir) - if err != nil { - return nil, errors.Wrapf(err, "failed to get app version archive for app %s", a.Slug) - } - - kotsKinds, err := kotsutil.LoadKotsKinds(archiveDir) - if err != nil { - return nil, errors.Wrap(err, "failed to load kots kinds from path") - } - + for slug, appMeta := range metadata.apps { // if there is both a backup and a restore spec this is using the new improved DR - if kotsKinds.Backup == nil || kotsKinds.Restore == nil { + if appMeta.kotsKinds.Backup == nil || appMeta.kotsKinds.Restore == nil { continue } - if len(apps) > 0 { + if len(metadata.apps) > 0 { return nil, errors.New("cannot create backup for Embedded Cluster with multiple apps") } - backupSpec, err := kotsKinds.Marshal("velero.io", "v1", "Backup") + backupSpec, err := appMeta.kotsKinds.Marshal("velero.io", "v1", "Backup") if err != nil { return nil, errors.Wrap(err, "failed to get backup spec from kotskinds") } @@ -556,7 +567,7 @@ func getAppInstanceBackup(apps []*apptypes.App, kotsadmNamespace string) (*veler return nil, errors.New("backup spec is empty, this is unexpected") } - renderedBackup, err := helper.RenderAppFile(a, nil, []byte(backupSpec), kotsKinds, kotsadmNamespace) + renderedBackup, err := helper.RenderAppFile(appMeta.app, nil, []byte(backupSpec), appMeta.kotsKinds, metadata.kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to render backup") } @@ -566,7 +577,7 @@ func getAppInstanceBackup(apps []*apptypes.App, kotsadmNamespace string) (*veler } appVeleroBackup.Name = "" - appVeleroBackup.GenerateName = a.Slug + "-" + appVeleroBackup.GenerateName = slug + "-" return appVeleroBackup, nil } From 7cefe6f0399089f1ef556f9044cde205853cb7ba Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Tue, 3 Dec 2024 09:44:26 -0800 Subject: [PATCH 13/44] f --- pkg/kotsadmsnapshot/backup.go | 345 +++++++++++++++++++--------------- 1 file changed, 194 insertions(+), 151 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 64fcb76203..01a37bab62 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -36,7 +36,6 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" "k8s.io/utils/ptr" - kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -208,6 +207,7 @@ type instanceBackupMetadata struct { apps map[string]appInstanceBackupMetadata isScheduled bool snapshotTTL time.Duration + ec *ecInstanceBackupMetadata } type appInstanceBackupMetadata struct { @@ -216,6 +216,11 @@ type appInstanceBackupMetadata struct { parentSequence int64 } +type ecInstanceBackupMetadata struct { + installation *embeddedclusterv1beta1.Installation + seaweedFSS3ServiceIP string +} + func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstream, isScheduled bool) (string, error) { logger.Debug("creating instance backup") @@ -239,7 +244,12 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return "", errors.Wrap(err, "failed to get instance backup metadata") } - veleroBackup, appVeleroBackup, err := getInstanceBackupSpecs(ctx, k8sClient, metadata) + appVeleroBackup, err := getAppInstanceBackupSpec(ctx, k8sClient, metadata) + if err != nil { + return "", errors.Wrap(err, "failed to get app instance backup spec") + } + + veleroBackup, err := getInstanceBackupSpec(ctx, k8sClient, metadata, appVeleroBackup != nil) if err != nil { return "", errors.Wrap(err, "failed to get instance backup specs") } @@ -259,6 +269,30 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return metadata.backupName, nil } +func GetBackupName(veleroBackup velerov1.Backup) string { + if val, ok := veleroBackup.GetAnnotations()[InstanceBackupNameAnnotation]; ok { + return val + } + return veleroBackup.GetName() +} + +func GetInstanceBackupType(veleroBackup velerov1.Backup) string { + if val, ok := veleroBackup.GetAnnotations()[InstanceBackupTypeAnnotation]; ok { + return val + } + return "" +} + +func GetInstanceBackupsExpected(veleroBackup velerov1.Backup) int { + if val, ok := veleroBackup.GetAnnotations()[InstanceBackupsExpectedAnnotation]; ok { + num, _ := strconv.Atoi(val) + if num > 0 { + return num + } + } + return 1 +} + func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interface, veleroClient veleroclientv1.VeleroV1Interface, cluster *downstreamtypes.Downstream, isScheduled bool) (instanceBackupMetadata, error) { metadata := instanceBackupMetadata{ backupName: fmt.Sprintf("backup-%d", time.Now().UnixNano()), @@ -335,12 +369,45 @@ func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interfa _ = os.RemoveAll(archiveDir) } + if util.IsEmbeddedCluster() { + metadata.ec, err = getECInstanceBackupMetadata(ctx) + if err != nil { + return metadata, errors.Wrap(err, "failed to get embedded cluster metadata") + } + } + return metadata, nil } -func getInstanceBackupSpecs(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata) (veleroBackup *velerov1.Backup, appVeleroBackup *velerov1.Backup, err error) { +func getECInstanceBackupMetadata(ctx context.Context) (*ecInstanceBackupMetadata, error) { + if !util.IsEmbeddedCluster() { + return nil, nil + } + + kbClient, err := k8sutil.GetKubeClient(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get kubeclient: %w", err) + } + + installation, err := embeddedcluster.GetCurrentInstallation(ctx, kbClient) + if err != nil { + return nil, fmt.Errorf("failed to get current installation: %w", err) + } + + seaweedFSS3ServiceIP, err := embeddedcluster.GetSeaweedFSS3ServiceIP(ctx, kbClient) + if err != nil { + return nil, fmt.Errorf("failed to get seaweedfs s3 service ip: %w", err) + } + + return &ecInstanceBackupMetadata{ + installation: installation, + seaweedFSS3ServiceIP: seaweedFSS3ServiceIP, + }, nil +} + +func getInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata, hasAppSpec bool) (*velerov1.Backup, error) { // veleroBackup is the kotsadm backup or combined backup if usesImprovedDR is false - veleroBackup = &velerov1.Backup{ + veleroBackup := &velerov1.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "", GenerateName: "instance-", @@ -369,7 +436,7 @@ func getInstanceBackupSpecs(ctx context.Context, k8sClient kubernetes.Interface, isKurl, err := kurl.IsKurl(k8sClient) if err != nil { - return nil, nil, errors.Wrap(err, "failed to check if cluster is kurl") + return nil, errors.Wrap(err, "failed to check if cluster is kurl") } if isKurl { veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, "kurl") @@ -382,172 +449,55 @@ func getInstanceBackupSpecs(ctx context.Context, k8sClient kubernetes.Interface, veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, metadata.backupStorageLocationNamespace) } - appVeleroBackup, err = getAppInstanceBackup(metadata) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to get app instance backup") - } - - appSequences := map[string]int64{} - appVersions := map[string]string{} - - for slug, appMeta := range metadata.apps { + for _, appMeta := range metadata.apps { // Don't merge the backup spec if we are using the new improved DR. - if appVeleroBackup == nil { + if !hasAppSpec { err := mergeAppBackupSpec(veleroBackup, appMeta.kotsKinds, appMeta.app, metadata.kotsadmNamespace) if err != nil { - return nil, nil, errors.Wrap(err, "failed to merge app backup spec") + return nil, errors.Wrap(err, "failed to merge app backup spec") } } - - appSequences[slug] = appMeta.parentSequence - appVersions[slug] = appMeta.kotsKinds.Installation.Spec.VersionLabel - } - - kotsadmImage, err := k8sutil.FindKotsadmImage(metadata.kotsadmNamespace) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to find kotsadm image") - } - - snapshotTrigger := "manual" - if metadata.isScheduled { - snapshotTrigger = "schedule" } - // marshal apps sequences map - b, err := json.Marshal(appSequences) + veleroBackup.Annotations, err = appendCommonAnnotations(ctx, k8sClient, veleroBackup.Annotations, metadata, hasAppSpec) if err != nil { - return nil, nil, errors.Wrap(err, "failed to marshal apps sequences") + return nil, errors.Wrap(err, "failed to add annotations to backup") } - marshalledAppSequences := string(b) - - // marshal apps versions map - b, err = json.Marshal(appVersions) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to marshal apps versions") - } - marshalledAppVersions := string(b) - - numBackups := 1 - if appVeleroBackup != nil { - numBackups = 2 - } - - now := time.Now() - appendCommonAnnotations := func(annotations map[string]string) map[string]string { - if annotations == nil { - annotations = make(map[string]string, 0) - } - annotations["kots.io/snapshot-trigger"] = snapshotTrigger - annotations["kots.io/snapshot-requested"] = now.UTC().Format(time.RFC3339) - annotations["kots.io/instance"] = "true" - annotations["kots.io/kotsadm-image"] = kotsadmImage - annotations["kots.io/kotsadm-deploy-namespace"] = metadata.kotsadmNamespace - annotations["kots.io/apps-sequences"] = marshalledAppSequences - annotations["kots.io/apps-versions"] = marshalledAppVersions - annotations["kots.io/is-airgap"] = strconv.FormatBool(kotsadm.IsAirgap()) - embeddedRegistryHost, _, _ := kotsutil.GetEmbeddedRegistryCreds(k8sClient) - if embeddedRegistryHost != "" { - annotations["kots.io/embedded-registry"] = embeddedRegistryHost - } - - // Add improved disaster recovery annotation labels - annotations[InstanceBackupNameAnnotation] = metadata.backupName - annotations[InstanceBackupsExpectedAnnotation] = strconv.Itoa(numBackups) - - return annotations - } - - veleroBackup.Annotations = appendCommonAnnotations(veleroBackup.Annotations) - if appVeleroBackup != nil { + if hasAppSpec { veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeKotsadm - appVeleroBackup.Annotations = appendCommonAnnotations(appVeleroBackup.Annotations) - appVeleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeApp } else { veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeCombined } - if util.IsEmbeddedCluster() { - kbClient, err := k8sutil.GetKubeClient(ctx) - if err != nil { - return nil, nil, fmt.Errorf("failed to get kubeclient: %w", err) - } - installation, err := embeddedcluster.GetCurrentInstallation(ctx, kbClient) - if err != nil { - return nil, nil, fmt.Errorf("failed to get current installation: %w", err) - } - - veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, ecIncludedNamespaces(installation)...) - - appVeleroBackup.Annotations, err = appendECAnnotations(ctx, appVeleroBackup.Annotations, kbClient, installation) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to add annotations to backup for embedded cluster") - } - if appVeleroBackup != nil { - appVeleroBackup.Annotations, err = appendECAnnotations(ctx, appVeleroBackup.Annotations, kbClient, installation) - if err != nil { - return nil, nil, errors.Wrap(err, "failed to add annotations to application backup for embedded cluster") - } - } + if metadata.ec != nil { + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, ecIncludedNamespaces(metadata.ec.installation)...) } if metadata.snapshotTTL > 0 { veleroBackup.Spec.TTL = metav1.Duration{ Duration: metadata.snapshotTTL, } - if appVeleroBackup != nil { - appVeleroBackup.Spec.TTL = metav1.Duration{ - Duration: metadata.snapshotTTL, - } - } } err = excludeShutdownPodsFromBackup(ctx, k8sClient, veleroBackup) if err != nil { logger.Errorf("Failed to exclude shutdown pods from backup: %v", err) } - if appVeleroBackup != nil { - err = excludeShutdownPodsFromBackup(ctx, k8sClient, appVeleroBackup) - if err != nil { - logger.Errorf("Failed to exclude shutdown pods from application backup: %v", err) - } - } veleroBackup.Spec.IncludedNamespaces = prepareIncludedNamespaces(veleroBackup.Spec.IncludedNamespaces) - return veleroBackup, appVeleroBackup, nil -} - -func GetBackupName(veleroBackup velerov1.Backup) string { - if val, ok := veleroBackup.GetAnnotations()[InstanceBackupNameAnnotation]; ok { - return val - } - return veleroBackup.GetName() -} - -func GetInstanceBackupType(veleroBackup velerov1.Backup) string { - if val, ok := veleroBackup.GetAnnotations()[InstanceBackupTypeAnnotation]; ok { - return val - } - return "" -} - -func GetInstanceBackupsExpected(veleroBackup velerov1.Backup) int { - if val, ok := veleroBackup.GetAnnotations()[InstanceBackupsExpectedAnnotation]; ok { - num, _ := strconv.Atoi(val) - if num > 0 { - return num - } - } - return 1 + return veleroBackup, nil } // getAppInstanceBackup returns a backup spec only if this is Embedded Cluster and the vendor has // defined both a backup and restore custom resource. -func getAppInstanceBackup(metadata instanceBackupMetadata) (*velerov1.Backup, error) { +func getAppInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata) (*velerov1.Backup, error) { if !util.IsEmbeddedCluster() { return nil, nil } + var appVeleroBackup *velerov1.Backup + for slug, appMeta := range metadata.apps { // if there is both a backup and a restore spec this is using the new improved DR if appMeta.kotsKinds.Backup == nil || appMeta.kotsKinds.Restore == nil { @@ -571,7 +521,7 @@ func getAppInstanceBackup(metadata instanceBackupMetadata) (*velerov1.Backup, er if err != nil { return nil, errors.Wrap(err, "failed to render backup") } - appVeleroBackup, err := kotsutil.LoadBackupFromContents(renderedBackup) + appVeleroBackup, err = kotsutil.LoadBackupFromContents(renderedBackup) if err != nil { return nil, errors.Wrap(err, "failed to load backup from contents") } @@ -579,10 +529,40 @@ func getAppInstanceBackup(metadata instanceBackupMetadata) (*velerov1.Backup, er appVeleroBackup.Name = "" appVeleroBackup.GenerateName = slug + "-" - return appVeleroBackup, nil + break + } + + if appVeleroBackup == nil { + return nil, nil + } + + appSequences := map[string]int64{} + appVersions := map[string]string{} + + for slug, appMeta := range metadata.apps { + appSequences[slug] = appMeta.parentSequence + appVersions[slug] = appMeta.kotsKinds.Installation.Spec.VersionLabel } - return nil, nil + var err error + appVeleroBackup.Annotations, err = appendCommonAnnotations(ctx, k8sClient, appVeleroBackup.Annotations, metadata, true) + if err != nil { + return nil, errors.Wrap(err, "failed to add annotations to application backup") + } + appVeleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeApp + + if metadata.snapshotTTL > 0 { + appVeleroBackup.Spec.TTL = metav1.Duration{ + Duration: metadata.snapshotTTL, + } + } + + err = excludeShutdownPodsFromBackup(ctx, k8sClient, appVeleroBackup) + if err != nil { + logger.Errorf("Failed to exclude shutdown pods from application backup: %v", err) + } + + return appVeleroBackup, nil } // mergeAppBackupSpec merges the app backup spec into the velero backup spec when improved DR is @@ -654,6 +634,73 @@ func getDefaultEmbeddedClusterBackupSpec() *velerov1.Backup { } } +func appendCommonAnnotations(ctx context.Context, k8sClient kubernetes.Interface, annotations map[string]string, metadata instanceBackupMetadata, hasAppSpec bool) (map[string]string, error) { + kotsadmImage, err := k8sutil.FindKotsadmImage(metadata.kotsadmNamespace) + if err != nil { + return nil, errors.Wrap(err, "failed to find kotsadm image") + } + + snapshotTrigger := "manual" + if metadata.isScheduled { + snapshotTrigger = "schedule" + } + + appSequences := map[string]int64{} + appVersions := map[string]string{} + + for slug, appMeta := range metadata.apps { + appSequences[slug] = appMeta.parentSequence + appVersions[slug] = appMeta.kotsKinds.Installation.Spec.VersionLabel + } + + // marshal apps sequences map + b, err := json.Marshal(appSequences) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal app sequences") + } + marshalledAppSequences := string(b) + + // marshal apps versions map + b, err = json.Marshal(appVersions) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal app versions") + } + marshalledAppVersions := string(b) + + numBackups := 1 + if hasAppSpec { + numBackups = 2 + } + + now := time.Now() // TODO + + if annotations == nil { + annotations = make(map[string]string, 0) + } + annotations["kots.io/snapshot-trigger"] = snapshotTrigger + annotations["kots.io/snapshot-requested"] = now.UTC().Format(time.RFC3339) + annotations["kots.io/instance"] = "true" + annotations["kots.io/kotsadm-image"] = kotsadmImage + annotations["kots.io/kotsadm-deploy-namespace"] = metadata.kotsadmNamespace + annotations["kots.io/apps-sequences"] = marshalledAppSequences + annotations["kots.io/apps-versions"] = marshalledAppVersions + annotations["kots.io/is-airgap"] = strconv.FormatBool(kotsadm.IsAirgap()) + embeddedRegistryHost, _, _ := kotsutil.GetEmbeddedRegistryCreds(k8sClient) + if embeddedRegistryHost != "" { + annotations["kots.io/embedded-registry"] = embeddedRegistryHost + } + + // Add improved disaster recovery annotation labels + annotations[InstanceBackupNameAnnotation] = metadata.backupName + annotations[InstanceBackupsExpectedAnnotation] = strconv.Itoa(numBackups) + + if metadata.ec != nil { + annotations = appendECAnnotations(ctx, annotations, *metadata.ec) + } + + return annotations, nil +} + func ListBackupsForApp(ctx context.Context, kotsadmNamespace string, appID string) ([]*types.Backup, error) { cfg, err := k8sutil.GetClusterConfig() if err != nil { @@ -1199,37 +1246,33 @@ func mergeLabelSelector(kots metav1.LabelSelector, app metav1.LabelSelector) met } // appendECAnnotations appends annotations that should be added to an embedded cluster backup -func appendECAnnotations(ctx context.Context, annotations map[string]string, kbClient kbclient.Client, in *embeddedclusterv1beta1.Installation) (map[string]string, error) { +func appendECAnnotations(ctx context.Context, annotations map[string]string, ecMeta ecInstanceBackupMetadata) map[string]string { if annotations == nil { annotations = make(map[string]string, 0) } - seaweedFSS3ServiceIP, err := embeddedcluster.GetSeaweedFSS3ServiceIP(ctx, kbClient) - if err != nil { - return annotations, fmt.Errorf("failed to get seaweedfs s3 service ip: %w", err) - } - if seaweedFSS3ServiceIP != "" { - annotations["kots.io/embedded-cluster-seaweedfs-s3-ip"] = seaweedFSS3ServiceIP + if ecMeta.seaweedFSS3ServiceIP != "" { + annotations["kots.io/embedded-cluster-seaweedfs-s3-ip"] = ecMeta.seaweedFSS3ServiceIP } annotations["kots.io/embedded-cluster"] = "true" annotations["kots.io/embedded-cluster-id"] = util.EmbeddedClusterID() annotations["kots.io/embedded-cluster-version"] = util.EmbeddedClusterVersion() - annotations["kots.io/embedded-cluster-is-ha"] = strconv.FormatBool(in.Spec.HighAvailability) + annotations["kots.io/embedded-cluster-is-ha"] = strconv.FormatBool(ecMeta.installation.Spec.HighAvailability) - if in.Spec.Network != nil { - annotations["kots.io/embedded-cluster-pod-cidr"] = in.Spec.Network.PodCIDR - annotations["kots.io/embedded-cluster-service-cidr"] = in.Spec.Network.ServiceCIDR + if ecMeta.installation.Spec.Network != nil { + annotations["kots.io/embedded-cluster-pod-cidr"] = ecMeta.installation.Spec.Network.PodCIDR + annotations["kots.io/embedded-cluster-service-cidr"] = ecMeta.installation.Spec.Network.ServiceCIDR } - if in.Spec.RuntimeConfig != nil { - rcAnnotations := ecRuntimeConfigToBackupAnnotations(in.Spec.RuntimeConfig) + if ecMeta.installation.Spec.RuntimeConfig != nil { + rcAnnotations := ecRuntimeConfigToBackupAnnotations(ecMeta.installation.Spec.RuntimeConfig) for k, v := range rcAnnotations { annotations[k] = v } } - return annotations, nil + return annotations } func ecRuntimeConfigToBackupAnnotations(runtimeConfig *embeddedclusterv1beta1.RuntimeConfigSpec) map[string]string { From 76df58e2a7a0a2749b93bfc58474838a29d1b142 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Tue, 3 Dec 2024 09:49:19 -0800 Subject: [PATCH 14/44] f --- pkg/kotsadmsnapshot/backup_test.go | 46 ++++++++++++------------------ 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index cddc709231..d38cc7bd8e 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -5,7 +5,6 @@ import ( "testing" embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" - "github.com/replicatedhq/kots/pkg/embeddedcluster" kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -19,8 +18,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" coretest "k8s.io/client-go/testing" - kbclient "sigs.k8s.io/controller-runtime/pkg/client" - fakekbclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestPrepareIncludedNamespaces(t *testing.T) { @@ -688,20 +685,20 @@ func Test_appendECAnnotations(t *testing.T) { embeddedclusterv1beta1.AddToScheme(scheme) tests := []struct { - name string - prev map[string]string - kbClient kbclient.Client - in *embeddedclusterv1beta1.Installation - env map[string]string - want map[string]string + name string + prev map[string]string + in *embeddedclusterv1beta1.Installation + seaweedFSS3ServiceIP string + env map[string]string + want map[string]string }{ { name: "basic", prev: map[string]string{ "prev-key": "prev-value", }, - kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).Build(), - in: &embeddedclusterv1beta1.Installation{}, + in: &embeddedclusterv1beta1.Installation{}, + seaweedFSS3ServiceIP: "", env: map[string]string{ "EMBEDDED_CLUSTER_ID": "embedded-cluster-id", "EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version", @@ -715,13 +712,13 @@ func Test_appendECAnnotations(t *testing.T) { }, }, { - name: "online ha", - kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).Build(), + name: "online ha", in: &embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ HighAvailability: true, }, }, + seaweedFSS3ServiceIP: "", env: map[string]string{ "EMBEDDED_CLUSTER_ID": "embedded-cluster-id", "EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version", @@ -735,23 +732,13 @@ func Test_appendECAnnotations(t *testing.T) { }, { name: "airgap ha", - kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).WithObjects( - &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: embeddedcluster.SeaweedfsS3SVCName, - Namespace: embeddedcluster.SeaweedfsNamespace, - }, - Spec: corev1.ServiceSpec{ - ClusterIP: "10.96.0.10", - }, - }, - ).Build(), in: &embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ HighAvailability: true, AirGap: true, }, }, + seaweedFSS3ServiceIP: "10.96.0.10", env: map[string]string{ "EMBEDDED_CLUSTER_ID": "embedded-cluster-id", "EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version", @@ -765,8 +752,7 @@ func Test_appendECAnnotations(t *testing.T) { }, }, { - name: "with pod and service cidrs", - kbClient: fakekbclient.NewClientBuilder().WithScheme(scheme).Build(), + name: "with pod and service cidrs", in: &embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ Network: &embeddedclusterv1beta1.NetworkSpec{ @@ -775,6 +761,7 @@ func Test_appendECAnnotations(t *testing.T) { }, }, }, + seaweedFSS3ServiceIP: "", env: map[string]string{ "EMBEDDED_CLUSTER_ID": "embedded-cluster-id", "EMBEDDED_CLUSTER_VERSION": "embedded-cluster-version", @@ -795,8 +782,11 @@ func Test_appendECAnnotations(t *testing.T) { for k, v := range tt.env { t.Setenv(k, v) } - got, err := appendECAnnotations(context.TODO(), tt.prev, tt.kbClient, tt.in) - req.NoError(err) + ecMeta := ecInstanceBackupMetadata{ + installation: tt.in, + seaweedFSS3ServiceIP: tt.seaweedFSS3ServiceIP, + } + got := appendECAnnotations(context.TODO(), tt.prev, ecMeta) req.Equal(tt.want, got) }) } From c2588c9e860bbd025167f35857faa6674989e1fe Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Tue, 3 Dec 2024 11:24:58 -0800 Subject: [PATCH 15/44] f --- pkg/k8sutil/kotsadm.go | 7 +- pkg/kotsadmsnapshot/backup.go | 93 ++--- pkg/kotsadmsnapshot/backup_test.go | 605 +++++++++++++++++++++++++++++ pkg/snapshot/backup.go | 1 - 4 files changed, 649 insertions(+), 57 deletions(-) diff --git a/pkg/k8sutil/kotsadm.go b/pkg/k8sutil/kotsadm.go index a3b625bf50..606918ae11 100644 --- a/pkg/k8sutil/kotsadm.go +++ b/pkg/k8sutil/kotsadm.go @@ -21,12 +21,7 @@ const ( KotsadmIDConfigMapName = "kotsadm-id" ) -func FindKotsadmImage(namespace string) (string, error) { - clientset, err := GetClientset() - if err != nil { - return "", errors.Wrap(err, "failed to get k8s client set") - } - +func FindKotsadmImage(clientset kubernetes.Interface, namespace string) (string, error) { var containers []corev1.Container if os.Getenv("POD_OWNER_KIND") == "deployment" { kotsadmDeployment, err := clientset.AppsV1().Deployments(namespace).Get(context.TODO(), "kotsadm", metav1.GetOptions{}) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 01a37bab62..4b2c8bee5a 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -202,6 +202,7 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b type instanceBackupMetadata struct { backupName string + backupReqestedAt time.Time kotsadmNamespace string backupStorageLocationNamespace string apps map[string]appInstanceBackupMetadata @@ -294,8 +295,10 @@ func GetInstanceBackupsExpected(veleroBackup velerov1.Backup) int { } func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interface, veleroClient veleroclientv1.VeleroV1Interface, cluster *downstreamtypes.Downstream, isScheduled bool) (instanceBackupMetadata, error) { + now := time.Now().UTC() metadata := instanceBackupMetadata{ - backupName: fmt.Sprintf("backup-%d", time.Now().UnixNano()), + backupName: fmt.Sprintf("backup-%d", now.UnixNano()), + backupReqestedAt: now, kotsadmNamespace: util.PodNamespace, isScheduled: isScheduled, } @@ -369,11 +372,9 @@ func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interfa _ = os.RemoveAll(archiveDir) } - if util.IsEmbeddedCluster() { - metadata.ec, err = getECInstanceBackupMetadata(ctx) - if err != nil { - return metadata, errors.Wrap(err, "failed to get embedded cluster metadata") - } + metadata.ec, err = getECInstanceBackupMetadata(ctx) + if err != nil { + return metadata, errors.Wrap(err, "failed to get embedded cluster metadata") } return metadata, nil @@ -418,7 +419,7 @@ func getInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, IncludedNamespaces: []string{}, ExcludedNamespaces: []string{}, IncludeClusterResources: ptr.To(true), - OrLabelSelectors: instanceBackupLabelSelectors(util.IsEmbeddedCluster()), + OrLabelSelectors: instanceBackupLabelSelectors(metadata.ec != nil), OrderedResources: map[string]string{}, Hooks: velerov1.BackupHooks{ Resources: []velerov1.BackupResourceHookSpec{}, @@ -452,7 +453,7 @@ func getInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, for _, appMeta := range metadata.apps { // Don't merge the backup spec if we are using the new improved DR. if !hasAppSpec { - err := mergeAppBackupSpec(veleroBackup, appMeta.kotsKinds, appMeta.app, metadata.kotsadmNamespace) + err := mergeAppBackupSpec(veleroBackup, appMeta, metadata.kotsadmNamespace, metadata.ec != nil) if err != nil { return nil, errors.Wrap(err, "failed to merge app backup spec") } @@ -492,7 +493,7 @@ func getInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, // getAppInstanceBackup returns a backup spec only if this is Embedded Cluster and the vendor has // defined both a backup and restore custom resource. func getAppInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata) (*velerov1.Backup, error) { - if !util.IsEmbeddedCluster() { + if metadata.ec == nil { return nil, nil } @@ -571,47 +572,55 @@ func getAppInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interfac // - includedResources // - excludedResources // - labelSelector -func mergeAppBackupSpec(backup *velerov1.Backup, kotsKinds *kotsutil.KotsKinds, app *apptypes.App, kotsadmNamespace string) error { - backupSpec, err := kotsKinds.Marshal("velero.io", "v1", "Backup") +func mergeAppBackupSpec(backup *velerov1.Backup, appMeta appInstanceBackupMetadata, kotsadmNamespace string, isEC bool) error { + backupSpec, err := appMeta.kotsKinds.Marshal("velero.io", "v1", "Backup") if err != nil { return errors.Wrap(err, "failed to get backup spec from kotskinds") } - var kotskindsBackup *velerov1.Backup if backupSpec == "" { - // If this is Embedded Cluster, backups are always enabled and we will use a default - // minimal backup spec - if util.IsEmbeddedCluster() { - kotskindsBackup = getDefaultEmbeddedClusterBackupSpec() - } else { - return nil - } - } else { - renderedBackup, err := helper.RenderAppFile(app, nil, []byte(backupSpec), kotsKinds, kotsadmNamespace) - if err != nil { - return errors.Wrap(err, "failed to render backup") - } - kotskindsBackup, err = kotsutil.LoadBackupFromContents(renderedBackup) - if err != nil { - return errors.Wrap(err, "failed to load backup from contents") + // If this is Embedded Cluster, backups are always enabled and we must include the + // namespace. + if isEC { + backup.Spec.IncludedNamespaces = append(backup.Spec.IncludedNamespaces, appMeta.kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) } + return nil + } + + renderedBackup, err := helper.RenderAppFile(appMeta.app, nil, []byte(backupSpec), appMeta.kotsKinds, kotsadmNamespace) + if err != nil { + return errors.Wrap(err, "failed to render backup") + } + kotskindsBackup, err := kotsutil.LoadBackupFromContents(renderedBackup) + if err != nil { + return errors.Wrap(err, "failed to load backup from contents") } // included namespaces - backup.Spec.IncludedNamespaces = append(backup.Spec.IncludedNamespaces, kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) + backup.Spec.IncludedNamespaces = append(backup.Spec.IncludedNamespaces, appMeta.kotsKinds.KotsApplication.Spec.AdditionalNamespaces...) backup.Spec.IncludedNamespaces = append(backup.Spec.IncludedNamespaces, kotskindsBackup.Spec.IncludedNamespaces...) // excluded namespaces backup.Spec.ExcludedNamespaces = append(backup.Spec.ExcludedNamespaces, kotskindsBackup.Spec.ExcludedNamespaces...) // annotations - for k, v := range kotskindsBackup.Annotations { - backup.Annotations[k] = v + if len(kotskindsBackup.ObjectMeta.Annotations) > 0 { + if backup.Annotations == nil { + backup.Annotations = map[string]string{} + } + for k, v := range kotskindsBackup.ObjectMeta.Annotations { + backup.Annotations[k] = v + } } // ordered resources - for k, v := range kotskindsBackup.Spec.OrderedResources { - backup.Spec.OrderedResources[k] = v + if len(kotskindsBackup.Spec.OrderedResources) > 0 { + if backup.Spec.OrderedResources == nil { + backup.Spec.OrderedResources = map[string]string{} + } + for k, v := range kotskindsBackup.Spec.OrderedResources { + backup.Spec.OrderedResources[k] = v + } } // backup hooks @@ -620,22 +629,8 @@ func mergeAppBackupSpec(backup *velerov1.Backup, kotsKinds *kotsutil.KotsKinds, return nil } -// getDefaultEmbeddedClusterBackupSpec returns a default velero backup spec for an embedded -// cluster using the old DR (no backup and restore custom resource). -func getDefaultEmbeddedClusterBackupSpec() *velerov1.Backup { - return &velerov1.Backup{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "velero.io/v1", - Kind: "Backup", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "backup", - }, - } -} - func appendCommonAnnotations(ctx context.Context, k8sClient kubernetes.Interface, annotations map[string]string, metadata instanceBackupMetadata, hasAppSpec bool) (map[string]string, error) { - kotsadmImage, err := k8sutil.FindKotsadmImage(metadata.kotsadmNamespace) + kotsadmImage, err := k8sutil.FindKotsadmImage(k8sClient, metadata.kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find kotsadm image") } @@ -672,13 +667,11 @@ func appendCommonAnnotations(ctx context.Context, k8sClient kubernetes.Interface numBackups = 2 } - now := time.Now() // TODO - if annotations == nil { annotations = make(map[string]string, 0) } annotations["kots.io/snapshot-trigger"] = snapshotTrigger - annotations["kots.io/snapshot-requested"] = now.UTC().Format(time.RFC3339) + annotations["kots.io/snapshot-requested"] = metadata.backupReqestedAt.Format(time.RFC3339) annotations["kots.io/instance"] = "true" annotations["kots.io/kotsadm-image"] = kotsadmImage annotations["kots.io/kotsadm-deploy-namespace"] = metadata.kotsadmNamespace diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index d38cc7bd8e..9b1f69fd18 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -3,12 +3,21 @@ package snapshot import ( "context" "testing" + "time" + gomock "github.com/golang/mock/gomock" embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + apptypes "github.com/replicatedhq/kots/pkg/app/types" kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" + "github.com/replicatedhq/kots/pkg/kotsutil" + registrytypes "github.com/replicatedhq/kots/pkg/registry/types" + "github.com/replicatedhq/kots/pkg/store" + mock_store "github.com/replicatedhq/kots/pkg/store/mock" + kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -859,3 +868,599 @@ func Test_ecIncludedNamespaces(t *testing.T) { }) } } + +func Test_appendCommonAnnotations(t *testing.T) { + scheme := runtime.NewScheme() + corev1.AddToScheme(scheme) + embeddedclusterv1beta1.AddToScheme(scheme) + + kotsadmSts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm", + Namespace: "kotsadm", + }, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kotsadm", + Image: "kotsadm/kotsadm:1.0.0", + }, + }, + }, + }, + }, + } + registryCredsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "registry-creds", + Namespace: "kotsadm", + }, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + ".dockerconfigjson": []byte(`{"auths":{"host":{"username":"kurl","password":"password"}}}`), + }, + } + + type args struct { + k8sClient kubernetes.Interface + annotations map[string]string + metadata instanceBackupMetadata + hasAppSpec bool + } + tests := []struct { + name string + setup func(t *testing.T) + args args + want map[string]string + wantErr bool + }{ + { + name: "cli install, airgap, multiple apps, not scheduled, has ttl", + setup: func(t *testing.T) { + t.Setenv("DISABLE_OUTBOUND_CONNECTIONS", "true") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + annotations: map[string]string{}, + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: &apptypes.App{}, + kotsKinds: &kotsutil.KotsKinds{ + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + }, + parentSequence: 1, + }, + "app-2": { + app: &apptypes.App{}, + kotsKinds: &kotsutil.KotsKinds{ + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.2", + }, + }, + }, + parentSequence: 2, + }, + }, + isScheduled: false, + snapshotTTL: 24 * time.Hour, + ec: nil, + }, + hasAppSpec: false, + }, + want: map[string]string{ + "kots.io/apps-sequences": "{\"app-1\":1,\"app-2\":2}", + "kots.io/apps-versions": "{\"app-1\":\"1.0.1\",\"app-2\":\"1.0.2\"}", + "kots.io/embedded-registry": "host", + "kots.io/instance": "true", + "kots.io/is-airgap": "true", + "kots.io/kotsadm-deploy-namespace": "kotsadm", + "kots.io/kotsadm-image": "kotsadm/kotsadm:1.0.0", + "kots.io/snapshot-requested": "2024-01-01T00:00:00Z", + "kots.io/snapshot-trigger": "manual", + "replicated.com/backup-name": "backup-17332487841234", + "replicated.com/backups-expected": "1", + }, + }, + { + name: "ec install, scheduled, no ttl, improved dr", + setup: func(t *testing.T) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + annotations: map[string]string{}, + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: &apptypes.App{}, + kotsKinds: &kotsutil.KotsKinds{ + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + }, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: &ecInstanceBackupMetadata{ + installation: &embeddedclusterv1beta1.Installation{ + Spec: embeddedclusterv1beta1.InstallationSpec{ + HighAvailability: true, + Network: &embeddedclusterv1beta1.NetworkSpec{ + PodCIDR: "10.128.0.0/20", + ServiceCIDR: "10.129.0.0/20", + }, + RuntimeConfig: &embeddedclusterv1beta1.RuntimeConfigSpec{ + DataDir: "/var/lib/ec", + AdminConsole: embeddedclusterv1beta1.AdminConsoleSpec{ + Port: 30001, + }, + LocalArtifactMirror: embeddedclusterv1beta1.LocalArtifactMirrorSpec{ + Port: 50001, + }, + }, + }, + }, + seaweedFSS3ServiceIP: "10.96.0.10", + }, + }, + hasAppSpec: true, + }, + want: map[string]string{ + "kots.io/apps-sequences": "{\"app-1\":1}", + "kots.io/apps-versions": "{\"app-1\":\"1.0.1\"}", + "kots.io/embedded-registry": "host", + "kots.io/instance": "true", + "kots.io/is-airgap": "false", + "kots.io/kotsadm-deploy-namespace": "kotsadm", + "kots.io/kotsadm-image": "kotsadm/kotsadm:1.0.0", + "kots.io/snapshot-requested": "2024-01-01T00:00:00Z", + "kots.io/snapshot-trigger": "schedule", + "replicated.com/backup-name": "backup-17332487841234", + "replicated.com/backups-expected": "2", + "kots.io/embedded-cluster": "true", + "kots.io/embedded-cluster-id": "embedded-cluster-id", + "kots.io/embedded-cluster-version": "embedded-cluster-version", + "kots.io/embedded-cluster-is-ha": "true", + "kots.io/embedded-cluster-pod-cidr": "10.128.0.0/20", + "kots.io/embedded-cluster-service-cidr": "10.129.0.0/20", + "kots.io/embedded-cluster-seaweedfs-s3-ip": "10.96.0.10", + "kots.io/embedded-cluster-admin-console-port": "30001", + "kots.io/embedded-cluster-local-artifact-mirror-port": "50001", + "kots.io/embedded-cluster-data-dir": "/var/lib/ec", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setup != nil { + tt.setup(t) + } + got, err := appendCommonAnnotations(context.Background(), tt.args.k8sClient, tt.args.annotations, tt.args.metadata, tt.args.hasAppSpec) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_mergeAppBackupSpec(t *testing.T) { + type args struct { + backup *velerov1.Backup + appMeta appInstanceBackupMetadata + kotsadmNamespace string + isEC bool + } + tests := []struct { + name string + setup func(t *testing.T, mockStore *mock_store.MockStore) + args args + want *velerov1.Backup + wantErr bool + }{ + { + name: "no backup spec", + args: args{ + backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm"}, + }, + }, + appMeta: appInstanceBackupMetadata{ + app: &apptypes.App{ + ID: "1", + Slug: "app-1", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + Spec: kotsv1beta1.ApplicationSpec{ + AdditionalNamespaces: []string{"another-namespace-1", "another-namespace-2"}, + }, + }, + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + }, + parentSequence: 1, + }, + kotsadmNamespace: "kotsadm", + isEC: false, + }, + want: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm"}, + }, + }, + }, + { + name: "has backup spec", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + mockStore.EXPECT().GetLatestAppSequence("1", true).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Return(registrytypes.RegistrySettings{ + Hostname: "hostname", + Username: "username", + Password: "password", + Namespace: "namespace", + IsReadOnly: true, + }, nil) + }, + args: args{ + backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm"}, + }, + }, + appMeta: appInstanceBackupMetadata{ + app: &apptypes.App{ + ID: "1", + Slug: "app-1", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + Spec: kotsv1beta1.ApplicationSpec{ + AdditionalNamespaces: []string{"another-namespace-1", "another-namespace-2"}, + }, + }, + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + Backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "annotation-1": "true", + "annotation-2": "false", + }, + }, + Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"include-namespace-1", "include-namespace-2", "template-isairgap-{{repl IsAirgap }}"}, + ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, + OrderedResources: map[string]string{ + "resource-1": "true", + "resource-2": "false", + }, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{ + { + Name: "hook-1", + }, + { + Name: "hook-2", + }, + }, + }, + }, + }, + }, + parentSequence: 1, + }, + kotsadmNamespace: "kotsadm", + isEC: false, + }, + want: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + "annotation-1": "true", + "annotation-2": "false", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm", "another-namespace-1", "another-namespace-2", "include-namespace-1", "include-namespace-2", "template-isairgap-true"}, + ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, + OrderedResources: map[string]string{ + "resource-1": "true", + "resource-2": "false", + }, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{ + { + Name: "hook-1", + }, + { + Name: "hook-2", + }, + }, + }, + }, + }, + }, + { + name: "ec, no backup spec", + args: args{ + backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm"}, + }, + }, + appMeta: appInstanceBackupMetadata{ + app: &apptypes.App{ + ID: "1", + Slug: "app-1", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + Spec: kotsv1beta1.ApplicationSpec{ + AdditionalNamespaces: []string{"another-namespace-1", "another-namespace-2"}, + }, + }, + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + }, + parentSequence: 1, + }, + kotsadmNamespace: "kotsadm", + isEC: true, + }, + want: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm", "another-namespace-1", "another-namespace-2"}, + }, + }, + }, + { + name: "ec, has backup spec", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + mockStore.EXPECT().GetLatestAppSequence("1", true).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Return(registrytypes.RegistrySettings{ + Hostname: "hostname", + Username: "username", + Password: "password", + Namespace: "namespace", + IsReadOnly: true, + }, nil) + }, + args: args{ + backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm"}, + }, + }, + appMeta: appInstanceBackupMetadata{ + app: &apptypes.App{ + ID: "1", + Slug: "app-1", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + Spec: kotsv1beta1.ApplicationSpec{ + AdditionalNamespaces: []string{"another-namespace-1", "another-namespace-2"}, + }, + }, + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + Backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "annotation-1": "true", + "annotation-2": "false", + }, + }, + Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"include-namespace-1", "include-namespace-2", "template-isairgap-{{repl IsAirgap }}"}, + ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, + OrderedResources: map[string]string{ + "resource-1": "true", + "resource-2": "false", + }, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{ + { + Name: "hook-1", + }, + { + Name: "hook-2", + }, + }, + }, + }, + }, + }, + parentSequence: 1, + }, + kotsadmNamespace: "kotsadm", + isEC: true, + }, + want: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "instance-", + Annotations: map[string]string{ + "annotation": "true", + "annotation-1": "true", + "annotation-2": "false", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"kotsadm", "another-namespace-1", "another-namespace-2", "include-namespace-1", "include-namespace-2", "template-isairgap-true"}, + ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, + OrderedResources: map[string]string{ + "resource-1": "true", + "resource-2": "false", + }, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{ + { + Name: "hook-1", + }, + { + Name: "hook-2", + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStore := mock_store.NewMockStore(ctrl) + store.SetStore(mockStore) + + t.Cleanup(func() { + store.SetStore(nil) + }) + + if tt.setup != nil { + tt.setup(t, mockStore) + } + err := mergeAppBackupSpec(tt.args.backup, tt.args.appMeta, tt.args.kotsadmNamespace, tt.args.isEC) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tt.want, tt.args.backup) + }) + } +} diff --git a/pkg/snapshot/backup.go b/pkg/snapshot/backup.go index 4404f4fb73..0ef92bfa41 100644 --- a/pkg/snapshot/backup.go +++ b/pkg/snapshot/backup.go @@ -166,7 +166,6 @@ func CreateInstanceBackup(ctx context.Context, options CreateInstanceBackupOptio } func ListInstanceBackups(ctx context.Context, options ListInstanceBackupsOptions) ([]velerov1.Backup, error) { - b, err := ListAllBackups(ctx, options) if err != nil { return nil, errors.Wrap(err, "failed to get backup list") From 0e922e911c2dc21ff23f0ebad4928572be1f03fd Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Tue, 3 Dec 2024 12:48:46 -0800 Subject: [PATCH 16/44] f --- pkg/kotsadmsnapshot/backup.go | 54 +++-- pkg/kotsadmsnapshot/backup_test.go | 326 ++++++++++++++++++++++++++++- 2 files changed, 358 insertions(+), 22 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 4b2c8bee5a..5bb2d2b14b 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -255,6 +255,18 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return "", errors.Wrap(err, "failed to get instance backup specs") } + err = excludeShutdownPodsFromBackup(ctx, k8sClient, veleroBackup) + if err != nil { + logger.Errorf("Failed to exclude shutdown pods from backup: %v", err) + } + + if appVeleroBackup != nil { + err = excludeShutdownPodsFromBackup(ctx, k8sClient, appVeleroBackup) + if err != nil { + logger.Errorf("Failed to exclude shutdown pods from application backup: %v", err) + } + } + _, err = veleroClient.Backups(metadata.backupStorageLocationNamespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) if err != nil { return "", errors.Wrap(err, "failed to create velero backup") @@ -294,6 +306,8 @@ func GetInstanceBackupsExpected(veleroBackup velerov1.Backup) int { return 1 } +// getInstanceBackupMetadata returns metadata about the instance backup for use in creating an +// instance backup. func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interface, veleroClient veleroclientv1.VeleroV1Interface, cluster *downstreamtypes.Downstream, isScheduled bool) (instanceBackupMetadata, error) { now := time.Now().UTC() metadata := instanceBackupMetadata{ @@ -380,6 +394,8 @@ func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interfa return metadata, nil } +// getECInstanceBackupMetadata returns metadata about the embedded cluster for use in creating an +// instance backup. func getECInstanceBackupMetadata(ctx context.Context) (*ecInstanceBackupMetadata, error) { if !util.IsEmbeddedCluster() { return nil, nil @@ -406,6 +422,8 @@ func getECInstanceBackupMetadata(ctx context.Context) (*ecInstanceBackupMetadata }, nil } +// getInstanceBackupSpec returns the velero backup spec for the instance backup. This is either the +// kotsadm backup or the combined backup if this is not using improved DR. func getInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata, hasAppSpec bool) (*velerov1.Backup, error) { // veleroBackup is the kotsadm backup or combined backup if usesImprovedDR is false veleroBackup := &velerov1.Backup{ @@ -480,18 +498,13 @@ func getInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, } } - err = excludeShutdownPodsFromBackup(ctx, k8sClient, veleroBackup) - if err != nil { - logger.Errorf("Failed to exclude shutdown pods from backup: %v", err) - } - veleroBackup.Spec.IncludedNamespaces = prepareIncludedNamespaces(veleroBackup.Spec.IncludedNamespaces) return veleroBackup, nil } // getAppInstanceBackup returns a backup spec only if this is Embedded Cluster and the vendor has -// defined both a backup and restore custom resource. +// defined both a backup and restore custom resource (improved DR). func getAppInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata) (*velerov1.Backup, error) { if metadata.ec == nil { return nil, nil @@ -505,7 +518,7 @@ func getAppInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interfac continue } - if len(metadata.apps) > 0 { + if len(metadata.apps) > 1 { return nil, errors.New("cannot create backup for Embedded Cluster with multiple apps") } @@ -552,17 +565,14 @@ func getAppInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interfac } appVeleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeApp + appVeleroBackup.Spec.StorageLocation = "default" + if metadata.snapshotTTL > 0 { appVeleroBackup.Spec.TTL = metav1.Duration{ Duration: metadata.snapshotTTL, } } - err = excludeShutdownPodsFromBackup(ctx, k8sClient, appVeleroBackup) - if err != nil { - logger.Errorf("Failed to exclude shutdown pods from application backup: %v", err) - } - return appVeleroBackup, nil } @@ -629,6 +639,7 @@ func mergeAppBackupSpec(backup *velerov1.Backup, appMeta appInstanceBackupMetada return nil } +// appendCommonAnnotations appends common annotations to the backup annotations func appendCommonAnnotations(ctx context.Context, k8sClient kubernetes.Interface, annotations map[string]string, metadata instanceBackupMetadata, hasAppSpec bool) (map[string]string, error) { kotsadmImage, err := k8sutil.FindKotsadmImage(k8sClient, metadata.kotsadmNamespace) if err != nil { @@ -1340,17 +1351,24 @@ func excludeShutdownPodsFromBackup(ctx context.Context, clientset kubernetes.Int labelSets = append(labelSets, orLabelSet...) } - for _, labelSet := range labelSets { + for _, namespace := range veleroBackup.Spec.IncludedNamespaces { + if namespace == "*" { + namespace = "" // specifying an empty ("") namespace in client-go retrieves resources from all namespaces + } + podListOption := metav1.ListOptions{ - LabelSelector: labelSet, FieldSelector: fields.SelectorFromSet(selectorMap).String(), } - for _, namespace := range veleroBackup.Spec.IncludedNamespaces { - if namespace == "*" { - namespace = "" // specifying an empty ("") namespace in client-go retrieves resources from all namespaces - } + if len(labelSets) > 0 { + for _, labelSet := range labelSets { + podListOption.LabelSelector = labelSet + if err := excludeShutdownPodsFromBackupInNamespace(ctx, clientset, namespace, podListOption); err != nil { + return errors.Wrap(err, "failed to exclude shutdown pods from backup") + } + } + } else { if err := excludeShutdownPodsFromBackupInNamespace(ctx, clientset, namespace, podListOption); err != nil { return errors.Wrap(err, "failed to exclude shutdown pods from backup") } diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 9b1f69fd18..232715beb2 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -184,6 +184,17 @@ func mockK8sClientWithShutdownPods() kubernetes.Interface { Reason: "Shutdown", }, }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shutdown-no-label", + Namespace: "test", + Labels: map[string]string{}, + }, + Status: corev1.PodStatus{ + Phase: "Failed", + Reason: "Shutdown", + }, + }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup-shutdown", @@ -211,6 +222,16 @@ func mockK8sClientWithShutdownPods() kubernetes.Interface { Phase: "Running", }, }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-running-no-label", + Namespace: "test", + Labels: map[string]string{}, + }, + Status: corev1.PodStatus{ + Phase: "Running", + }, + }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "test-backup-running", @@ -321,6 +342,19 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { wantErr: false, wantNumOfPodsWithExcludeAnnotation: 1, }, + { + name: "expect no error when shutdown pods are found and updated for no label selector", + args: args{ + ctx: context.TODO(), + clientset: mockK8sClientWithShutdownPods(), + namespace: "test", + failedPodListOptions: metav1.ListOptions{ + FieldSelector: fields.SelectorFromSet(selectorMap).String(), + }, + }, + wantErr: false, + wantNumOfPodsWithExcludeAnnotation: 2, + }, { name: "expect no error when shutdown pods are found and updated for app slug label", args: args{ @@ -487,6 +521,19 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { }, wantErr: false, }, + { + name: "expect no error when shutdown pods are found and updated for app slug label and no label selector", + args: args{ + ctx: context.TODO(), + clientset: mockK8sClientWithShutdownPods(), + veleroBackup: &velerov1.Backup{ + Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"test"}, + }, + }, + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -870,10 +917,6 @@ func Test_ecIncludedNamespaces(t *testing.T) { } func Test_appendCommonAnnotations(t *testing.T) { - scheme := runtime.NewScheme() - corev1.AddToScheme(scheme) - embeddedclusterv1beta1.AddToScheme(scheme) - kotsadmSts := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: "kotsadm", @@ -1464,3 +1507,278 @@ func Test_mergeAppBackupSpec(t *testing.T) { }) } } + +func Test_getAppInstanceBackupSpec(t *testing.T) { + kotsadmSts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm", + Namespace: "kotsadm", + }, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kotsadm", + Image: "kotsadm/kotsadm:1.0.0", + }, + }, + }, + }, + }, + } + registryCredsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "registry-creds", + Namespace: "kotsadm", + }, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + ".dockerconfigjson": []byte(`{"auths":{"host":{"username":"kurl","password":"password"}}}`), + }, + } + + type args struct { + k8sClient kubernetes.Interface + metadata instanceBackupMetadata + } + tests := []struct { + name string + setup func(t *testing.T, mockStore *mock_store.MockStore) + args args + want *velerov1.Backup + wantErr bool + }{ + { + name: "not ec", + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: &apptypes.App{}, + kotsKinds: &kotsutil.KotsKinds{ + Backup: &velerov1.Backup{}, + Restore: &velerov1.Restore{}, + }, + parentSequence: 1, + }, + }, + ec: nil, + }, + }, + want: nil, + }, + { + name: "ec, no restore spec", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: &apptypes.App{}, + kotsKinds: &kotsutil.KotsKinds{ + Backup: &velerov1.Backup{}, + }, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: &ecInstanceBackupMetadata{}, + }, + }, + want: nil, + }, + { + name: "ec, improved DR", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStore.EXPECT().GetLatestAppSequence("1", true).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Return(registrytypes.RegistrySettings{ + Hostname: "hostname", + Username: "username", + Password: "password", + Namespace: "namespace", + IsReadOnly: true, + }, nil) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: &apptypes.App{ + ID: "1", + Slug: "app-1", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + Spec: kotsv1beta1.ApplicationSpec{ + AdditionalNamespaces: []string{"another-namespace-1", "another-namespace-2"}, + }, + }, + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + Backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-backup", + Annotations: map[string]string{ + "annotation-1": "true", + "annotation-2": "false", + }, + }, + Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"include-namespace-1", "include-namespace-2", "template-isairgap-{{repl IsAirgap }}"}, + ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, + OrderedResources: map[string]string{ + "resource-1": "true", + "resource-2": "false", + }, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{ + { + Name: "hook-1", + }, + { + Name: "hook-2", + }, + }, + }, + }, + }, + Restore: &velerov1.Restore{}, + }, + parentSequence: 1, + }, + }, + isScheduled: true, + snapshotTTL: 24 * time.Hour, + ec: &ecInstanceBackupMetadata{ + installation: &embeddedclusterv1beta1.Installation{ + Spec: embeddedclusterv1beta1.InstallationSpec{ + HighAvailability: true, + Network: &embeddedclusterv1beta1.NetworkSpec{ + PodCIDR: "10.128.0.0/20", + ServiceCIDR: "10.129.0.0/20", + }, + RuntimeConfig: &embeddedclusterv1beta1.RuntimeConfigSpec{ + DataDir: "/var/lib/ec", + AdminConsole: embeddedclusterv1beta1.AdminConsoleSpec{ + Port: 30001, + }, + LocalArtifactMirror: embeddedclusterv1beta1.LocalArtifactMirrorSpec{ + Port: 50001, + }, + }, + }, + }, + seaweedFSS3ServiceIP: "10.96.0.10", + }, + }, + }, + want: &velerov1.Backup{ + // TODO: excludeShutdownPodsFromBackup + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "", + GenerateName: "app-1-", + Annotations: map[string]string{ + "annotation-1": "true", + "annotation-2": "false", + "kots.io/apps-sequences": "{\"app-1\":1}", + "kots.io/apps-versions": "{\"app-1\":\"1.0.1\"}", + "kots.io/embedded-registry": "host", + "kots.io/instance": "true", + "kots.io/is-airgap": "false", + "kots.io/kotsadm-deploy-namespace": "kotsadm", + "kots.io/kotsadm-image": "kotsadm/kotsadm:1.0.0", + "kots.io/snapshot-requested": "2024-01-01T00:00:00Z", + "kots.io/snapshot-trigger": "schedule", + "replicated.com/backup-name": "backup-17332487841234", + "replicated.com/backup-type": "app", + "replicated.com/backups-expected": "2", + "kots.io/embedded-cluster": "true", + "kots.io/embedded-cluster-id": "embedded-cluster-id", + "kots.io/embedded-cluster-version": "embedded-cluster-version", + "kots.io/embedded-cluster-is-ha": "true", + "kots.io/embedded-cluster-pod-cidr": "10.128.0.0/20", + "kots.io/embedded-cluster-service-cidr": "10.129.0.0/20", + "kots.io/embedded-cluster-seaweedfs-s3-ip": "10.96.0.10", + "kots.io/embedded-cluster-admin-console-port": "30001", + "kots.io/embedded-cluster-local-artifact-mirror-port": "50001", + "kots.io/embedded-cluster-data-dir": "/var/lib/ec", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{"include-namespace-1", "include-namespace-2", "template-isairgap-true"}, + ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, + OrderedResources: map[string]string{ + "resource-1": "true", + "resource-2": "false", + }, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{ + { + Name: "hook-1", + }, + { + Name: "hook-2", + }, + }, + }, + TTL: metav1.Duration{24 * time.Hour}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStore := mock_store.NewMockStore(ctrl) + store.SetStore(mockStore) + + t.Cleanup(func() { + store.SetStore(nil) + }) + + if tt.setup != nil { + tt.setup(t, mockStore) + } + got, err := getAppInstanceBackupSpec(context.Background(), tt.args.k8sClient, tt.args.metadata) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tt.want, got) + }) + } +} From 16f2d4212e2fa992317292884717135cda480ae3 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Tue, 3 Dec 2024 13:06:19 -0800 Subject: [PATCH 17/44] f --- pkg/kotsadmsnapshot/backup.go | 4 +-- pkg/kotsadmsnapshot/backup_test.go | 41 +++++++++++++++--------------- pkg/kotsadmsnapshot/restore.go | 1 + 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 5bb2d2b14b..ff50b8883e 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -699,7 +699,7 @@ func appendCommonAnnotations(ctx context.Context, k8sClient kubernetes.Interface annotations[InstanceBackupsExpectedAnnotation] = strconv.Itoa(numBackups) if metadata.ec != nil { - annotations = appendECAnnotations(ctx, annotations, *metadata.ec) + annotations = appendECAnnotations(annotations, *metadata.ec) } return annotations, nil @@ -1250,7 +1250,7 @@ func mergeLabelSelector(kots metav1.LabelSelector, app metav1.LabelSelector) met } // appendECAnnotations appends annotations that should be added to an embedded cluster backup -func appendECAnnotations(ctx context.Context, annotations map[string]string, ecMeta ecInstanceBackupMetadata) map[string]string { +func appendECAnnotations(annotations map[string]string, ecMeta ecInstanceBackupMetadata) map[string]string { if annotations == nil { annotations = make(map[string]string, 0) } diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 232715beb2..64089212c1 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -303,7 +303,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { { name: "expect error when k8s client list pod returns error", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockGetPodsInANamespaceErrorClient(), namespace: "test", failedPodListOptions: kotsadmPodListOption, @@ -313,7 +313,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { { name: "expect error when k8s client update shutdown pod returns error", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockUpdateShutdownPodErrorClient(), namespace: "test", failedPodListOptions: kotsadmPodListOption, @@ -323,7 +323,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { { name: "expect no error when no shutdown pods are found", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockGetRunningPodsClient(), namespace: "test", failedPodListOptions: kotsadmPodListOption, @@ -334,7 +334,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for kotsadm backup label", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), namespace: "test", failedPodListOptions: kotsadmPodListOption, @@ -345,7 +345,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for no label selector", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), namespace: "test", failedPodListOptions: metav1.ListOptions{ @@ -358,7 +358,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for app slug label", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), namespace: "test-2", failedPodListOptions: appSlugPodListOption, @@ -369,7 +369,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for app slug label with all namespaces", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), namespace: "", failedPodListOptions: appSlugPodListOption, @@ -380,7 +380,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for kotsadm backup label with all namespaces", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), namespace: "", failedPodListOptions: kotsadmPodListOption, @@ -398,7 +398,7 @@ func Test_excludeShutdownPodsFromBackupInNamespace(t *testing.T) { foundNumofPodsWithExcludeAnnotation := 0 if !tt.wantErr { // get pods in test namespace and check if they have the velero exclude annotation for Shutdown pods - pods, err := tt.args.clientset.CoreV1().Pods(tt.args.namespace).List(context.TODO(), tt.args.failedPodListOptions) + pods, err := tt.args.clientset.CoreV1().Pods(tt.args.namespace).List(context.Background(), tt.args.failedPodListOptions) if err != nil { t.Errorf("excludeShutdownPodsFromBackupInNamespace() error = %v, wantErr %v", err, tt.wantErr) } @@ -440,7 +440,7 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { { name: "expect no error when namespaces are empty", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), veleroBackup: &velerov1.Backup{ Spec: velerov1.BackupSpec{ @@ -454,7 +454,7 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { { name: "expect no error when pods are running", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockGetRunningPodsClient(), veleroBackup: &velerov1.Backup{ Spec: velerov1.BackupSpec{ @@ -468,7 +468,7 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { { name: "expect error when k8s client list pods returns error", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockGetPodsInANamespaceErrorClient(), veleroBackup: &velerov1.Backup{ Spec: velerov1.BackupSpec{ @@ -482,7 +482,7 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for app slug label", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), veleroBackup: &velerov1.Backup{ Spec: velerov1.BackupSpec{ @@ -496,7 +496,7 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for kotsadm backup label and namespace is *", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), veleroBackup: &velerov1.Backup{ Spec: velerov1.BackupSpec{ @@ -510,7 +510,7 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for app slug match expression", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), veleroBackup: &velerov1.Backup{ Spec: velerov1.BackupSpec{ @@ -524,7 +524,7 @@ func Test_excludeShutdownPodsFromBackup(t *testing.T) { { name: "expect no error when shutdown pods are found and updated for app slug label and no label selector", args: args{ - ctx: context.TODO(), + ctx: context.Background(), clientset: mockK8sClientWithShutdownPods(), veleroBackup: &velerov1.Backup{ Spec: velerov1.BackupSpec{ @@ -659,11 +659,11 @@ func Test_excludeShutdownPodsFromBackup_check(t *testing.T) { req := require.New(t) mockClient := fake.NewSimpleClientset(tt.resources...) - err := excludeShutdownPodsFromBackup(context.TODO(), mockClient, tt.args.veleroBackup) + err := excludeShutdownPodsFromBackup(context.Background(), mockClient, tt.args.veleroBackup) req.NoError(err) // count the number of pods with exclude annotation - testPods, err := mockClient.CoreV1().Pods("test").List(context.TODO(), metav1.ListOptions{}) + testPods, err := mockClient.CoreV1().Pods("test").List(context.Background(), metav1.ListOptions{}) req.NoError(err) foundExcluded := []string{} @@ -842,7 +842,7 @@ func Test_appendECAnnotations(t *testing.T) { installation: tt.in, seaweedFSS3ServiceIP: tt.seaweedFSS3ServiceIP, } - got := appendECAnnotations(context.TODO(), tt.prev, ecMeta) + got := appendECAnnotations(context.Background(), tt.prev, ecMeta) req.Equal(tt.want, got) }) } @@ -1699,7 +1699,6 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { }, }, want: &velerov1.Backup{ - // TODO: excludeShutdownPodsFromBackup TypeMeta: metav1.TypeMeta{ APIVersion: "velero.io/v1", Kind: "Backup", @@ -1752,7 +1751,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { }, }, }, - TTL: metav1.Duration{24 * time.Hour}, + TTL: metav1.Duration{Duration: 24 * time.Hour}, }, }, }, diff --git a/pkg/kotsadmsnapshot/restore.go b/pkg/kotsadmsnapshot/restore.go index 71192d0d62..7d4d23d5c6 100644 --- a/pkg/kotsadmsnapshot/restore.go +++ b/pkg/kotsadmsnapshot/restore.go @@ -127,6 +127,7 @@ func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snap return errors.Wrap(err, "failed to create restore") } + // TODO // create the kotsKinds.Restore included with the yaml from the vendor // Add the EC annotation From 20fdea9ddb1a24ad6e4cd8e0140309e74cd8b09f Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Tue, 3 Dec 2024 13:06:36 -0800 Subject: [PATCH 18/44] f --- pkg/kotsadmsnapshot/backup_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 64089212c1..48db438000 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -842,7 +842,7 @@ func Test_appendECAnnotations(t *testing.T) { installation: tt.in, seaweedFSS3ServiceIP: tt.seaweedFSS3ServiceIP, } - got := appendECAnnotations(context.Background(), tt.prev, ecMeta) + got := appendECAnnotations(tt.prev, ecMeta) req.Equal(tt.want, got) }) } From bb26741af50cc740f1e0c2f3dc36b3471fe76bd1 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Tue, 3 Dec 2024 13:52:11 -0800 Subject: [PATCH 19/44] f --- pkg/kotsutil/kots.go | 12 +++++++++++- pkg/store/kotsstore/version_store.go | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/kotsutil/kots.go b/pkg/kotsutil/kots.go index 5bb7e6adbe..4b69b7f5a8 100644 --- a/pkg/kotsutil/kots.go +++ b/pkg/kotsutil/kots.go @@ -438,7 +438,8 @@ func (o KotsKinds) Marshal(g string, v string, k string) (string, error) { if g == "velero.io" { if v == "v1" { - if k == "Backup" { + switch k { + case "Backup": if o.Backup == nil { return "", nil } @@ -447,6 +448,15 @@ func (o KotsKinds) Marshal(g string, v string, k string) (string, error) { return "", errors.Wrap(err, "failed to encode backup") } return string(b.Bytes()), nil + case "Restore": + if o.Restore == nil { + return "", nil + } + var b bytes.Buffer + if err := s.Encode(o.Restore, &b); err != nil { + return "", errors.Wrap(err, "failed to encode restore") + } + return string(b.Bytes()), nil } } } diff --git a/pkg/store/kotsstore/version_store.go b/pkg/store/kotsstore/version_store.go index ea16f66cc2..eca2791be0 100644 --- a/pkg/store/kotsstore/version_store.go +++ b/pkg/store/kotsstore/version_store.go @@ -616,7 +616,7 @@ func (s *KOTSStore) upsertAppVersionRecordStatements(appID string, sequence int6 } restoreSpec, err := kotsKinds.Marshal("velero.io", "v1", "Restore") if err != nil { - return nil, errors.Wrap(err, "failed to marshal backup spec") + return nil, errors.Wrap(err, "failed to marshal restore spec") } identitySpec, err := kotsKinds.Marshal("kots.io", "v1beta1", "Identity") if err != nil { From ee09deea6bbc6b3f5f4acd2c461f7db27b1a4c27 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Tue, 3 Dec 2024 15:02:58 -0800 Subject: [PATCH 20/44] f --- deploy/assets/postgres/tables/app_version.yaml | 2 -- migrations/tables/app_version.yaml | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/assets/postgres/tables/app_version.yaml b/deploy/assets/postgres/tables/app_version.yaml index a3f95182fb..32c488180a 100644 --- a/deploy/assets/postgres/tables/app_version.yaml +++ b/deploy/assets/postgres/tables/app_version.yaml @@ -66,8 +66,6 @@ spec: type: text - name: backup_spec type: text - - name: restore_spec - type: text - name: identity_spec type: text - name: branding_archive diff --git a/migrations/tables/app_version.yaml b/migrations/tables/app_version.yaml index f24b8116d2..b9d2054800 100644 --- a/migrations/tables/app_version.yaml +++ b/migrations/tables/app_version.yaml @@ -67,6 +67,8 @@ spec: type: text - name: backup_spec type: text + - name: restore_spec + type: text - name: identity_spec type: text - name: branding_archive From ac702acedab847834393e768837e810acba8e22c Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Tue, 3 Dec 2024 15:21:53 -0800 Subject: [PATCH 21/44] f --- pkg/kotsadmsnapshot/backup.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index ff50b8883e..9f63a8f253 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -314,6 +314,7 @@ func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interfa backupName: fmt.Sprintf("backup-%d", now.UnixNano()), backupReqestedAt: now, kotsadmNamespace: util.PodNamespace, + apps: make(map[string]appInstanceBackupMetadata, 0), isScheduled: isScheduled, } From becffdb11ba60671441a7f972aeb9f965c03cd8e Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Tue, 3 Dec 2024 16:04:55 -0800 Subject: [PATCH 22/44] f --- pkg/kotsadmsnapshot/backup.go | 23 +- pkg/kotsadmsnapshot/backup_test.go | 368 ++++++++++++++++++++++++++++- 2 files changed, 374 insertions(+), 17 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 9f63a8f253..e9c6534fc0 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -36,6 +36,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes" "k8s.io/utils/ptr" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -235,12 +236,17 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return "", errors.Wrap(err, "failed to create clientset") } + ctrlClient, err := k8sutil.GetKubeClient(ctx) + if err != nil { + return "", fmt.Errorf("failed to get kubeclient: %w", err) + } + veleroClient, err := veleroclientv1.NewForConfig(cfg) if err != nil { return "", errors.Wrap(err, "failed to create velero clientset") } - metadata, err := getInstanceBackupMetadata(ctx, k8sClient, veleroClient, cluster, isScheduled) + metadata, err := getInstanceBackupMetadata(ctx, k8sClient, ctrlClient, veleroClient, cluster, isScheduled) if err != nil { return "", errors.Wrap(err, "failed to get instance backup metadata") } @@ -308,7 +314,7 @@ func GetInstanceBackupsExpected(veleroBackup velerov1.Backup) int { // getInstanceBackupMetadata returns metadata about the instance backup for use in creating an // instance backup. -func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interface, veleroClient veleroclientv1.VeleroV1Interface, cluster *downstreamtypes.Downstream, isScheduled bool) (instanceBackupMetadata, error) { +func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interface, ctrlClient ctrlclient.Client, veleroClient veleroclientv1.VeleroV1Interface, cluster *downstreamtypes.Downstream, isScheduled bool) (instanceBackupMetadata, error) { now := time.Now().UTC() metadata := instanceBackupMetadata{ backupName: fmt.Sprintf("backup-%d", now.UnixNano()), @@ -387,7 +393,7 @@ func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interfa _ = os.RemoveAll(archiveDir) } - metadata.ec, err = getECInstanceBackupMetadata(ctx) + metadata.ec, err = getECInstanceBackupMetadata(ctx, ctrlClient) if err != nil { return metadata, errors.Wrap(err, "failed to get embedded cluster metadata") } @@ -397,22 +403,17 @@ func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interfa // getECInstanceBackupMetadata returns metadata about the embedded cluster for use in creating an // instance backup. -func getECInstanceBackupMetadata(ctx context.Context) (*ecInstanceBackupMetadata, error) { +func getECInstanceBackupMetadata(ctx context.Context, ctrlClient ctrlclient.Client) (*ecInstanceBackupMetadata, error) { if !util.IsEmbeddedCluster() { return nil, nil } - kbClient, err := k8sutil.GetKubeClient(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get kubeclient: %w", err) - } - - installation, err := embeddedcluster.GetCurrentInstallation(ctx, kbClient) + installation, err := embeddedcluster.GetCurrentInstallation(ctx, ctrlClient) if err != nil { return nil, fmt.Errorf("failed to get current installation: %w", err) } - seaweedFSS3ServiceIP, err := embeddedcluster.GetSeaweedFSS3ServiceIP(ctx, kbClient) + seaweedFSS3ServiceIP, err := embeddedcluster.GetSeaweedFSS3ServiceIP(ctx, ctrlClient) if err != nil { return nil, fmt.Errorf("failed to get seaweedfs s3 service ip: %w", err) } diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 48db438000..05204c1b4a 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -2,21 +2,27 @@ package snapshot import ( "context" + "os" + "path/filepath" "testing" "time" gomock "github.com/golang/mock/gomock" embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types" apptypes "github.com/replicatedhq/kots/pkg/app/types" kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" "github.com/replicatedhq/kots/pkg/kotsutil" registrytypes "github.com/replicatedhq/kots/pkg/registry/types" "github.com/replicatedhq/kots/pkg/store" mock_store "github.com/replicatedhq/kots/pkg/store/mock" + "github.com/replicatedhq/kots/pkg/util" kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerofake "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake" + veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" @@ -27,6 +33,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" coretest "k8s.io/client-go/testing" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + ctrlclientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func TestPrepareIncludedNamespaces(t *testing.T) { @@ -1187,8 +1195,8 @@ func Test_mergeAppBackupSpec(t *testing.T) { { name: "has backup spec", setup: func(t *testing.T, mockStore *mock_store.MockStore) { - mockStore.EXPECT().GetLatestAppSequence("1", true).Return(int64(1), nil) - mockStore.EXPECT().GetRegistryDetailsForApp("1").Return(registrytypes.RegistrySettings{ + mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ Hostname: "hostname", Username: "username", Password: "password", @@ -1366,8 +1374,8 @@ func Test_mergeAppBackupSpec(t *testing.T) { { name: "ec, has backup spec", setup: func(t *testing.T, mockStore *mock_store.MockStore) { - mockStore.EXPECT().GetLatestAppSequence("1", true).Return(int64(1), nil) - mockStore.EXPECT().GetRegistryDetailsForApp("1").Return(registrytypes.RegistrySettings{ + mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ Hostname: "hostname", Username: "username", Password: "password", @@ -1603,8 +1611,8 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") - mockStore.EXPECT().GetLatestAppSequence("1", true).Return(int64(1), nil) - mockStore.EXPECT().GetRegistryDetailsForApp("1").Return(registrytypes.RegistrySettings{ + mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ Hostname: "hostname", Username: "username", Password: "password", @@ -1781,3 +1789,351 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { }) } } + +func Test_getInstanceBackupMetadata(t *testing.T) { + scheme := runtime.NewScheme() + corev1.AddToScheme(scheme) + embeddedclusterv1beta1.AddToScheme(scheme) + + testBsl := &velerov1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "velero", + }, + Spec: velerov1.BackupStorageLocationSpec{ + Provider: "aws", + Default: true, + }, + } + veleroNamespaceConfigmap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm-velero-namespace", + }, + Data: map[string]string{ + "veleroNamespace": "velero", + }, + } + veleroDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "velero", + Namespace: "velero", + }, + } + + installation := &embeddedclusterv1beta1.Installation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "20060102150405", + }, + Spec: embeddedclusterv1beta1.InstallationSpec{ + BinaryName: "my-app", + }, + } + seaweedFSS3Service := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ec-seaweedfs-s3", + Namespace: "seaweedfs", + }, + Spec: corev1.ServiceSpec{ + ClusterIP: "10.96.0.10", + }, + } + + type args struct { + k8sClient kubernetes.Interface + ctrlClient ctrlclient.Client + veleroClient veleroclientv1.VeleroV1Interface + cluster *downstreamtypes.Downstream + isScheduled bool + } + tests := []struct { + name string + setup func(t *testing.T, mockStore *mock_store.MockStore) + args args + want instanceBackupMetadata + wantErr bool + }{ + { + name: "cli install", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + util.PodNamespace = "test" + t.Cleanup(func() { + util.PodNamespace = "" + }) + + mockStore.EXPECT().ListInstalledApps().Times(1).Return([]*apptypes.App{ + { + ID: "1", + Name: "App 1", + Slug: "app-1", + IsAirgap: true, + }, + { + ID: "2", + Name: "App 2", + Slug: "app-2", + IsAirgap: true, + }, + }, nil) + mockStore.EXPECT().ListDownstreamsForApp(gomock.Any()).Times(2).Return([]downstreamtypes.Downstream{ + { + ClusterID: "cluster-id", + ClusterSlug: "cluster-slug", + Name: "cluster-name", + CurrentSequence: 1, + SnapshotSchedule: "manual", + SnapshotTTL: "24h", + }, + }, nil) + mockStore.EXPECT().GetCurrentParentSequence("1", "cluster-id").Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetCurrentParentSequence("2", "cluster-id").Times(1).Return(int64(2), nil) + mockStore.EXPECT().GetAppVersionArchive("1", int64(1), gomock.Any()).Times(1).DoAndReturn(func(appID string, sequence int64, archiveDir string) error { + err := setupArchiveDirectoriesAndFiles(archiveDir, map[string]string{ + "upstream/app.yaml": ` +apiVersion: kots.io/v1beta1 +kind: Application +metadata: + name: app-1 +spec: + title: My App 1`, + }) + require.NoError(t, err) + return nil + }) + mockStore.EXPECT().GetAppVersionArchive("2", int64(2), gomock.Any()).Times(1).DoAndReturn(func(appID string, sequence int64, archiveDir string) error { + err := setupArchiveDirectoriesAndFiles(archiveDir, map[string]string{ + "upstream/app.yaml": ` +apiVersion: kots.io/v1beta1 +kind: Application +metadata: + name: app-2 +spec: + title: My App 2`, + }) + require.NoError(t, err) + return nil + }) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), + ctrlClient: ctrlclientfake.NewClientBuilder().WithScheme(scheme).WithObjects().Build(), + veleroClient: velerofake.NewSimpleClientset(testBsl).VeleroV1(), + cluster: &downstreamtypes.Downstream{ + SnapshotTTL: "24h", + }, + isScheduled: true, + }, + want: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "test", + backupStorageLocationNamespace: "velero", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: &apptypes.App{ + ID: "1", + Name: "App 1", + Slug: "app-1", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kots.io/v1beta1", + Kind: "Application", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "app-1", + }, + Spec: kotsv1beta1.ApplicationSpec{ + Title: "My App 1", + }, + }, + Installation: kotsv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kots.io/v1beta1", + Kind: "Installation", + }, + }, + }, + parentSequence: 1, + }, + "app-2": { + app: &apptypes.App{ + ID: "2", + Name: "App 2", + Slug: "app-2", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kots.io/v1beta1", + Kind: "Application", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "app-2", + }, + Spec: kotsv1beta1.ApplicationSpec{ + Title: "My App 2", + }, + }, + Installation: kotsv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kots.io/v1beta1", + Kind: "Installation", + }, + }, + }, + parentSequence: 2, + }, + }, + isScheduled: true, + snapshotTTL: 24 * time.Hour, + ec: nil, + }, + }, + { + name: "ec install", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + + util.PodNamespace = "test" + t.Cleanup(func() { + util.PodNamespace = "" + }) + + mockStore.EXPECT().ListInstalledApps().Times(1).Return([]*apptypes.App{ + { + ID: "1", + Name: "App 1", + Slug: "app-1", + IsAirgap: true, + }, + }, nil) + mockStore.EXPECT().ListDownstreamsForApp(gomock.Any()).Times(1).Return([]downstreamtypes.Downstream{ + { + ClusterID: "cluster-id", + ClusterSlug: "cluster-slug", + Name: "cluster-name", + CurrentSequence: 1, + SnapshotSchedule: "manual", + SnapshotTTL: "24h", + }, + }, nil) + mockStore.EXPECT().GetCurrentParentSequence("1", "cluster-id").Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetAppVersionArchive("1", int64(1), gomock.Any()).Times(1).DoAndReturn(func(appID string, sequence int64, archiveDir string) error { + err := setupArchiveDirectoriesAndFiles(archiveDir, map[string]string{ + "upstream/app.yaml": ` +apiVersion: kots.io/v1beta1 +kind: Application +metadata: + name: app-1 +spec: + title: My App 1`, + }) + require.NoError(t, err) + return nil + }) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), + ctrlClient: ctrlclientfake.NewClientBuilder().WithScheme(scheme).WithObjects( + installation, + seaweedFSS3Service, + ).Build(), + veleroClient: velerofake.NewSimpleClientset(testBsl).VeleroV1(), + cluster: &downstreamtypes.Downstream{ + SnapshotTTL: "24h", + }, + isScheduled: true, + }, + want: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "test", + backupStorageLocationNamespace: "velero", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: &apptypes.App{ + ID: "1", + Name: "App 1", + Slug: "app-1", + IsAirgap: true, + }, + kotsKinds: &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kots.io/v1beta1", + Kind: "Application", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "app-1", + }, + Spec: kotsv1beta1.ApplicationSpec{ + Title: "My App 1", + }, + }, + Installation: kotsv1beta1.Installation{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "kots.io/v1beta1", + Kind: "Installation", + }, + }, + }, + parentSequence: 1, + }, + }, + isScheduled: true, + snapshotTTL: 24 * time.Hour, + ec: &ecInstanceBackupMetadata{ + installation: installation, + seaweedFSS3ServiceIP: "10.96.0.10", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStore := mock_store.NewMockStore(ctrl) + store.SetStore(mockStore) + + t.Cleanup(func() { + store.SetStore(nil) + }) + + if tt.setup != nil { + tt.setup(t, mockStore) + } + + got, err := getInstanceBackupMetadata(context.Background(), tt.args.k8sClient, tt.args.ctrlClient, tt.args.veleroClient, tt.args.cluster, tt.args.isScheduled) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + assert.Regexp(t, "^backup-", got.backupName) + assert.NotZero(t, got.backupReqestedAt) + tt.want.backupName = got.backupName + tt.want.backupReqestedAt = got.backupReqestedAt + + assert.Equal(t, tt.want, got) + }) + } +} + +func setupArchiveDirectoriesAndFiles(archiveDir string, files map[string]string) error { + for path, content := range files { + dir := filepath.Dir(path) + if err := os.MkdirAll(filepath.Join(archiveDir, dir), 0744); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(archiveDir, path), []byte(content), 0644); err != nil { + return err + } + } + return nil +} From a7a0cc1df686298a1ae4b7151ad6a03c4a6c3ca7 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Tue, 3 Dec 2024 22:26:59 -0800 Subject: [PATCH 23/44] f --- pkg/kotsadmsnapshot/backup.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index e9c6534fc0..9e3dfae53a 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -224,7 +224,7 @@ type ecInstanceBackupMetadata struct { } func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstream, isScheduled bool) (string, error) { - logger.Debug("creating instance backup") + logger.Info("Creating instance backup") cfg, err := k8sutil.GetClusterConfig() if err != nil { @@ -273,12 +273,14 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre } } + logger.Info("Creating instance backup CR", zap.String("name", veleroBackup.Name)) _, err = veleroClient.Backups(metadata.backupStorageLocationNamespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) if err != nil { return "", errors.Wrap(err, "failed to create velero backup") } if appVeleroBackup != nil { + logger.Info("Creating instance app backup CR", zap.String("name", appVeleroBackup.Name)) _, err := veleroClient.Backups(metadata.backupStorageLocationNamespace).Create(ctx, appVeleroBackup, metav1.CreateOptions{}) if err != nil { return "", errors.Wrap(err, "failed to create application velero backup") @@ -745,7 +747,7 @@ func ListBackupsForApp(ctx context.Context, kotsadmNamespace string, appID strin } backup := types.Backup{ - Name: veleroBackup.Name, + Name: veleroBackup.Name, // TODO: GetBackupName(veleroBackup), Status: string(veleroBackup.Status.Phase), AppID: appID, } @@ -884,7 +886,7 @@ func ListInstanceBackups(ctx context.Context, kotsadmNamespace string) ([]*types } backup := types.Backup{ - Name: GetBackupName(veleroBackup), + Name: veleroBackup.Name, // TODO: GetBackupName(veleroBackup), Status: string(veleroBackup.Status.Phase), IncludedApps: make([]types.App, 0), } @@ -1166,7 +1168,7 @@ func GetBackupDetail(ctx context.Context, kotsadmNamespace string, backupName st } result := &types.BackupDetail{ - Name: GetBackupName(*backup), + Name: backup.Name, // TODO: GetBackupName(*backup), Status: string(backup.Status.Phase), Namespaces: backup.Spec.IncludedNamespaces, Volumes: listBackupVolumes(backupVolumes.Items), From 2b660d45089577b46e0c51e15524be273f0e893c Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Wed, 4 Dec 2024 05:08:50 -0800 Subject: [PATCH 24/44] f --- pkg/kotsadmsnapshot/backup.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 9e3dfae53a..77b4ac2e1d 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -274,7 +274,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre } logger.Info("Creating instance backup CR", zap.String("name", veleroBackup.Name)) - _, err = veleroClient.Backups(metadata.backupStorageLocationNamespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) + backup, err := veleroClient.Backups(metadata.backupStorageLocationNamespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) if err != nil { return "", errors.Wrap(err, "failed to create velero backup") } @@ -287,7 +287,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre } } - return metadata.backupName, nil + return backup.Name, nil // TODO: return metadata.BackupName } func GetBackupName(veleroBackup velerov1.Backup) string { From 937ee5a2e8d717b49bd74bbc92fbf09d89806d89 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Wed, 4 Dec 2024 05:43:34 -0800 Subject: [PATCH 25/44] f --- pkg/kotsadmsnapshot/backup.go | 24 +- pkg/kotsadmsnapshot/backup_test.go | 492 ++++++++++++++++++++--------- 2 files changed, 348 insertions(+), 168 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 77b4ac2e1d..6d34a7463e 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -219,7 +219,7 @@ type appInstanceBackupMetadata struct { } type ecInstanceBackupMetadata struct { - installation *embeddedclusterv1beta1.Installation + installation embeddedclusterv1beta1.Installation seaweedFSS3ServiceIP string } @@ -251,7 +251,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return "", errors.Wrap(err, "failed to get instance backup metadata") } - appVeleroBackup, err := getAppInstanceBackupSpec(ctx, k8sClient, metadata) + appVeleroBackup, err := getAppInstanceBackupSpec(k8sClient, metadata) if err != nil { return "", errors.Wrap(err, "failed to get app instance backup spec") } @@ -421,7 +421,7 @@ func getECInstanceBackupMetadata(ctx context.Context, ctrlClient ctrlclient.Clie } return &ecInstanceBackupMetadata{ - installation: installation, + installation: *installation, seaweedFSS3ServiceIP: seaweedFSS3ServiceIP, }, nil } @@ -482,7 +482,7 @@ func getInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, } } - veleroBackup.Annotations, err = appendCommonAnnotations(ctx, k8sClient, veleroBackup.Annotations, metadata, hasAppSpec) + veleroBackup.Annotations, err = appendCommonAnnotations(k8sClient, veleroBackup.Annotations, metadata, hasAppSpec) if err != nil { return nil, errors.Wrap(err, "failed to add annotations to backup") } @@ -509,7 +509,7 @@ func getInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, // getAppInstanceBackup returns a backup spec only if this is Embedded Cluster and the vendor has // defined both a backup and restore custom resource (improved DR). -func getAppInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata) (*velerov1.Backup, error) { +func getAppInstanceBackupSpec(k8sClient kubernetes.Interface, metadata instanceBackupMetadata) (*velerov1.Backup, error) { if metadata.ec == nil { return nil, nil } @@ -554,16 +554,8 @@ func getAppInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interfac return nil, nil } - appSequences := map[string]int64{} - appVersions := map[string]string{} - - for slug, appMeta := range metadata.apps { - appSequences[slug] = appMeta.parentSequence - appVersions[slug] = appMeta.kotsKinds.Installation.Spec.VersionLabel - } - var err error - appVeleroBackup.Annotations, err = appendCommonAnnotations(ctx, k8sClient, appVeleroBackup.Annotations, metadata, true) + appVeleroBackup.Annotations, err = appendCommonAnnotations(k8sClient, appVeleroBackup.Annotations, metadata, true) if err != nil { return nil, errors.Wrap(err, "failed to add annotations to application backup") } @@ -644,7 +636,7 @@ func mergeAppBackupSpec(backup *velerov1.Backup, appMeta appInstanceBackupMetada } // appendCommonAnnotations appends common annotations to the backup annotations -func appendCommonAnnotations(ctx context.Context, k8sClient kubernetes.Interface, annotations map[string]string, metadata instanceBackupMetadata, hasAppSpec bool) (map[string]string, error) { +func appendCommonAnnotations(k8sClient kubernetes.Interface, annotations map[string]string, metadata instanceBackupMetadata, hasAppSpec bool) (map[string]string, error) { kotsadmImage, err := k8sutil.FindKotsadmImage(k8sClient, metadata.kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find kotsadm image") @@ -1300,7 +1292,7 @@ func ecRuntimeConfigToBackupAnnotations(runtimeConfig *embeddedclusterv1beta1.Ru } // ecIncludedNamespaces returns the namespaces that should be included in an embedded cluster backup -func ecIncludedNamespaces(in *embeddedclusterv1beta1.Installation) []string { +func ecIncludedNamespaces(in embeddedclusterv1beta1.Installation) []string { includedNamespaces := []string{"embedded-cluster", "kube-system", "openebs"} if in.Spec.AirGap { includedNamespaces = append(includedNamespaces, "registry") diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 05204c1b4a..8f88695e46 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -751,7 +751,7 @@ func Test_appendECAnnotations(t *testing.T) { tests := []struct { name string prev map[string]string - in *embeddedclusterv1beta1.Installation + in embeddedclusterv1beta1.Installation seaweedFSS3ServiceIP string env map[string]string want map[string]string @@ -761,7 +761,7 @@ func Test_appendECAnnotations(t *testing.T) { prev: map[string]string{ "prev-key": "prev-value", }, - in: &embeddedclusterv1beta1.Installation{}, + in: embeddedclusterv1beta1.Installation{}, seaweedFSS3ServiceIP: "", env: map[string]string{ "EMBEDDED_CLUSTER_ID": "embedded-cluster-id", @@ -777,7 +777,7 @@ func Test_appendECAnnotations(t *testing.T) { }, { name: "online ha", - in: &embeddedclusterv1beta1.Installation{ + in: embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ HighAvailability: true, }, @@ -796,7 +796,7 @@ func Test_appendECAnnotations(t *testing.T) { }, { name: "airgap ha", - in: &embeddedclusterv1beta1.Installation{ + in: embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ HighAvailability: true, AirGap: true, @@ -817,7 +817,7 @@ func Test_appendECAnnotations(t *testing.T) { }, { name: "with pod and service cidrs", - in: &embeddedclusterv1beta1.Installation{ + in: embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ Network: &embeddedclusterv1beta1.NetworkSpec{ PodCIDR: "10.128.0.0/20", @@ -859,12 +859,12 @@ func Test_appendECAnnotations(t *testing.T) { func Test_ecIncludedNamespaces(t *testing.T) { tests := []struct { name string - in *embeddedclusterv1beta1.Installation + in embeddedclusterv1beta1.Installation want []string }{ { name: "online", - in: &embeddedclusterv1beta1.Installation{}, + in: embeddedclusterv1beta1.Installation{}, want: []string{ "embedded-cluster", "kube-system", @@ -873,7 +873,7 @@ func Test_ecIncludedNamespaces(t *testing.T) { }, { name: "online ha", - in: &embeddedclusterv1beta1.Installation{ + in: embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ HighAvailability: true, }, @@ -886,7 +886,7 @@ func Test_ecIncludedNamespaces(t *testing.T) { }, { name: "airgap", - in: &embeddedclusterv1beta1.Installation{ + in: embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ AirGap: true, }, @@ -900,7 +900,7 @@ func Test_ecIncludedNamespaces(t *testing.T) { }, { name: "airgap ha", - in: &embeddedclusterv1beta1.Installation{ + in: embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ HighAvailability: true, AirGap: true, @@ -1053,7 +1053,7 @@ func Test_appendCommonAnnotations(t *testing.T) { }, isScheduled: true, ec: &ecInstanceBackupMetadata{ - installation: &embeddedclusterv1beta1.Installation{ + installation: embeddedclusterv1beta1.Installation{ Spec: embeddedclusterv1beta1.InstallationSpec{ HighAvailability: true, Network: &embeddedclusterv1beta1.NetworkSpec{ @@ -1106,7 +1106,7 @@ func Test_appendCommonAnnotations(t *testing.T) { if tt.setup != nil { tt.setup(t) } - got, err := appendCommonAnnotations(context.Background(), tt.args.k8sClient, tt.args.annotations, tt.args.metadata, tt.args.hasAppSpec) + got, err := appendCommonAnnotations(tt.args.k8sClient, tt.args.annotations, tt.args.metadata, tt.args.hasAppSpec) if tt.wantErr { require.Error(t, err) } else { @@ -1546,25 +1546,105 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { }, } + app1 := &apptypes.App{ + ID: "1", + Slug: "app-1", + IsAirgap: true, + } + + app2 := &apptypes.App{ + ID: "2", + Slug: "app-2", + IsAirgap: true, + } + + kotsKinds := &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + Spec: kotsv1beta1.ApplicationSpec{ + AdditionalNamespaces: []string{"another-namespace-1", "another-namespace-2"}, + }, + }, + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + Backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-backup", + Annotations: map[string]string{ + "annotation-1": "true", + "annotation-2": "false", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "blah", + TTL: metav1.Duration{Duration: 1 * time.Hour}, + IncludedNamespaces: []string{"include-namespace-1", "include-namespace-2", "template-isairgap-{{repl IsAirgap }}"}, + ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, + OrderedResources: map[string]string{ + "resource-1": "true", + "resource-2": "false", + }, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{ + { + Name: "hook-1", + }, + { + Name: "hook-2", + }, + }, + }, + }, + }, + Restore: &velerov1.Restore{}, + } + + ecMeta := &ecInstanceBackupMetadata{ + installation: embeddedclusterv1beta1.Installation{ + Spec: embeddedclusterv1beta1.InstallationSpec{ + HighAvailability: true, + Network: &embeddedclusterv1beta1.NetworkSpec{ + PodCIDR: "10.128.0.0/20", + ServiceCIDR: "10.129.0.0/20", + }, + RuntimeConfig: &embeddedclusterv1beta1.RuntimeConfigSpec{ + DataDir: "/var/lib/ec", + AdminConsole: embeddedclusterv1beta1.AdminConsoleSpec{ + Port: 30001, + }, + LocalArtifactMirror: embeddedclusterv1beta1.LocalArtifactMirrorSpec{ + Port: 50001, + }, + }, + }, + }, + seaweedFSS3ServiceIP: "10.96.0.10", + } + type args struct { k8sClient kubernetes.Interface metadata instanceBackupMetadata } tests := []struct { - name string - setup func(t *testing.T, mockStore *mock_store.MockStore) - args args - want *velerov1.Backup - wantErr bool + name string + setup func(t *testing.T, mockStore *mock_store.MockStore) + args args + assert func(t *testing.T, got *velerov1.Backup, err error) }{ { - name: "not ec", + name: "not ec with backup and restore spec should return nil", args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ apps: map[string]appInstanceBackupMetadata{ "app-1": { - app: &apptypes.App{}, + app: app1, kotsKinds: &kotsutil.KotsKinds{ Backup: &velerov1.Backup{}, Restore: &velerov1.Restore{}, @@ -1575,10 +1655,13 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { ec: nil, }, }, - want: nil, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Nil(t, got) + }, }, { - name: "ec, no restore spec", + name: "ec wihtout restore spec should return nil", setup: func(t *testing.T, mockStore *mock_store.MockStore) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") @@ -1592,7 +1675,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { backupStorageLocationNamespace: "kotsadm-backups", apps: map[string]appInstanceBackupMetadata{ "app-1": { - app: &apptypes.App{}, + app: app1, kotsKinds: &kotsutil.KotsKinds{ Backup: &velerov1.Backup{}, }, @@ -1600,13 +1683,81 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { }, }, isScheduled: true, - ec: &ecInstanceBackupMetadata{}, + ec: ecMeta, }, }, - want: nil, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Nil(t, got) + }, }, { - name: "ec, improved DR", + name: "ec with backup and restore spec and multiple apps should return error", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + "app-2": { + app: app2, + kotsKinds: kotsKinds, + parentSequence: 2, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.Error(t, err) + assert.Nil(t, got) + }, + }, + { + name: "not ec with backup and restore spec and multiple apps should not return error", + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: &kotsutil.KotsKinds{ + Backup: &velerov1.Backup{}, + Restore: &velerov1.Restore{}, + }, + parentSequence: 1, + }, + "app-2": { + app: app2, + kotsKinds: &kotsutil.KotsKinds{ + Backup: &velerov1.Backup{}, + Restore: &velerov1.Restore{}, + }, + parentSequence: 2, + }, + }, + ec: nil, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Nil(t, got) + }, + }, + { + name: "ec with backup and restore spec should override name", setup: func(t *testing.T, mockStore *mock_store.MockStore) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") @@ -1629,139 +1780,181 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { backupStorageLocationNamespace: "kotsadm-backups", apps: map[string]appInstanceBackupMetadata{ "app-1": { - app: &apptypes.App{ - ID: "1", - Slug: "app-1", - IsAirgap: true, - }, - kotsKinds: &kotsutil.KotsKinds{ - KotsApplication: kotsv1beta1.Application{ - Spec: kotsv1beta1.ApplicationSpec{ - AdditionalNamespaces: []string{"another-namespace-1", "another-namespace-2"}, - }, - }, - Installation: kotsv1beta1.Installation{ - Spec: kotsv1beta1.InstallationSpec{ - VersionLabel: "1.0.1", - }, - }, - Backup: &velerov1.Backup{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "velero.io/v1", - Kind: "Backup", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-backup", - Annotations: map[string]string{ - "annotation-1": "true", - "annotation-2": "false", - }, - }, - Spec: velerov1.BackupSpec{ - IncludedNamespaces: []string{"include-namespace-1", "include-namespace-2", "template-isairgap-{{repl IsAirgap }}"}, - ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, - OrderedResources: map[string]string{ - "resource-1": "true", - "resource-2": "false", - }, - Hooks: velerov1.BackupHooks{ - Resources: []velerov1.BackupResourceHookSpec{ - { - Name: "hook-1", - }, - { - Name: "hook-2", - }, - }, - }, - }, - }, - Restore: &velerov1.Restore{}, - }, + app: app1, + kotsKinds: kotsKinds, parentSequence: 1, }, }, isScheduled: true, - snapshotTTL: 24 * time.Hour, - ec: &ecInstanceBackupMetadata{ - installation: &embeddedclusterv1beta1.Installation{ - Spec: embeddedclusterv1beta1.InstallationSpec{ - HighAvailability: true, - Network: &embeddedclusterv1beta1.NetworkSpec{ - PodCIDR: "10.128.0.0/20", - ServiceCIDR: "10.129.0.0/20", - }, - RuntimeConfig: &embeddedclusterv1beta1.RuntimeConfigSpec{ - DataDir: "/var/lib/ec", - AdminConsole: embeddedclusterv1beta1.AdminConsoleSpec{ - Port: 30001, - }, - LocalArtifactMirror: embeddedclusterv1beta1.LocalArtifactMirrorSpec{ - Port: 50001, - }, - }, - }, + ec: ecMeta, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Equal(t, "", got.Name) + assert.Equal(t, "app-1-", got.GenerateName) + }, + }, + { + name: "ec with backup and restore spec should append common annotations", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ + Hostname: "hostname", + Username: "username", + Password: "password", + Namespace: "namespace", + IsReadOnly: true, + }, nil) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, }, - seaweedFSS3ServiceIP: "10.96.0.10", }, + isScheduled: true, + ec: ecMeta, }, }, - want: &velerov1.Backup{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "velero.io/v1", - Kind: "Backup", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "", - GenerateName: "app-1-", - Annotations: map[string]string{ - "annotation-1": "true", - "annotation-2": "false", - "kots.io/apps-sequences": "{\"app-1\":1}", - "kots.io/apps-versions": "{\"app-1\":\"1.0.1\"}", - "kots.io/embedded-registry": "host", - "kots.io/instance": "true", - "kots.io/is-airgap": "false", - "kots.io/kotsadm-deploy-namespace": "kotsadm", - "kots.io/kotsadm-image": "kotsadm/kotsadm:1.0.0", - "kots.io/snapshot-requested": "2024-01-01T00:00:00Z", - "kots.io/snapshot-trigger": "schedule", - "replicated.com/backup-name": "backup-17332487841234", - "replicated.com/backup-type": "app", - "replicated.com/backups-expected": "2", - "kots.io/embedded-cluster": "true", - "kots.io/embedded-cluster-id": "embedded-cluster-id", - "kots.io/embedded-cluster-version": "embedded-cluster-version", - "kots.io/embedded-cluster-is-ha": "true", - "kots.io/embedded-cluster-pod-cidr": "10.128.0.0/20", - "kots.io/embedded-cluster-service-cidr": "10.129.0.0/20", - "kots.io/embedded-cluster-seaweedfs-s3-ip": "10.96.0.10", - "kots.io/embedded-cluster-admin-console-port": "30001", - "kots.io/embedded-cluster-local-artifact-mirror-port": "50001", - "kots.io/embedded-cluster-data-dir": "/var/lib/ec", + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + if assert.Contains(t, got.Annotations, "replicated.com/backup-name") { + assert.Equal(t, "backup-17332487841234", got.Annotations["replicated.com/backup-name"]) + } + if assert.Contains(t, got.Annotations, "replicated.com/backup-type") { + assert.Equal(t, "app", got.Annotations["replicated.com/backup-type"]) + } + if assert.Contains(t, got.Annotations, "replicated.com/backups-expected") { + assert.Equal(t, "2", got.Annotations["replicated.com/backups-expected"]) + } + }, + }, + { + name: "ec with backup and restore spec overrides storage location", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ + Hostname: "hostname", + Username: "username", + Password: "password", + Namespace: "namespace", + IsReadOnly: true, + }, nil) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, }, + isScheduled: true, + ec: ecMeta, }, - Spec: velerov1.BackupSpec{ - StorageLocation: "default", - IncludedNamespaces: []string{"include-namespace-1", "include-namespace-2", "template-isairgap-true"}, - ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, - OrderedResources: map[string]string{ - "resource-1": "true", - "resource-2": "false", + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Equal(t, "default", got.Spec.StorageLocation) + }, + }, + { + name: "ec with backup and restore spec overrides snapshot ttl", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ + Hostname: "hostname", + Username: "username", + Password: "password", + Namespace: "namespace", + IsReadOnly: true, + }, nil) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, }, - Hooks: velerov1.BackupHooks{ - Resources: []velerov1.BackupResourceHookSpec{ - { - Name: "hook-1", - }, - { - Name: "hook-2", - }, + isScheduled: true, + snapshotTTL: 24 * time.Hour, + ec: ecMeta, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Equal(t, metav1.Duration{Duration: 24 * time.Hour}, got.Spec.TTL) + }, + }, + { + name: "ec with backup and restore spec does not override snapshot ttl if unset", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ + Hostname: "hostname", + Username: "username", + Password: "password", + Namespace: "namespace", + IsReadOnly: true, + }, nil) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, }, }, - TTL: metav1.Duration{Duration: 24 * time.Hour}, + isScheduled: true, + ec: ecMeta, }, }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Equal(t, metav1.Duration{Duration: 1 * time.Hour}, got.Spec.TTL) + }, }, } for _, tt := range tests { @@ -1779,13 +1972,8 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { if tt.setup != nil { tt.setup(t, mockStore) } - got, err := getAppInstanceBackupSpec(context.Background(), tt.args.k8sClient, tt.args.metadata) - if tt.wantErr { - require.Error(t, err) - } else { - require.NoError(t, err) - } - assert.Equal(t, tt.want, got) + got, err := getAppInstanceBackupSpec(tt.args.k8sClient, tt.args.metadata) + tt.assert(t, got, err) }) } } @@ -1820,7 +2008,7 @@ func Test_getInstanceBackupMetadata(t *testing.T) { }, } - installation := &embeddedclusterv1beta1.Installation{ + installation := embeddedclusterv1beta1.Installation{ ObjectMeta: metav1.ObjectMeta{ Name: "20060102150405", }, @@ -2038,7 +2226,7 @@ spec: args: args{ k8sClient: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), ctrlClient: ctrlclientfake.NewClientBuilder().WithScheme(scheme).WithObjects( - installation, + &installation, seaweedFSS3Service, ).Build(), veleroClient: velerofake.NewSimpleClientset(testBsl).VeleroV1(), From 0858524d3de42c74acdc59121b66e9798b12ef69 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Wed, 4 Dec 2024 12:28:46 -0800 Subject: [PATCH 26/44] f --- pkg/kotsadmsnapshot/backup_test.go | 608 ++++++++++++++++++++++++++--- 1 file changed, 552 insertions(+), 56 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 8f88695e46..0fcc244347 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -25,6 +25,7 @@ import ( veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -1118,6 +1119,17 @@ func Test_appendCommonAnnotations(t *testing.T) { } func Test_mergeAppBackupSpec(t *testing.T) { + mockStoreExpectApp1 := func(mockStore *mock_store.MockStore) { + mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ + Hostname: "hostname", + Username: "username", + Password: "password", + Namespace: "namespace", + IsReadOnly: true, + }, nil) + } + type args struct { backup *velerov1.Backup appMeta appInstanceBackupMetadata @@ -1195,14 +1207,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { { name: "has backup spec", setup: func(t *testing.T, mockStore *mock_store.MockStore) { - mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) - mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ - Hostname: "hostname", - Username: "username", - Password: "password", - Namespace: "namespace", - IsReadOnly: true, - }, nil) + mockStoreExpectApp1(mockStore) }, args: args{ backup: &velerov1.Backup{ @@ -1374,14 +1379,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { { name: "ec, has backup spec", setup: func(t *testing.T, mockStore *mock_store.MockStore) { - mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) - mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ - Hostname: "hostname", - Username: "username", - Password: "password", - Namespace: "namespace", - IsReadOnly: true, - }, nil) + mockStoreExpectApp1(mockStore) }, args: args{ backup: &velerov1.Backup{ @@ -1517,6 +1515,17 @@ func Test_mergeAppBackupSpec(t *testing.T) { } func Test_getAppInstanceBackupSpec(t *testing.T) { + mockStoreExpectApp1 := func(mockStore *mock_store.MockStore) { + mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ + Hostname: "hostname", + Username: "username", + Password: "password", + Namespace: "namespace", + IsReadOnly: true, + }, nil) + } + kotsadmSts := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: "kotsadm", @@ -1762,14 +1771,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") - mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) - mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ - Hostname: "hostname", - Username: "username", - Password: "password", - Namespace: "namespace", - IsReadOnly: true, - }, nil) + mockStoreExpectApp1(mockStore) }, args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), @@ -1801,14 +1803,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") - mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) - mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ - Hostname: "hostname", - Username: "username", - Password: "password", - Namespace: "namespace", - IsReadOnly: true, - }, nil) + mockStoreExpectApp1(mockStore) }, args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), @@ -1847,14 +1842,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") - mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) - mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ - Hostname: "hostname", - Username: "username", - Password: "password", - Namespace: "namespace", - IsReadOnly: true, - }, nil) + mockStoreExpectApp1(mockStore) }, args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), @@ -1885,14 +1873,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") - mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) - mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ - Hostname: "hostname", - Username: "username", - Password: "password", - Namespace: "namespace", - IsReadOnly: true, - }, nil) + mockStoreExpectApp1(mockStore) }, args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), @@ -1924,14 +1905,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") - mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) - mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ - Hostname: "hostname", - Username: "username", - Password: "password", - Namespace: "namespace", - IsReadOnly: true, - }, nil) + mockStoreExpectApp1(mockStore) }, args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), @@ -1978,6 +1952,528 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { } } +func Test_getInstanceBackupSpec(t *testing.T) { + kotsadmSts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm", + Namespace: "kotsadm", + }, + Spec: appsv1.StatefulSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kotsadm", + Image: "kotsadm/kotsadm:1.0.0", + }, + }, + }, + }, + }, + } + registryCredsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "registry-creds", + Namespace: "kotsadm", + }, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + ".dockerconfigjson": []byte(`{"auths":{"host":{"username":"kurl","password":"password"}}}`), + }, + } + + app1 := &apptypes.App{ + ID: "1", + Slug: "app-1", + IsAirgap: true, + } + + kotsKinds := &kotsutil.KotsKinds{ + KotsApplication: kotsv1beta1.Application{ + Spec: kotsv1beta1.ApplicationSpec{ + AdditionalNamespaces: []string{"another-namespace-1", "another-namespace-2", "duplicate-namespace"}, + }, + }, + Installation: kotsv1beta1.Installation{ + Spec: kotsv1beta1.InstallationSpec{ + VersionLabel: "1.0.1", + }, + }, + Backup: &velerov1.Backup{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "velero.io/v1", + Kind: "Backup", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-backup", + Annotations: map[string]string{ + "annotation-1": "true", + "annotation-2": "false", + }, + }, + Spec: velerov1.BackupSpec{ + StorageLocation: "blah", + TTL: metav1.Duration{Duration: 1 * time.Hour}, + IncludedNamespaces: []string{"include-namespace-1", "include-namespace-2", "template-isairgap-{{repl IsAirgap }}", "duplicate-namespace"}, + ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, + OrderedResources: map[string]string{ + "resource-1": "true", + "resource-2": "false", + }, + Hooks: velerov1.BackupHooks{ + Resources: []velerov1.BackupResourceHookSpec{ + { + Name: "hook-1", + }, + { + Name: "hook-2", + }, + }, + }, + }, + }, + Restore: &velerov1.Restore{}, + } + + ecMeta := &ecInstanceBackupMetadata{ + installation: embeddedclusterv1beta1.Installation{ + Spec: embeddedclusterv1beta1.InstallationSpec{ + HighAvailability: true, + Network: &embeddedclusterv1beta1.NetworkSpec{ + PodCIDR: "10.128.0.0/20", + ServiceCIDR: "10.129.0.0/20", + }, + RuntimeConfig: &embeddedclusterv1beta1.RuntimeConfigSpec{ + DataDir: "/var/lib/ec", + AdminConsole: embeddedclusterv1beta1.AdminConsoleSpec{ + Port: 30001, + }, + LocalArtifactMirror: embeddedclusterv1beta1.LocalArtifactMirrorSpec{ + Port: 50001, + }, + }, + }, + }, + seaweedFSS3ServiceIP: "10.96.0.10", + } + + mockStoreExpectApp1 := func(mockStore *mock_store.MockStore) { + mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) + mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ + Hostname: "hostname", + Username: "username", + Password: "password", + Namespace: "namespace", + IsReadOnly: true, + }, nil) + } + + type args struct { + k8sClient kubernetes.Interface + metadata instanceBackupMetadata + hasAppSpec bool + } + tests := []struct { + name string + setup func(t *testing.T, mockStore *mock_store.MockStore) + args args + assert func(t *testing.T, got *velerov1.Backup, err error) + }{ + { + name: "KOTSADM_TARGET_NAMESPACE should be added to includedNamespaces", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("KOTSADM_TARGET_NAMESPACE", "kotsadm-target") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: nil, + }, + hasAppSpec: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "kotsadm-target") + }, + }, + { + name: "if kurl should be added to includedNamespaces", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kurl-config", + Namespace: "kube-system", + }, + }), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: nil, + }, + hasAppSpec: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "kurl") + }, + }, + { + name: "not cluster scoped should include backup storage location namespace", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: nil, + }, + hasAppSpec: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "kotsadm-backups") + }, + }, + { + name: "cluster scoped should not include backup storage location namespace", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret, &rbacv1.ClusterRoleBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "rbac.authorization.k8s.io/v1", + Kind: "ClusterRoleBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm-rolebinding", + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "kotsadm", + Namespace: "kotsadm", + }, + }, + }), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: nil, + }, + hasAppSpec: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.NotContains(t, got.Spec.IncludedNamespaces, "kotsadm-backups") + }, + }, + { + name: "should merge backup spec when not using improved dr", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppSpec: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "include-namespace-1") + if assert.Contains(t, got.Annotations, "replicated.com/backup-type") { + assert.Equal(t, "combined", got.Annotations["replicated.com/backup-type"]) + } + }, + }, + { + name: "should not merge backup spec when using improved dr", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppSpec: true, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.NotContains(t, got.Spec.IncludedNamespaces, "include-namespace-1") + if assert.Contains(t, got.Annotations, "replicated.com/backup-type") { + assert.Equal(t, "kotsadm", got.Annotations["replicated.com/backup-type"]) + } + }, + }, + { + name: "should add ec namespaces to includedNamespaces if ec", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppSpec: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "embedded-cluster") + }, + }, + { + name: "should add ec namespaces to includedNamespaces if ec", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppSpec: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "embedded-cluster") + }, + }, + { + name: "should override snapshot ttl if set", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + snapshotTTL: 24 * time.Hour, + ec: ecMeta, + }, + hasAppSpec: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Equal(t, metav1.Duration{Duration: 24 * time.Hour}, got.Spec.TTL) + }, + }, + { + name: "should not override snapshot ttl if unset", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppSpec: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Zero(t, got.Spec.TTL) + }, + }, + { + name: "should deduplicate includedNamespaces", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "backup-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppSpec: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + count := 0 + for _, ns := range got.Spec.IncludedNamespaces { + if ns == "duplicate-namespace" { + count++ + } + } + assert.Equal(t, 1, count, "Duplicate namespace should be removed") + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mockStore := mock_store.NewMockStore(ctrl) + store.SetStore(mockStore) + + t.Cleanup(func() { + store.SetStore(nil) + }) + + if tt.setup != nil { + tt.setup(t, mockStore) + } + got, err := getInstanceBackupSpec(context.Background(), tt.args.k8sClient, tt.args.metadata, tt.args.hasAppSpec) + tt.assert(t, got, err) + }) + } +} + func Test_getInstanceBackupMetadata(t *testing.T) { scheme := runtime.NewScheme() corev1.AddToScheme(scheme) From fb6c03c920f779520ae83d5f678e967c0a1a0e6f Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Wed, 4 Dec 2024 13:56:02 -0800 Subject: [PATCH 27/44] f --- pkg/kotsadmsnapshot/backup.go | 21 ++++++++++++++------- pkg/kotsadmsnapshot/backup_test.go | 5 +++++ pkg/kotsadmsnapshot/restore.go | 2 +- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 6d34a7463e..232b3a629c 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -273,21 +273,21 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre } } - logger.Info("Creating instance backup CR", zap.String("name", veleroBackup.Name)) + logger.Infof("Creating instance backup CR %s", veleroBackup.Name) backup, err := veleroClient.Backups(metadata.backupStorageLocationNamespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) if err != nil { return "", errors.Wrap(err, "failed to create velero backup") } if appVeleroBackup != nil { - logger.Info("Creating instance app backup CR", zap.String("name", appVeleroBackup.Name)) + logger.Infof("Creating instance app backup CR %s", appVeleroBackup.Name) _, err := veleroClient.Backups(metadata.backupStorageLocationNamespace).Create(ctx, appVeleroBackup, metav1.CreateOptions{}) if err != nil { return "", errors.Wrap(err, "failed to create application velero backup") } } - return backup.Name, nil // TODO: return metadata.BackupName + return backup.Name, nil // TODO(improveddr): return metadata.BackupName } func GetBackupName(veleroBackup velerov1.Backup) string { @@ -507,9 +507,16 @@ func getInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, return veleroBackup, nil } +var EnableImprovedDR = false + // getAppInstanceBackup returns a backup spec only if this is Embedded Cluster and the vendor has // defined both a backup and restore custom resource (improved DR). func getAppInstanceBackupSpec(k8sClient kubernetes.Interface, metadata instanceBackupMetadata) (*velerov1.Backup, error) { + // TODO(improveddr): remove this once we have fully implemented the improved DR + if !EnableImprovedDR { + return nil, nil + } + if metadata.ec == nil { return nil, nil } @@ -739,7 +746,7 @@ func ListBackupsForApp(ctx context.Context, kotsadmNamespace string, appID strin } backup := types.Backup{ - Name: veleroBackup.Name, // TODO: GetBackupName(veleroBackup), + Name: veleroBackup.Name, // TODO(improveddr): GetBackupName(veleroBackup), Status: string(veleroBackup.Status.Phase), AppID: appID, } @@ -872,13 +879,13 @@ func ListInstanceBackups(ctx context.Context, kotsadmNamespace string) ([]*types continue } - // TODO: support for improved DR in UI + // TODO(improveddr): support for improved DR in UI if GetInstanceBackupType(veleroBackup) == InstanceBackupTypeApp { continue } backup := types.Backup{ - Name: veleroBackup.Name, // TODO: GetBackupName(veleroBackup), + Name: veleroBackup.Name, // TODO(improveddr): GetBackupName(veleroBackup), Status: string(veleroBackup.Status.Phase), IncludedApps: make([]types.App, 0), } @@ -1160,7 +1167,7 @@ func GetBackupDetail(ctx context.Context, kotsadmNamespace string, backupName st } result := &types.BackupDetail{ - Name: backup.Name, // TODO: GetBackupName(*backup), + Name: backup.Name, // TODO(improveddr): GetBackupName(*backup), Status: string(backup.Status.Phase), Namespaces: backup.Spec.IncludedNamespaces, Volumes: listBackupVolumes(backupVolumes.Items), diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 0fcc244347..3dc4eab8de 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -1515,6 +1515,11 @@ func Test_mergeAppBackupSpec(t *testing.T) { } func Test_getAppInstanceBackupSpec(t *testing.T) { + EnableImprovedDR = true + t.Cleanup(func() { + EnableImprovedDR = false + }) + mockStoreExpectApp1 := func(mockStore *mock_store.MockStore) { mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ diff --git a/pkg/kotsadmsnapshot/restore.go b/pkg/kotsadmsnapshot/restore.go index 7d4d23d5c6..c4e5a3a4f7 100644 --- a/pkg/kotsadmsnapshot/restore.go +++ b/pkg/kotsadmsnapshot/restore.go @@ -127,7 +127,7 @@ func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snap return errors.Wrap(err, "failed to create restore") } - // TODO + // TODO(improveddr) // create the kotsKinds.Restore included with the yaml from the vendor // Add the EC annotation From 93cd9a420ba9115c701737e18decfdec10325791 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Wed, 4 Dec 2024 13:59:32 -0800 Subject: [PATCH 28/44] f --- .github/actions/kots-e2e/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/kots-e2e/action.yml b/.github/actions/kots-e2e/action.yml index 9ca204ed26..7a6c6a637f 100644 --- a/.github/actions/kots-e2e/action.yml +++ b/.github/actions/kots-e2e/action.yml @@ -160,6 +160,8 @@ runs: kubectl -n $VELERO_NAMESPACE logs daemonset/node-agent echo "------previous node-agent logs" kubectl -n $VELERO_NAMESPACE logs -p daemonset/node-agent + echo "------velero backup details" + velero describe backups --details shell: bash - name: Generate support bundle on failure From ee38c7d993e7cae36187ed999c164f789a8e374b Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Wed, 4 Dec 2024 14:40:11 -0800 Subject: [PATCH 29/44] f --- .github/actions/kots-e2e/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/kots-e2e/action.yml b/.github/actions/kots-e2e/action.yml index 7a6c6a637f..1ca0c02e9f 100644 --- a/.github/actions/kots-e2e/action.yml +++ b/.github/actions/kots-e2e/action.yml @@ -161,7 +161,9 @@ runs: echo "------previous node-agent logs" kubectl -n $VELERO_NAMESPACE logs -p daemonset/node-agent echo "------velero backup details" - velero describe backups --details + curl -LO https://github.com/vmware-tanzu/velero/releases/download/v1.12.1/velero-v1.12.1-linux-amd64.tar.gz + tar -xvf velero-v1.12.1-linux-amd64.tar.gz + ./velero-v1.12.1-linux-amd64/velero describe backups --details shell: bash - name: Generate support bundle on failure From 7462fb2a7ab6734e70e7c3dca1d4830a9b828f21 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Wed, 4 Dec 2024 15:28:12 -0800 Subject: [PATCH 30/44] f --- pkg/kotsadmsnapshot/backup.go | 10 +++------- pkg/kotsadmsnapshot/backup_test.go | 6 +++++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 232b3a629c..e2a566cd94 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -438,7 +438,7 @@ func getInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, }, Spec: velerov1.BackupSpec{ StorageLocation: "default", - IncludedNamespaces: []string{}, + IncludedNamespaces: []string{metadata.kotsadmNamespace}, ExcludedNamespaces: []string{}, IncludeClusterResources: ptr.To(true), OrLabelSelectors: instanceBackupLabelSelectors(metadata.ec != nil), @@ -449,12 +449,8 @@ func getInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, }, } - appNamespace := metadata.kotsadmNamespace - if os.Getenv("KOTSADM_TARGET_NAMESPACE") != "" { - appNamespace = os.Getenv("KOTSADM_TARGET_NAMESPACE") - } - if appNamespace != metadata.kotsadmNamespace { - veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, appNamespace) + if util.AppNamespace() != metadata.kotsadmNamespace { + veleroBackup.Spec.IncludedNamespaces = append(veleroBackup.Spec.IncludedNamespaces, util.AppNamespace()) } isKurl, err := kurl.IsKurl(k8sClient) diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 3dc4eab8de..b3c1d25dca 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -2087,7 +2087,10 @@ func Test_getInstanceBackupSpec(t *testing.T) { { name: "KOTSADM_TARGET_NAMESPACE should be added to includedNamespaces", setup: func(t *testing.T, mockStore *mock_store.MockStore) { - t.Setenv("KOTSADM_TARGET_NAMESPACE", "kotsadm-target") + util.KotsadmTargetNamespace = "kotsadm-target" + t.Cleanup(func() { + util.KotsadmTargetNamespace = "" + }) mockStoreExpectApp1(mockStore) }, @@ -2112,6 +2115,7 @@ func Test_getInstanceBackupSpec(t *testing.T) { }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "kotsadm") assert.Contains(t, got.Spec.IncludedNamespaces, "kotsadm-target") }, }, From 3e26690682ccc20c743c0b92ae69dd82d0f76ef5 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Wed, 4 Dec 2024 15:58:54 -0800 Subject: [PATCH 31/44] f --- pkg/kotsadmsnapshot/backup.go | 13 ++++-- pkg/kotsadmsnapshot/backup_test.go | 70 +++++++++++++++--------------- 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index e2a566cd94..07ca05a7a9 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -319,7 +319,7 @@ func GetInstanceBackupsExpected(veleroBackup velerov1.Backup) int { func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interface, ctrlClient ctrlclient.Client, veleroClient veleroclientv1.VeleroV1Interface, cluster *downstreamtypes.Downstream, isScheduled bool) (instanceBackupMetadata, error) { now := time.Now().UTC() metadata := instanceBackupMetadata{ - backupName: fmt.Sprintf("backup-%d", now.UnixNano()), + backupName: fmt.Sprintf("instance-%d", now.UnixNano()), backupReqestedAt: now, kotsadmNamespace: util.PodNamespace, apps: make(map[string]appInstanceBackupMetadata, 0), @@ -391,6 +391,11 @@ func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interfa parentSequence: parentSequence, } + // if there's only one app, use the slug as the backup name + if len(apps) == 1 && len(metadata.apps) == 1 { + metadata.backupName = fmt.Sprintf("%s-%d", app.Slug, now.UnixNano()) + } + // optimization as we no longer need the archive dir _ = os.RemoveAll(archiveDir) } @@ -433,7 +438,7 @@ func getInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, veleroBackup := &velerov1.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "instance-", + GenerateName: "infrastructure-", Annotations: map[string]string{}, }, Spec: velerov1.BackupSpec{ @@ -519,7 +524,7 @@ func getAppInstanceBackupSpec(k8sClient kubernetes.Interface, metadata instanceB var appVeleroBackup *velerov1.Backup - for slug, appMeta := range metadata.apps { + for _, appMeta := range metadata.apps { // if there is both a backup and a restore spec this is using the new improved DR if appMeta.kotsKinds.Backup == nil || appMeta.kotsKinds.Restore == nil { continue @@ -548,7 +553,7 @@ func getAppInstanceBackupSpec(k8sClient kubernetes.Interface, metadata instanceB } appVeleroBackup.Name = "" - appVeleroBackup.GenerateName = slug + "-" + appVeleroBackup.GenerateName = "application-" break } diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index b3c1d25dca..db288bcb81 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -977,7 +977,7 @@ func Test_appendCommonAnnotations(t *testing.T) { k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), annotations: map[string]string{}, metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -1021,7 +1021,7 @@ func Test_appendCommonAnnotations(t *testing.T) { "kots.io/kotsadm-image": "kotsadm/kotsadm:1.0.0", "kots.io/snapshot-requested": "2024-01-01T00:00:00Z", "kots.io/snapshot-trigger": "manual", - "replicated.com/backup-name": "backup-17332487841234", + "replicated.com/backup-name": "instance-17332487841234", "replicated.com/backups-expected": "1", }, }, @@ -1035,7 +1035,7 @@ func Test_appendCommonAnnotations(t *testing.T) { k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), annotations: map[string]string{}, metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -1087,7 +1087,7 @@ func Test_appendCommonAnnotations(t *testing.T) { "kots.io/kotsadm-image": "kotsadm/kotsadm:1.0.0", "kots.io/snapshot-requested": "2024-01-01T00:00:00Z", "kots.io/snapshot-trigger": "schedule", - "replicated.com/backup-name": "backup-17332487841234", + "replicated.com/backup-name": "instance-17332487841234", "replicated.com/backups-expected": "2", "kots.io/embedded-cluster": "true", "kots.io/embedded-cluster-id": "embedded-cluster-id", @@ -1153,7 +1153,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "instance-", + GenerateName: "infrastructure-", Annotations: map[string]string{ "annotation": "true", }, @@ -1193,7 +1193,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "instance-", + GenerateName: "infrastructure-", Annotations: map[string]string{ "annotation": "true", }, @@ -1217,7 +1217,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "instance-", + GenerateName: "infrastructure-", Annotations: map[string]string{ "annotation": "true", }, @@ -1287,7 +1287,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "instance-", + GenerateName: "infrastructure-", Annotations: map[string]string{ "annotation": "true", "annotation-1": "true", @@ -1325,7 +1325,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "instance-", + GenerateName: "infrastructure-", Annotations: map[string]string{ "annotation": "true", }, @@ -1365,7 +1365,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "instance-", + GenerateName: "infrastructure-", Annotations: map[string]string{ "annotation": "true", }, @@ -1389,7 +1389,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "instance-", + GenerateName: "infrastructure-", Annotations: map[string]string{ "annotation": "true", }, @@ -1459,7 +1459,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "instance-", + GenerateName: "infrastructure-", Annotations: map[string]string{ "annotation": "true", "annotation-1": "true", @@ -1683,7 +1683,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -1714,7 +1714,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -1781,7 +1781,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -1799,7 +1799,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) assert.Equal(t, "", got.Name) - assert.Equal(t, "app-1-", got.GenerateName) + assert.Equal(t, "application-", got.GenerateName) }, }, { @@ -1813,7 +1813,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -1831,7 +1831,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) if assert.Contains(t, got.Annotations, "replicated.com/backup-name") { - assert.Equal(t, "backup-17332487841234", got.Annotations["replicated.com/backup-name"]) + assert.Equal(t, "instance-17332487841234", got.Annotations["replicated.com/backup-name"]) } if assert.Contains(t, got.Annotations, "replicated.com/backup-type") { assert.Equal(t, "app", got.Annotations["replicated.com/backup-type"]) @@ -1852,7 +1852,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -1883,7 +1883,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -1915,7 +1915,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2097,7 +2097,7 @@ func Test_getInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2132,7 +2132,7 @@ func Test_getInstanceBackupSpec(t *testing.T) { }, }), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2161,7 +2161,7 @@ func Test_getInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2205,7 +2205,7 @@ func Test_getInstanceBackupSpec(t *testing.T) { }, }), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2237,7 +2237,7 @@ func Test_getInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2270,7 +2270,7 @@ func Test_getInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2305,7 +2305,7 @@ func Test_getInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2337,7 +2337,7 @@ func Test_getInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2369,7 +2369,7 @@ func Test_getInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2402,7 +2402,7 @@ func Test_getInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2434,7 +2434,7 @@ func Test_getInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2616,7 +2616,7 @@ spec: isScheduled: true, }, want: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "instance-", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "test", backupStorageLocationNamespace: "velero", @@ -2741,7 +2741,7 @@ spec: isScheduled: true, }, want: instanceBackupMetadata{ - backupName: "backup-17332487841234", + backupName: "app-1-", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "test", backupStorageLocationNamespace: "velero", @@ -2808,7 +2808,7 @@ spec: require.NoError(t, err) } - assert.Regexp(t, "^backup-", got.backupName) + assert.Regexp(t, "^"+tt.want.backupName, got.backupName) assert.NotZero(t, got.backupReqestedAt) tt.want.backupName = got.backupName tt.want.backupReqestedAt = got.backupReqestedAt From 5408fe06d1319666f2381b5b9e3b4a56ac7889ea Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Wed, 4 Dec 2024 16:30:18 -0800 Subject: [PATCH 32/44] f --- pkg/kotsadmsnapshot/backup.go | 2 +- pkg/kotsadmsnapshot/backup_test.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 07ca05a7a9..3267efa9dc 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -438,7 +438,7 @@ func getInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, veleroBackup := &velerov1.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "infrastructure-", + GenerateName: "instance-", Annotations: map[string]string{}, }, Spec: velerov1.BackupSpec{ diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index db288bcb81..4c6992c5f1 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -1153,7 +1153,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "infrastructure-", + GenerateName: "instance-", Annotations: map[string]string{ "annotation": "true", }, @@ -1193,7 +1193,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "infrastructure-", + GenerateName: "instance-", Annotations: map[string]string{ "annotation": "true", }, @@ -1217,7 +1217,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "infrastructure-", + GenerateName: "instance-", Annotations: map[string]string{ "annotation": "true", }, @@ -1287,7 +1287,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "infrastructure-", + GenerateName: "instance-", Annotations: map[string]string{ "annotation": "true", "annotation-1": "true", @@ -1325,7 +1325,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "infrastructure-", + GenerateName: "instance-", Annotations: map[string]string{ "annotation": "true", }, @@ -1365,7 +1365,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "infrastructure-", + GenerateName: "instance-", Annotations: map[string]string{ "annotation": "true", }, @@ -1389,7 +1389,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "infrastructure-", + GenerateName: "instance-", Annotations: map[string]string{ "annotation": "true", }, @@ -1459,7 +1459,7 @@ func Test_mergeAppBackupSpec(t *testing.T) { }, ObjectMeta: metav1.ObjectMeta{ Name: "", - GenerateName: "infrastructure-", + GenerateName: "instance-", Annotations: map[string]string{ "annotation": "true", "annotation-1": "true", From 320b39e1a92e787b514f1fcbde8be72e41cb7927 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Thu, 5 Dec 2024 06:14:35 -0800 Subject: [PATCH 33/44] rename function --- pkg/kotsadmsnapshot/backup.go | 8 ++++---- pkg/kotsadmsnapshot/backup_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 3267efa9dc..e9961afb8c 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -256,7 +256,7 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return "", errors.Wrap(err, "failed to get app instance backup spec") } - veleroBackup, err := getInstanceBackupSpec(ctx, k8sClient, metadata, appVeleroBackup != nil) + veleroBackup, err := getInfrastructureInstanceBackupSpec(ctx, k8sClient, metadata, appVeleroBackup != nil) if err != nil { return "", errors.Wrap(err, "failed to get instance backup specs") } @@ -431,9 +431,9 @@ func getECInstanceBackupMetadata(ctx context.Context, ctrlClient ctrlclient.Clie }, nil } -// getInstanceBackupSpec returns the velero backup spec for the instance backup. This is either the -// kotsadm backup or the combined backup if this is not using improved DR. -func getInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata, hasAppSpec bool) (*velerov1.Backup, error) { +// getInfrastructureInstanceBackupSpec returns the velero backup spec for the instance backup. This +// is either the kotsadm backup or the combined backup if this is not using improved DR. +func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata, hasAppSpec bool) (*velerov1.Backup, error) { // veleroBackup is the kotsadm backup or combined backup if usesImprovedDR is false veleroBackup := &velerov1.Backup{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 4c6992c5f1..14c17b9ca6 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -1957,7 +1957,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { } } -func Test_getInstanceBackupSpec(t *testing.T) { +func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { kotsadmSts := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: "kotsadm", @@ -2477,7 +2477,7 @@ func Test_getInstanceBackupSpec(t *testing.T) { if tt.setup != nil { tt.setup(t, mockStore) } - got, err := getInstanceBackupSpec(context.Background(), tt.args.k8sClient, tt.args.metadata, tt.args.hasAppSpec) + got, err := getInfrastructureInstanceBackupSpec(context.Background(), tt.args.k8sClient, tt.args.metadata, tt.args.hasAppSpec) tt.assert(t, got, err) }) } From fb063e854279cc0b873ac78ee431f16db3862991 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Thu, 5 Dec 2024 12:35:26 -0600 Subject: [PATCH 34/44] feat(ec): use label for backup name, use common terminology (#5031) --- pkg/kotsadmsnapshot/backup.go | 114 ++++++++++++++++------- pkg/kotsadmsnapshot/backup_test.go | 139 ++++++++++++++++++++++++----- 2 files changed, 199 insertions(+), 54 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index e9961afb8c..5da8af25f8 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -2,14 +2,17 @@ package snapshot import ( "context" + "crypto/sha256" "encoding/json" "fmt" "math" "os" "strconv" + "strings" "time" units "github.com/docker/go-units" + "github.com/google/uuid" "github.com/pkg/errors" embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types" @@ -34,18 +37,28 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/kubernetes" "k8s.io/utils/ptr" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ) const ( - InstanceBackupNameAnnotation = "replicated.com/backup-name" - InstanceBackupTypeAnnotation = "replicated.com/backup-type" - InstanceBackupsExpectedAnnotation = "replicated.com/backups-expected" - - InstanceBackupTypeKotsadm = "kotsadm" - InstanceBackupTypeApp = "app" + // InstanceBackupNameLabel is the label used to store the name of the backup for an instance + // backup. + InstanceBackupNameLabel = "replicated.com/backup-name" + // InstanceBackupTypeAnnotation is the annotation used to store the type of backup for an + // instance backup. + InstanceBackupTypeAnnotation = "replicated.com/backup-type" + // InstanceBackupCountAnnotation is the annotation used to store the expected number of backups + // for an instance backup. + InstanceBackupCountAnnotation = "replicated.com/backup-count" + + // InstanceBackupTypeInfra indicates that the backup is of type infrastructure. + InstanceBackupTypeInfra = "infra" + // InstanceBackupTypeApp indicates that the backup is of type application. + InstanceBackupTypeApp = "app" + // InstanceBackupTypeCombined indicates that the backup is of type combined (infra + app). InstanceBackupTypeCombined = "combined" ) @@ -290,13 +303,15 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre return backup.Name, nil // TODO(improveddr): return metadata.BackupName } +// GetBackupName returns the name of the backup from the velero backup object label. func GetBackupName(veleroBackup velerov1.Backup) string { - if val, ok := veleroBackup.GetAnnotations()[InstanceBackupNameAnnotation]; ok { + if val, ok := veleroBackup.GetLabels()[InstanceBackupNameLabel]; ok { return val } return veleroBackup.GetName() } +// GetInstanceBackupType returns the type of the backup from the velero backup object annotation. func GetInstanceBackupType(veleroBackup velerov1.Backup) string { if val, ok := veleroBackup.GetAnnotations()[InstanceBackupTypeAnnotation]; ok { return val @@ -304,8 +319,10 @@ func GetInstanceBackupType(veleroBackup velerov1.Backup) string { return "" } -func GetInstanceBackupsExpected(veleroBackup velerov1.Backup) int { - if val, ok := veleroBackup.GetAnnotations()[InstanceBackupsExpectedAnnotation]; ok { +// GetInstanceBackupCount returns the expected number of backups from the velero backup object +// annotation. +func GetInstanceBackupCount(veleroBackup velerov1.Backup) int { + if val, ok := veleroBackup.GetAnnotations()[InstanceBackupCountAnnotation]; ok { num, _ := strconv.Atoi(val) if num > 0 { return num @@ -317,10 +334,9 @@ func GetInstanceBackupsExpected(veleroBackup velerov1.Backup) int { // getInstanceBackupMetadata returns metadata about the instance backup for use in creating an // instance backup. func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interface, ctrlClient ctrlclient.Client, veleroClient veleroclientv1.VeleroV1Interface, cluster *downstreamtypes.Downstream, isScheduled bool) (instanceBackupMetadata, error) { - now := time.Now().UTC() metadata := instanceBackupMetadata{ - backupName: fmt.Sprintf("instance-%d", now.UnixNano()), - backupReqestedAt: now, + backupName: getBackupNameFromPrefix("instance"), + backupReqestedAt: time.Now().UTC(), kotsadmNamespace: util.PodNamespace, apps: make(map[string]appInstanceBackupMetadata, 0), isScheduled: isScheduled, @@ -393,7 +409,7 @@ func getInstanceBackupMetadata(ctx context.Context, k8sClient kubernetes.Interfa // if there's only one app, use the slug as the backup name if len(apps) == 1 && len(metadata.apps) == 1 { - metadata.backupName = fmt.Sprintf("%s-%d", app.Slug, now.UnixNano()) + metadata.backupName = getBackupNameFromPrefix(app.Slug) } // optimization as we no longer need the archive dir @@ -487,8 +503,13 @@ func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernet if err != nil { return nil, errors.Wrap(err, "failed to add annotations to backup") } + // Add improved disaster recovery annotations and labels + if veleroBackup.Labels == nil { + veleroBackup.Labels = map[string]string{} + } + veleroBackup.Labels[InstanceBackupNameLabel] = metadata.backupName if hasAppSpec { - veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeKotsadm + veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeInfra } else { veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeCombined } @@ -567,6 +588,11 @@ func getAppInstanceBackupSpec(k8sClient kubernetes.Interface, metadata instanceB if err != nil { return nil, errors.Wrap(err, "failed to add annotations to application backup") } + // Add improved disaster recovery annotations and labels + if appVeleroBackup.Labels == nil { + appVeleroBackup.Labels = map[string]string{} + } + appVeleroBackup.Labels[InstanceBackupNameLabel] = metadata.backupName appVeleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeApp appVeleroBackup.Spec.StorageLocation = "default" @@ -699,8 +725,7 @@ func appendCommonAnnotations(k8sClient kubernetes.Interface, annotations map[str } // Add improved disaster recovery annotation labels - annotations[InstanceBackupNameAnnotation] = metadata.backupName - annotations[InstanceBackupsExpectedAnnotation] = strconv.Itoa(numBackups) + annotations[InstanceBackupCountAnnotation] = strconv.Itoa(numBackups) if metadata.ec != nil { annotations = appendECAnnotations(annotations, *metadata.ec) @@ -747,7 +772,7 @@ func ListBackupsForApp(ctx context.Context, kotsadmNamespace string, appID strin } backup := types.Backup{ - Name: veleroBackup.Name, // TODO(improveddr): GetBackupName(veleroBackup), + Name: veleroBackup.Name, Status: string(veleroBackup.Status.Phase), AppID: appID, } @@ -886,7 +911,7 @@ func ListInstanceBackups(ctx context.Context, kotsadmNamespace string) ([]*types } backup := types.Backup{ - Name: veleroBackup.Name, // TODO(improveddr): GetBackupName(veleroBackup), + Name: veleroBackup.Name, Status: string(veleroBackup.Status.Phase), IncludedApps: make([]types.App, 0), } @@ -1024,7 +1049,7 @@ func getSnapshotVolumeSummary(ctx context.Context, veleroBackup *velerov1.Backup return &volumeSummary, nil } -func GetBackup(ctx context.Context, kotsadmNamespace string, snapshotName string) (*velerov1.Backup, error) { +func GetBackup(ctx context.Context, kotsadmNamespace string, backupID string) (*velerov1.Backup, error) { cfg, err := k8sutil.GetClusterConfig() if err != nil { return nil, errors.Wrap(err, "failed to get cluster config") @@ -1050,8 +1075,7 @@ func GetBackup(ctx context.Context, kotsadmNamespace string, snapshotName string veleroNamespace := bsl.Namespace - // get the backup - backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, snapshotName, metav1.GetOptions{}) + backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, backupID, metav1.GetOptions{}) if err != nil { return nil, errors.Wrap(err, "failed to get backup") } @@ -1059,7 +1083,35 @@ func GetBackup(ctx context.Context, kotsadmNamespace string, snapshotName string return backup, nil } -func DeleteBackup(ctx context.Context, kotsadmNamespace string, snapshotName string) error { +func getBackupNameFromPrefix(appSlug string) string { + randStr := fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%d-%s", + time.Now().UnixNano(), + strings.Replace(uuid.New().String(), "-", "", 4), + ))))[:8] + backupName := appSlug + if len(backupName)+9 > validation.DNS1035LabelMaxLength { + backupName = backupName[:validation.DNS1035LabelMaxLength-9] + } + return fmt.Sprintf("%s-%s", backupName, randStr) +} + +func getBackupsFromName(ctx context.Context, veleroClient veleroclientv1.VeleroV1Interface, veleroNamespace string, backupName string) ([]velerov1.Backup, error) { + // try to get the restore from the backup name label + backups, err := veleroClient.Backups(veleroNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", InstanceBackupNameLabel, velerolabel.GetValidName(backupName)), + }) + if len(backups.Items) > 0 { + return backups.Items, nil + } + backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, backupName, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrap(err, "failed to get restore") + } + + return []velerov1.Backup{*backup}, nil +} + +func DeleteBackup(ctx context.Context, kotsadmNamespace string, backupID string) error { cfg, err := k8sutil.GetClusterConfig() if err != nil { return errors.Wrap(err, "failed to get cluster config") @@ -1086,11 +1138,11 @@ func DeleteBackup(ctx context.Context, kotsadmNamespace string, snapshotName str veleroNamespace := bsl.Namespace veleroDeleteBackupRequest := &velerov1.DeleteBackupRequest{ ObjectMeta: metav1.ObjectMeta{ - Name: snapshotName, + Name: backupID, Namespace: veleroNamespace, }, Spec: velerov1.DeleteBackupRequestSpec{ - BackupName: snapshotName, + BackupName: backupID, }, } @@ -1132,7 +1184,7 @@ func HasUnfinishedInstanceBackup(ctx context.Context, kotsadmNamespace string) ( return false, nil } -func GetBackupDetail(ctx context.Context, kotsadmNamespace string, backupName string) (*types.BackupDetail, error) { +func GetBackupDetail(ctx context.Context, kotsadmNamespace string, backupID string) (*types.BackupDetail, error) { cfg, err := k8sutil.GetClusterConfig() if err != nil { return nil, errors.Wrap(err, "failed to get cluster config") @@ -1155,20 +1207,20 @@ func GetBackupDetail(ctx context.Context, kotsadmNamespace string, backupName st veleroNamespace := backendStorageLocation.Namespace - backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, backupName, metav1.GetOptions{}) + backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, backupID, metav1.GetOptions{}) if err != nil { return nil, errors.Wrap(err, "failed to get backup") } backupVolumes, err := veleroClient.PodVolumeBackups(veleroNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: fmt.Sprintf("velero.io/backup-name=%s", velerolabel.GetValidName(backupName)), + LabelSelector: fmt.Sprintf("velero.io/backup-name=%s", velerolabel.GetValidName(backupID)), }) if err != nil { return nil, errors.Wrap(err, "failed to list volumes") } result := &types.BackupDetail{ - Name: backup.Name, // TODO(improveddr): GetBackupName(*backup), + Name: backup.Name, Status: string(backup.Status.Phase), Namespaces: backup.Spec.IncludedNamespaces, Volumes: listBackupVolumes(backupVolumes.Items), @@ -1181,7 +1233,7 @@ func GetBackupDetail(ctx context.Context, kotsadmNamespace string, backupName st result.VolumeSizeHuman = units.HumanSize(float64(totalBytesDone)) // TODO: should this be TotalBytes rather than BytesDone? if backup.Status.Phase == velerov1.BackupPhaseCompleted || backup.Status.Phase == velerov1.BackupPhasePartiallyFailed || backup.Status.Phase == velerov1.BackupPhaseFailed { - errs, warnings, execs, err := downloadBackupLogs(ctx, veleroNamespace, backupName) + errs, warnings, execs, err := downloadBackupLogs(ctx, veleroNamespace, backupID) result.Errors = errs result.Warnings = warnings result.Hooks = execs @@ -1229,8 +1281,8 @@ func listBackupVolumes(backupVolumes []velerov1.PodVolumeBackup) []types.Snapsho return volumes } -func downloadBackupLogs(ctx context.Context, veleroNamespace, backupName string) ([]types.SnapshotError, []types.SnapshotError, []*types.SnapshotHook, error) { - gzipReader, err := DownloadRequest(ctx, veleroNamespace, velerov1.DownloadTargetKindBackupLog, backupName) +func downloadBackupLogs(ctx context.Context, veleroNamespace, backupID string) ([]types.SnapshotError, []types.SnapshotError, []*types.SnapshotHook, error) { + gzipReader, err := DownloadRequest(ctx, veleroNamespace, velerov1.DownloadTargetKindBackupLog, backupID) if err != nil { return nil, nil, nil, errors.Wrap(err, "failed to download backup log") } diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 14c17b9ca6..bd23d383a4 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -31,6 +31,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" coretest "k8s.io/client-go/testing" @@ -1021,8 +1022,7 @@ func Test_appendCommonAnnotations(t *testing.T) { "kots.io/kotsadm-image": "kotsadm/kotsadm:1.0.0", "kots.io/snapshot-requested": "2024-01-01T00:00:00Z", "kots.io/snapshot-trigger": "manual", - "replicated.com/backup-name": "instance-17332487841234", - "replicated.com/backups-expected": "1", + "replicated.com/backup-count": "1", }, }, { @@ -1087,8 +1087,7 @@ func Test_appendCommonAnnotations(t *testing.T) { "kots.io/kotsadm-image": "kotsadm/kotsadm:1.0.0", "kots.io/snapshot-requested": "2024-01-01T00:00:00Z", "kots.io/snapshot-trigger": "schedule", - "replicated.com/backup-name": "instance-17332487841234", - "replicated.com/backups-expected": "2", + "replicated.com/backup-count": "2", "kots.io/embedded-cluster": "true", "kots.io/embedded-cluster-id": "embedded-cluster-id", "kots.io/embedded-cluster-version": "embedded-cluster-version", @@ -1781,7 +1780,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "instance-17332487841234", + backupName: "app-1-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -1803,7 +1802,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { }, }, { - name: "ec with backup and restore spec should append common annotations", + name: "ec with backup and restore spec should append backup name label", setup: func(t *testing.T, mockStore *mock_store.MockStore) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") @@ -1813,7 +1812,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "instance-17332487841234", + backupName: "app-1-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -1830,14 +1829,44 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) - if assert.Contains(t, got.Annotations, "replicated.com/backup-name") { - assert.Equal(t, "instance-17332487841234", got.Annotations["replicated.com/backup-name"]) + if assert.Contains(t, got.Labels, "replicated.com/backup-name") { + assert.Equal(t, "app-1-17332487841234", got.Labels["replicated.com/backup-name"]) } + }, + }, + { + name: "ec with backup and restore spec should append common annotations", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) if assert.Contains(t, got.Annotations, "replicated.com/backup-type") { assert.Equal(t, "app", got.Annotations["replicated.com/backup-type"]) } - if assert.Contains(t, got.Annotations, "replicated.com/backups-expected") { - assert.Equal(t, "2", got.Annotations["replicated.com/backups-expected"]) + if assert.Contains(t, got.Annotations, "replicated.com/backup-count") { + assert.Equal(t, "2", got.Annotations["replicated.com/backup-count"]) } }, }, @@ -2084,6 +2113,37 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { args args assert func(t *testing.T, got *velerov1.Backup, err error) }{ + { + name: "should append backup name label", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: nil, + }, + hasAppSpec: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + if assert.Contains(t, got.Labels, "replicated.com/backup-name") { + assert.Equal(t, "app-1-17332487841234", got.Labels["replicated.com/backup-name"]) + } + }, + }, { name: "KOTSADM_TARGET_NAMESPACE should be added to includedNamespaces", setup: func(t *testing.T, mockStore *mock_store.MockStore) { @@ -2097,7 +2157,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "instance-17332487841234", + backupName: "app-1-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2132,7 +2192,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { }, }), metadata: instanceBackupMetadata{ - backupName: "instance-17332487841234", + backupName: "app-1-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2161,7 +2221,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "instance-17332487841234", + backupName: "app-1-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2205,7 +2265,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { }, }), metadata: instanceBackupMetadata{ - backupName: "instance-17332487841234", + backupName: "app-1-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2237,7 +2297,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "instance-17332487841234", + backupName: "app-1-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2270,7 +2330,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "instance-17332487841234", + backupName: "app-1-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2290,7 +2350,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { require.NoError(t, err) assert.NotContains(t, got.Spec.IncludedNamespaces, "include-namespace-1") if assert.Contains(t, got.Annotations, "replicated.com/backup-type") { - assert.Equal(t, "kotsadm", got.Annotations["replicated.com/backup-type"]) + assert.Equal(t, "infra", got.Annotations["replicated.com/backup-type"]) } }, }, @@ -2305,7 +2365,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "instance-17332487841234", + backupName: "app-1-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2337,7 +2397,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "instance-17332487841234", + backupName: "app-1-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2369,7 +2429,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "instance-17332487841234", + backupName: "app-1-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2402,7 +2462,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "instance-17332487841234", + backupName: "app-1-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2434,7 +2494,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), metadata: instanceBackupMetadata{ - backupName: "instance-17332487841234", + backupName: "app-1-17332487841234", backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), kotsadmNamespace: "kotsadm", backupStorageLocationNamespace: "kotsadm-backups", @@ -2830,3 +2890,36 @@ func setupArchiveDirectoriesAndFiles(archiveDir string, files map[string]string) } return nil } + +func Test_getBackupNameFromPrefix(t *testing.T) { + type args struct { + appSlug string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "basic", + args: args{ + appSlug: "test", + }, + want: `^test-[a-f0-9]{8}$`, + }, + { + name: "truncate", + args: args{ + appSlug: "test-truncate-this-string-to-a-valid-backup-name-length", + }, + want: `^test-truncate-this-string-to-a-valid-backup-name-lengt-[a-f0-9]{8}$`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getBackupNameFromPrefix(tt.args.appSlug) + assert.Regexp(t, tt.want, got) + assert.LessOrEqual(t, len(got), validation.DNS1035LabelMaxLength) + }) + } +} From 4727497c7d1d685af24a0da3b2f0e3cfd72d16b4 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Thu, 5 Dec 2024 10:36:50 -0800 Subject: [PATCH 35/44] rename to hasAppBackupSpec --- pkg/kotsadmsnapshot/backup.go | 12 ++++---- pkg/kotsadmsnapshot/backup_test.go | 46 +++++++++++++++--------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 5da8af25f8..5d9cd0026f 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -449,7 +449,7 @@ func getECInstanceBackupMetadata(ctx context.Context, ctrlClient ctrlclient.Clie // getInfrastructureInstanceBackupSpec returns the velero backup spec for the instance backup. This // is either the kotsadm backup or the combined backup if this is not using improved DR. -func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata, hasAppSpec bool) (*velerov1.Backup, error) { +func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata, hasAppBackupSpec bool) (*velerov1.Backup, error) { // veleroBackup is the kotsadm backup or combined backup if usesImprovedDR is false veleroBackup := &velerov1.Backup{ ObjectMeta: metav1.ObjectMeta{ @@ -491,7 +491,7 @@ func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernet for _, appMeta := range metadata.apps { // Don't merge the backup spec if we are using the new improved DR. - if !hasAppSpec { + if !hasAppBackupSpec { err := mergeAppBackupSpec(veleroBackup, appMeta, metadata.kotsadmNamespace, metadata.ec != nil) if err != nil { return nil, errors.Wrap(err, "failed to merge app backup spec") @@ -499,7 +499,7 @@ func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernet } } - veleroBackup.Annotations, err = appendCommonAnnotations(k8sClient, veleroBackup.Annotations, metadata, hasAppSpec) + veleroBackup.Annotations, err = appendCommonAnnotations(k8sClient, veleroBackup.Annotations, metadata, hasAppBackupSpec) if err != nil { return nil, errors.Wrap(err, "failed to add annotations to backup") } @@ -508,7 +508,7 @@ func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernet veleroBackup.Labels = map[string]string{} } veleroBackup.Labels[InstanceBackupNameLabel] = metadata.backupName - if hasAppSpec { + if hasAppBackupSpec { veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeInfra } else { veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeCombined @@ -670,7 +670,7 @@ func mergeAppBackupSpec(backup *velerov1.Backup, appMeta appInstanceBackupMetada } // appendCommonAnnotations appends common annotations to the backup annotations -func appendCommonAnnotations(k8sClient kubernetes.Interface, annotations map[string]string, metadata instanceBackupMetadata, hasAppSpec bool) (map[string]string, error) { +func appendCommonAnnotations(k8sClient kubernetes.Interface, annotations map[string]string, metadata instanceBackupMetadata, hasAppBackupSpec bool) (map[string]string, error) { kotsadmImage, err := k8sutil.FindKotsadmImage(k8sClient, metadata.kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find kotsadm image") @@ -704,7 +704,7 @@ func appendCommonAnnotations(k8sClient kubernetes.Interface, annotations map[str marshalledAppVersions := string(b) numBackups := 1 - if hasAppSpec { + if hasAppBackupSpec { numBackups = 2 } diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index bd23d383a4..803f871146 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -957,10 +957,10 @@ func Test_appendCommonAnnotations(t *testing.T) { } type args struct { - k8sClient kubernetes.Interface - annotations map[string]string - metadata instanceBackupMetadata - hasAppSpec bool + k8sClient kubernetes.Interface + annotations map[string]string + metadata instanceBackupMetadata + hasAppBackupSpec bool } tests := []struct { name string @@ -1010,7 +1010,7 @@ func Test_appendCommonAnnotations(t *testing.T) { snapshotTTL: 24 * time.Hour, ec: nil, }, - hasAppSpec: false, + hasAppBackupSpec: false, }, want: map[string]string{ "kots.io/apps-sequences": "{\"app-1\":1,\"app-2\":2}", @@ -1075,7 +1075,7 @@ func Test_appendCommonAnnotations(t *testing.T) { seaweedFSS3ServiceIP: "10.96.0.10", }, }, - hasAppSpec: true, + hasAppBackupSpec: true, }, want: map[string]string{ "kots.io/apps-sequences": "{\"app-1\":1}", @@ -1106,7 +1106,7 @@ func Test_appendCommonAnnotations(t *testing.T) { if tt.setup != nil { tt.setup(t) } - got, err := appendCommonAnnotations(tt.args.k8sClient, tt.args.annotations, tt.args.metadata, tt.args.hasAppSpec) + got, err := appendCommonAnnotations(tt.args.k8sClient, tt.args.annotations, tt.args.metadata, tt.args.hasAppBackupSpec) if tt.wantErr { require.Error(t, err) } else { @@ -2103,9 +2103,9 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { } type args struct { - k8sClient kubernetes.Interface - metadata instanceBackupMetadata - hasAppSpec bool + k8sClient kubernetes.Interface + metadata instanceBackupMetadata + hasAppBackupSpec bool } tests := []struct { name string @@ -2135,7 +2135,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: nil, }, - hasAppSpec: false, + hasAppBackupSpec: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2171,7 +2171,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: nil, }, - hasAppSpec: false, + hasAppBackupSpec: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2206,7 +2206,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: nil, }, - hasAppSpec: false, + hasAppBackupSpec: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2235,7 +2235,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: nil, }, - hasAppSpec: false, + hasAppBackupSpec: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2279,7 +2279,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: nil, }, - hasAppSpec: false, + hasAppBackupSpec: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2311,7 +2311,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: ecMeta, }, - hasAppSpec: false, + hasAppBackupSpec: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2344,7 +2344,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: ecMeta, }, - hasAppSpec: true, + hasAppBackupSpec: true, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2379,7 +2379,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: ecMeta, }, - hasAppSpec: false, + hasAppBackupSpec: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2411,7 +2411,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: ecMeta, }, - hasAppSpec: false, + hasAppBackupSpec: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2444,7 +2444,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { snapshotTTL: 24 * time.Hour, ec: ecMeta, }, - hasAppSpec: false, + hasAppBackupSpec: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2476,7 +2476,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: ecMeta, }, - hasAppSpec: false, + hasAppBackupSpec: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2508,7 +2508,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: ecMeta, }, - hasAppSpec: false, + hasAppBackupSpec: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2537,7 +2537,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { if tt.setup != nil { tt.setup(t, mockStore) } - got, err := getInfrastructureInstanceBackupSpec(context.Background(), tt.args.k8sClient, tt.args.metadata, tt.args.hasAppSpec) + got, err := getInfrastructureInstanceBackupSpec(context.Background(), tt.args.k8sClient, tt.args.metadata, tt.args.hasAppBackupSpec) tt.assert(t, got, err) }) } From 6ebf5963dd7ea8ce2ce48551b95b26de6bb55916 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Thu, 5 Dec 2024 10:38:15 -0800 Subject: [PATCH 36/44] type legacy --- pkg/kotsadmsnapshot/backup.go | 10 +++++----- pkg/kotsadmsnapshot/backup_test.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 5d9cd0026f..47dcb30fb1 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -58,8 +58,8 @@ const ( InstanceBackupTypeInfra = "infra" // InstanceBackupTypeApp indicates that the backup is of type application. InstanceBackupTypeApp = "app" - // InstanceBackupTypeCombined indicates that the backup is of type combined (infra + app). - InstanceBackupTypeCombined = "combined" + // InstanceBackupTypeLegacy indicates that the backup is of type legacy (infra + app). + InstanceBackupTypeLegacy = "legacy" ) func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled bool) (*velerov1.Backup, error) { @@ -448,9 +448,9 @@ func getECInstanceBackupMetadata(ctx context.Context, ctrlClient ctrlclient.Clie } // getInfrastructureInstanceBackupSpec returns the velero backup spec for the instance backup. This -// is either the kotsadm backup or the combined backup if this is not using improved DR. +// is either the kotsadm backup or the legacy backup if this is not using improved DR. func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata, hasAppBackupSpec bool) (*velerov1.Backup, error) { - // veleroBackup is the kotsadm backup or combined backup if usesImprovedDR is false + // veleroBackup is the kotsadm backup or legacy backup if usesImprovedDR is false veleroBackup := &velerov1.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "", @@ -511,7 +511,7 @@ func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernet if hasAppBackupSpec { veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeInfra } else { - veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeCombined + veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeLegacy } if metadata.ec != nil { diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 803f871146..72cd712b38 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -2317,7 +2317,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { require.NoError(t, err) assert.Contains(t, got.Spec.IncludedNamespaces, "include-namespace-1") if assert.Contains(t, got.Annotations, "replicated.com/backup-type") { - assert.Equal(t, "combined", got.Annotations["replicated.com/backup-type"]) + assert.Equal(t, "legacy", got.Annotations["replicated.com/backup-type"]) } }, }, From f92b5a142ceca5873a339d2495dc2d6f03816f57 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Thu, 5 Dec 2024 10:49:11 -0800 Subject: [PATCH 37/44] dont render app spec for improved dr --- pkg/kotsadmsnapshot/backup.go | 10 +++--- pkg/kotsadmsnapshot/backup_test.go | 57 +++++++++++++++++------------- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 47dcb30fb1..b5a062949e 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -134,6 +134,8 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b return nil, errors.Errorf("application %s does not have a backup spec", a.Slug) } + // We have to render the backup spec as older versions of kots stored the unrendered spec in + // the database. renderedBackup, err := helper.RenderAppFile(a, nil, []byte(backupSpec), kotsKinds, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to render backup") @@ -564,11 +566,7 @@ func getAppInstanceBackupSpec(k8sClient kubernetes.Interface, metadata instanceB return nil, errors.New("backup spec is empty, this is unexpected") } - renderedBackup, err := helper.RenderAppFile(appMeta.app, nil, []byte(backupSpec), appMeta.kotsKinds, metadata.kotsadmNamespace) - if err != nil { - return nil, errors.Wrap(err, "failed to render backup") - } - appVeleroBackup, err = kotsutil.LoadBackupFromContents(renderedBackup) + appVeleroBackup, err = kotsutil.LoadBackupFromContents([]byte(backupSpec)) if err != nil { return nil, errors.Wrap(err, "failed to load backup from contents") } @@ -627,6 +625,8 @@ func mergeAppBackupSpec(backup *velerov1.Backup, appMeta appInstanceBackupMetada return nil } + // We have to render the backup spec as older versions of kots stored the unrendered spec in + // the database. renderedBackup, err := helper.RenderAppFile(appMeta.app, nil, []byte(backupSpec), appMeta.kotsKinds, kotsadmNamespace) if err != nil { return errors.Wrap(err, "failed to render backup") diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 72cd712b38..84a2d3ac90 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -1519,17 +1519,6 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { EnableImprovedDR = false }) - mockStoreExpectApp1 := func(mockStore *mock_store.MockStore) { - mockStore.EXPECT().GetLatestAppSequence("1", true).Times(1).Return(int64(1), nil) - mockStore.EXPECT().GetRegistryDetailsForApp("1").Times(1).Return(registrytypes.RegistrySettings{ - Hostname: "hostname", - Username: "username", - Password: "password", - Namespace: "namespace", - IsReadOnly: true, - }, nil) - } - kotsadmSts := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: "kotsadm", @@ -1597,7 +1586,7 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { Spec: velerov1.BackupSpec{ StorageLocation: "blah", TTL: metav1.Duration{Duration: 1 * time.Hour}, - IncludedNamespaces: []string{"include-namespace-1", "include-namespace-2", "template-isairgap-{{repl IsAirgap }}"}, + IncludedNamespaces: []string{"include-namespace-1", "include-namespace-2"}, ExcludedNamespaces: []string{"exclude-namespace-1", "exclude-namespace-2"}, OrderedResources: map[string]string{ "resource-1": "true", @@ -1774,8 +1763,6 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { setup: func(t *testing.T, mockStore *mock_store.MockStore) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") - - mockStoreExpectApp1(mockStore) }, args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), @@ -1806,8 +1793,6 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { setup: func(t *testing.T, mockStore *mock_store.MockStore) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") - - mockStoreExpectApp1(mockStore) }, args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), @@ -1839,8 +1824,6 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { setup: func(t *testing.T, mockStore *mock_store.MockStore) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") - - mockStoreExpectApp1(mockStore) }, args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), @@ -1875,8 +1858,6 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { setup: func(t *testing.T, mockStore *mock_store.MockStore) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") - - mockStoreExpectApp1(mockStore) }, args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), @@ -1906,8 +1887,6 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { setup: func(t *testing.T, mockStore *mock_store.MockStore) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") - - mockStoreExpectApp1(mockStore) }, args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), @@ -1938,8 +1917,6 @@ func Test_getAppInstanceBackupSpec(t *testing.T) { setup: func(t *testing.T, mockStore *mock_store.MockStore) { t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") - - mockStoreExpectApp1(mockStore) }, args: args{ k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), @@ -2521,6 +2498,38 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { assert.Equal(t, 1, count, "Duplicate namespace should be removed") }, }, + { + name: "should render app backup spec", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppBackupSpec: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.Contains(t, got.Spec.IncludedNamespaces, "template-isairgap-true") + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 29d5e55416e0cf28590b943a49f30cc9d4b1ae40 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Thu, 5 Dec 2024 10:50:59 -0800 Subject: [PATCH 38/44] dont render app spec for improved dr --- pkg/kotsadmsnapshot/backup.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index b5a062949e..7c56140acf 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -557,19 +557,11 @@ func getAppInstanceBackupSpec(k8sClient kubernetes.Interface, metadata instanceB return nil, errors.New("cannot create backup for Embedded Cluster with multiple apps") } - backupSpec, err := appMeta.kotsKinds.Marshal("velero.io", "v1", "Backup") - if err != nil { - return nil, errors.Wrap(err, "failed to get backup spec from kotskinds") - } - - if backupSpec == "" { + if appMeta.kotsKinds.Backup == nil { return nil, errors.New("backup spec is empty, this is unexpected") } - appVeleroBackup, err = kotsutil.LoadBackupFromContents([]byte(backupSpec)) - if err != nil { - return nil, errors.Wrap(err, "failed to load backup from contents") - } + appVeleroBackup = appMeta.kotsKinds.Backup appVeleroBackup.Name = "" appVeleroBackup.GenerateName = "application-" From a1170e3790b363cc70a545a2da567a5e62cdcf2d Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Thu, 5 Dec 2024 10:56:57 -0800 Subject: [PATCH 39/44] rename to hasAppBackup --- pkg/kotsadmsnapshot/backup.go | 12 ++++---- pkg/kotsadmsnapshot/backup_test.go | 48 +++++++++++++++--------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 7c56140acf..bb7fd1284d 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -451,7 +451,7 @@ func getECInstanceBackupMetadata(ctx context.Context, ctrlClient ctrlclient.Clie // getInfrastructureInstanceBackupSpec returns the velero backup spec for the instance backup. This // is either the kotsadm backup or the legacy backup if this is not using improved DR. -func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata, hasAppBackupSpec bool) (*velerov1.Backup, error) { +func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernetes.Interface, metadata instanceBackupMetadata, hasAppBackup bool) (*velerov1.Backup, error) { // veleroBackup is the kotsadm backup or legacy backup if usesImprovedDR is false veleroBackup := &velerov1.Backup{ ObjectMeta: metav1.ObjectMeta{ @@ -493,7 +493,7 @@ func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernet for _, appMeta := range metadata.apps { // Don't merge the backup spec if we are using the new improved DR. - if !hasAppBackupSpec { + if !hasAppBackup { err := mergeAppBackupSpec(veleroBackup, appMeta, metadata.kotsadmNamespace, metadata.ec != nil) if err != nil { return nil, errors.Wrap(err, "failed to merge app backup spec") @@ -501,7 +501,7 @@ func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernet } } - veleroBackup.Annotations, err = appendCommonAnnotations(k8sClient, veleroBackup.Annotations, metadata, hasAppBackupSpec) + veleroBackup.Annotations, err = appendCommonAnnotations(k8sClient, veleroBackup.Annotations, metadata, hasAppBackup) if err != nil { return nil, errors.Wrap(err, "failed to add annotations to backup") } @@ -510,7 +510,7 @@ func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernet veleroBackup.Labels = map[string]string{} } veleroBackup.Labels[InstanceBackupNameLabel] = metadata.backupName - if hasAppBackupSpec { + if hasAppBackup { veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeInfra } else { veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeLegacy @@ -662,7 +662,7 @@ func mergeAppBackupSpec(backup *velerov1.Backup, appMeta appInstanceBackupMetada } // appendCommonAnnotations appends common annotations to the backup annotations -func appendCommonAnnotations(k8sClient kubernetes.Interface, annotations map[string]string, metadata instanceBackupMetadata, hasAppBackupSpec bool) (map[string]string, error) { +func appendCommonAnnotations(k8sClient kubernetes.Interface, annotations map[string]string, metadata instanceBackupMetadata, hasAppBackup bool) (map[string]string, error) { kotsadmImage, err := k8sutil.FindKotsadmImage(k8sClient, metadata.kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find kotsadm image") @@ -696,7 +696,7 @@ func appendCommonAnnotations(k8sClient kubernetes.Interface, annotations map[str marshalledAppVersions := string(b) numBackups := 1 - if hasAppBackupSpec { + if hasAppBackup { numBackups = 2 } diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 84a2d3ac90..3ef6597ab6 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -957,10 +957,10 @@ func Test_appendCommonAnnotations(t *testing.T) { } type args struct { - k8sClient kubernetes.Interface - annotations map[string]string - metadata instanceBackupMetadata - hasAppBackupSpec bool + k8sClient kubernetes.Interface + annotations map[string]string + metadata instanceBackupMetadata + hasAppBackup bool } tests := []struct { name string @@ -1010,7 +1010,7 @@ func Test_appendCommonAnnotations(t *testing.T) { snapshotTTL: 24 * time.Hour, ec: nil, }, - hasAppBackupSpec: false, + hasAppBackup: false, }, want: map[string]string{ "kots.io/apps-sequences": "{\"app-1\":1,\"app-2\":2}", @@ -1075,7 +1075,7 @@ func Test_appendCommonAnnotations(t *testing.T) { seaweedFSS3ServiceIP: "10.96.0.10", }, }, - hasAppBackupSpec: true, + hasAppBackup: true, }, want: map[string]string{ "kots.io/apps-sequences": "{\"app-1\":1}", @@ -1106,7 +1106,7 @@ func Test_appendCommonAnnotations(t *testing.T) { if tt.setup != nil { tt.setup(t) } - got, err := appendCommonAnnotations(tt.args.k8sClient, tt.args.annotations, tt.args.metadata, tt.args.hasAppBackupSpec) + got, err := appendCommonAnnotations(tt.args.k8sClient, tt.args.annotations, tt.args.metadata, tt.args.hasAppBackup) if tt.wantErr { require.Error(t, err) } else { @@ -2080,9 +2080,9 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { } type args struct { - k8sClient kubernetes.Interface - metadata instanceBackupMetadata - hasAppBackupSpec bool + k8sClient kubernetes.Interface + metadata instanceBackupMetadata + hasAppBackup bool } tests := []struct { name string @@ -2112,7 +2112,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: nil, }, - hasAppBackupSpec: false, + hasAppBackup: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2148,7 +2148,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: nil, }, - hasAppBackupSpec: false, + hasAppBackup: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2183,7 +2183,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: nil, }, - hasAppBackupSpec: false, + hasAppBackup: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2212,7 +2212,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: nil, }, - hasAppBackupSpec: false, + hasAppBackup: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2256,7 +2256,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: nil, }, - hasAppBackupSpec: false, + hasAppBackup: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2288,7 +2288,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: ecMeta, }, - hasAppBackupSpec: false, + hasAppBackup: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2321,7 +2321,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: ecMeta, }, - hasAppBackupSpec: true, + hasAppBackup: true, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2356,7 +2356,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: ecMeta, }, - hasAppBackupSpec: false, + hasAppBackup: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2388,7 +2388,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: ecMeta, }, - hasAppBackupSpec: false, + hasAppBackup: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2421,7 +2421,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { snapshotTTL: 24 * time.Hour, ec: ecMeta, }, - hasAppBackupSpec: false, + hasAppBackup: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2453,7 +2453,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: ecMeta, }, - hasAppBackupSpec: false, + hasAppBackup: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2485,7 +2485,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: ecMeta, }, - hasAppBackupSpec: false, + hasAppBackup: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2523,7 +2523,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { isScheduled: true, ec: ecMeta, }, - hasAppBackupSpec: false, + hasAppBackup: false, }, assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) @@ -2546,7 +2546,7 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { if tt.setup != nil { tt.setup(t, mockStore) } - got, err := getInfrastructureInstanceBackupSpec(context.Background(), tt.args.k8sClient, tt.args.metadata, tt.args.hasAppBackupSpec) + got, err := getInfrastructureInstanceBackupSpec(context.Background(), tt.args.k8sClient, tt.args.metadata, tt.args.hasAppBackup) tt.assert(t, got, err) }) } From 77833f3b7002a4a30182f60b78e6aff1e8aeecf8 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Thu, 5 Dec 2024 11:00:37 -0800 Subject: [PATCH 40/44] disable GetInstanceBackupType check in list handler --- pkg/kotsadmsnapshot/backup.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index bb7fd1284d..475304959e 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -897,11 +897,6 @@ func ListInstanceBackups(ctx context.Context, kotsadmNamespace string) ([]*types continue } - // TODO(improveddr): support for improved DR in UI - if GetInstanceBackupType(veleroBackup) == InstanceBackupTypeApp { - continue - } - backup := types.Backup{ Name: veleroBackup.Name, Status: string(veleroBackup.Status.Phase), From 1d8198e3b814d23456e8ab15e52c0cfe1ebabde3 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Thu, 5 Dec 2024 12:52:48 -0800 Subject: [PATCH 41/44] remove backup detail from ci --- .github/actions/kots-e2e/action.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/actions/kots-e2e/action.yml b/.github/actions/kots-e2e/action.yml index 1ca0c02e9f..9ca204ed26 100644 --- a/.github/actions/kots-e2e/action.yml +++ b/.github/actions/kots-e2e/action.yml @@ -160,10 +160,6 @@ runs: kubectl -n $VELERO_NAMESPACE logs daemonset/node-agent echo "------previous node-agent logs" kubectl -n $VELERO_NAMESPACE logs -p daemonset/node-agent - echo "------velero backup details" - curl -LO https://github.com/vmware-tanzu/velero/releases/download/v1.12.1/velero-v1.12.1-linux-amd64.tar.gz - tar -xvf velero-v1.12.1-linux-amd64.tar.gz - ./velero-v1.12.1-linux-amd64/velero describe backups --details shell: bash - name: Generate support bundle on failure From cd0e471e62d0deb7b5bcc56f9207a28a1c81a820 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Thu, 5 Dec 2024 12:53:08 -0800 Subject: [PATCH 42/44] fix tests --- pkg/kotsadmsnapshot/backup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 475304959e..3701c6afbd 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -561,7 +561,7 @@ func getAppInstanceBackupSpec(k8sClient kubernetes.Interface, metadata instanceB return nil, errors.New("backup spec is empty, this is unexpected") } - appVeleroBackup = appMeta.kotsKinds.Backup + appVeleroBackup = appMeta.kotsKinds.Backup.DeepCopy() appVeleroBackup.Name = "" appVeleroBackup.GenerateName = "application-" From 9d8cd122f9a758f349b7049dbe9c9fe6cddb2681 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Thu, 5 Dec 2024 12:53:50 -0800 Subject: [PATCH 43/44] remove todo comment --- pkg/kotsadmsnapshot/restore.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/kotsadmsnapshot/restore.go b/pkg/kotsadmsnapshot/restore.go index c4e5a3a4f7..97c1d20f0c 100644 --- a/pkg/kotsadmsnapshot/restore.go +++ b/pkg/kotsadmsnapshot/restore.go @@ -127,10 +127,6 @@ func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snap return errors.Wrap(err, "failed to create restore") } - // TODO(improveddr) - // create the kotsKinds.Restore included with the yaml from the vendor - // Add the EC annotation - return nil } From 3daa2157fa19586eb2d308a8238c00803f21d987 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Thu, 5 Dec 2024 14:50:26 -0800 Subject: [PATCH 44/44] dont add new annotations and labels if not improved dr --- pkg/handlers/restore.go | 12 +++- pkg/kotsadmsnapshot/backup.go | 83 +++++++---------------- pkg/kotsadmsnapshot/backup_test.go | 105 +++++++++++++++++++---------- pkg/kotsadmsnapshot/restore.go | 36 +++++----- pkg/kotsadmsnapshot/types/types.go | 23 +++++++ pkg/operator/operator.go | 4 +- pkg/snapshot/backup.go | 7 +- pkg/snapshot/restore.go | 38 +++++++---- 8 files changed, 176 insertions(+), 132 deletions(-) diff --git a/pkg/handlers/restore.go b/pkg/handlers/restore.go index 322539282b..47bbd7919f 100644 --- a/pkg/handlers/restore.go +++ b/pkg/handlers/restore.go @@ -47,6 +47,14 @@ func (h *Handler) CreateApplicationRestore(w http.ResponseWriter, r *http.Reques return } + if snapshot.IsInstanceBackup(*backup) && snapshot.GetInstanceBackupType(*backup) != snapshottypes.InstanceBackupTypeLegacy { + err := errors.New("only legacy type instance backups are restorable") + logger.Error(err) + createRestoreResponse.Error = err.Error() + JSON(w, http.StatusInternalServerError, createRestoreResponse) + return + } + appID := backup.Annotations["kots.io/app-id"] sequence, err := strconv.ParseInt(backup.Annotations["kots.io/app-sequence"], 10, 64) if err != nil { @@ -149,7 +157,7 @@ func (h *Handler) RestoreApps(w http.ResponseWriter, r *http.Request) { return } - if backup.Annotations["kots.io/instance"] != "true" { + if backup.Annotations[snapshottypes.InstanceBackupAnnotation] != "true" { err := errors.Errorf("backup %s is not an instance backup", backup.ObjectMeta.Name) logger.Error(err) restoreResponse.Error = err.Error() @@ -244,7 +252,7 @@ func (h *Handler) GetRestoreAppsStatus(w http.ResponseWriter, r *http.Request) { return } - if backup.Annotations["kots.io/instance"] != "true" { + if backup.Annotations[snapshottypes.InstanceBackupAnnotation] != "true" { err := errors.Errorf("backup %s is not an instance backup", backup.ObjectMeta.Name) logger.Error(err) response.Error = err.Error() diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index 3701c6afbd..dbe8e98e6a 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -43,25 +43,6 @@ import ( ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" ) -const ( - // InstanceBackupNameLabel is the label used to store the name of the backup for an instance - // backup. - InstanceBackupNameLabel = "replicated.com/backup-name" - // InstanceBackupTypeAnnotation is the annotation used to store the type of backup for an - // instance backup. - InstanceBackupTypeAnnotation = "replicated.com/backup-type" - // InstanceBackupCountAnnotation is the annotation used to store the expected number of backups - // for an instance backup. - InstanceBackupCountAnnotation = "replicated.com/backup-count" - - // InstanceBackupTypeInfra indicates that the backup is of type infrastructure. - InstanceBackupTypeInfra = "infra" - // InstanceBackupTypeApp indicates that the backup is of type application. - InstanceBackupTypeApp = "app" - // InstanceBackupTypeLegacy indicates that the backup is of type legacy (infra + app). - InstanceBackupTypeLegacy = "legacy" -) - func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled bool) (*velerov1.Backup, error) { downstreams, err := store.GetStore().ListDownstreamsForApp(a.ID) if err != nil { @@ -307,24 +288,32 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre // GetBackupName returns the name of the backup from the velero backup object label. func GetBackupName(veleroBackup velerov1.Backup) string { - if val, ok := veleroBackup.GetLabels()[InstanceBackupNameLabel]; ok { + if val, ok := veleroBackup.GetLabels()[types.InstanceBackupNameLabel]; ok { return val } return veleroBackup.GetName() } +// IsInstanceBackup returns true if the backup is an instance backup. +func IsInstanceBackup(veleroBackup velerov1.Backup) bool { + if val, ok := veleroBackup.GetAnnotations()[types.InstanceBackupAnnotation]; ok { + return val == "true" + } + return false +} + // GetInstanceBackupType returns the type of the backup from the velero backup object annotation. func GetInstanceBackupType(veleroBackup velerov1.Backup) string { - if val, ok := veleroBackup.GetAnnotations()[InstanceBackupTypeAnnotation]; ok { + if val, ok := veleroBackup.GetAnnotations()[types.InstanceBackupTypeAnnotation]; ok { return val } - return "" + return types.InstanceBackupTypeLegacy } // GetInstanceBackupCount returns the expected number of backups from the velero backup object // annotation. func GetInstanceBackupCount(veleroBackup velerov1.Backup) int { - if val, ok := veleroBackup.GetAnnotations()[InstanceBackupCountAnnotation]; ok { + if val, ok := veleroBackup.GetAnnotations()[types.InstanceBackupCountAnnotation]; ok { num, _ := strconv.Atoi(val) if num > 0 { return num @@ -505,15 +494,14 @@ func getInfrastructureInstanceBackupSpec(ctx context.Context, k8sClient kubernet if err != nil { return nil, errors.Wrap(err, "failed to add annotations to backup") } - // Add improved disaster recovery annotations and labels - if veleroBackup.Labels == nil { - veleroBackup.Labels = map[string]string{} - } - veleroBackup.Labels[InstanceBackupNameLabel] = metadata.backupName if hasAppBackup { - veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeInfra - } else { - veleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeLegacy + // Only add improved disaster recovery annotations and labels if we have an app backup + if veleroBackup.Labels == nil { + veleroBackup.Labels = map[string]string{} + } + veleroBackup.Labels[types.InstanceBackupNameLabel] = metadata.backupName + veleroBackup.Annotations[types.InstanceBackupTypeAnnotation] = types.InstanceBackupTypeInfra + veleroBackup.Annotations[types.InstanceBackupCountAnnotation] = strconv.Itoa(2) } if metadata.ec != nil { @@ -582,8 +570,9 @@ func getAppInstanceBackupSpec(k8sClient kubernetes.Interface, metadata instanceB if appVeleroBackup.Labels == nil { appVeleroBackup.Labels = map[string]string{} } - appVeleroBackup.Labels[InstanceBackupNameLabel] = metadata.backupName - appVeleroBackup.Annotations[InstanceBackupTypeAnnotation] = InstanceBackupTypeApp + appVeleroBackup.Labels[types.InstanceBackupNameLabel] = metadata.backupName + appVeleroBackup.Annotations[types.InstanceBackupTypeAnnotation] = types.InstanceBackupTypeApp + appVeleroBackup.Annotations[types.InstanceBackupCountAnnotation] = strconv.Itoa(2) appVeleroBackup.Spec.StorageLocation = "default" @@ -695,17 +684,12 @@ func appendCommonAnnotations(k8sClient kubernetes.Interface, annotations map[str } marshalledAppVersions := string(b) - numBackups := 1 - if hasAppBackup { - numBackups = 2 - } - if annotations == nil { annotations = make(map[string]string, 0) } annotations["kots.io/snapshot-trigger"] = snapshotTrigger annotations["kots.io/snapshot-requested"] = metadata.backupReqestedAt.Format(time.RFC3339) - annotations["kots.io/instance"] = "true" + annotations[types.InstanceBackupAnnotation] = "true" annotations["kots.io/kotsadm-image"] = kotsadmImage annotations["kots.io/kotsadm-deploy-namespace"] = metadata.kotsadmNamespace annotations["kots.io/apps-sequences"] = marshalledAppSequences @@ -716,9 +700,6 @@ func appendCommonAnnotations(k8sClient kubernetes.Interface, annotations map[str annotations["kots.io/embedded-registry"] = embeddedRegistryHost } - // Add improved disaster recovery annotation labels - annotations[InstanceBackupCountAnnotation] = strconv.Itoa(numBackups) - if metadata.ec != nil { annotations = appendECAnnotations(annotations, *metadata.ec) } @@ -893,7 +874,7 @@ func ListInstanceBackups(ctx context.Context, kotsadmNamespace string) ([]*types for _, veleroBackup := range veleroBackups.Items { // TODO: Enforce version? - if veleroBackup.Annotations["kots.io/instance"] != "true" { + if !IsInstanceBackup(veleroBackup) { continue } @@ -1082,22 +1063,6 @@ func getBackupNameFromPrefix(appSlug string) string { return fmt.Sprintf("%s-%s", backupName, randStr) } -func getBackupsFromName(ctx context.Context, veleroClient veleroclientv1.VeleroV1Interface, veleroNamespace string, backupName string) ([]velerov1.Backup, error) { - // try to get the restore from the backup name label - backups, err := veleroClient.Backups(veleroNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s", InstanceBackupNameLabel, velerolabel.GetValidName(backupName)), - }) - if len(backups.Items) > 0 { - return backups.Items, nil - } - backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, backupName, metav1.GetOptions{}) - if err != nil { - return nil, errors.Wrap(err, "failed to get restore") - } - - return []velerov1.Backup{*backup}, nil -} - func DeleteBackup(ctx context.Context, kotsadmNamespace string, backupID string) error { cfg, err := k8sutil.GetClusterConfig() if err != nil { diff --git a/pkg/kotsadmsnapshot/backup_test.go b/pkg/kotsadmsnapshot/backup_test.go index 3ef6597ab6..f289067aa7 100644 --- a/pkg/kotsadmsnapshot/backup_test.go +++ b/pkg/kotsadmsnapshot/backup_test.go @@ -1022,7 +1022,6 @@ func Test_appendCommonAnnotations(t *testing.T) { "kots.io/kotsadm-image": "kotsadm/kotsadm:1.0.0", "kots.io/snapshot-requested": "2024-01-01T00:00:00Z", "kots.io/snapshot-trigger": "manual", - "replicated.com/backup-count": "1", }, }, { @@ -1087,7 +1086,6 @@ func Test_appendCommonAnnotations(t *testing.T) { "kots.io/kotsadm-image": "kotsadm/kotsadm:1.0.0", "kots.io/snapshot-requested": "2024-01-01T00:00:00Z", "kots.io/snapshot-trigger": "schedule", - "replicated.com/backup-count": "2", "kots.io/embedded-cluster": "true", "kots.io/embedded-cluster-id": "embedded-cluster-id", "kots.io/embedded-cluster-version": "embedded-cluster-version", @@ -2090,37 +2088,6 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { args args assert func(t *testing.T, got *velerov1.Backup, err error) }{ - { - name: "should append backup name label", - setup: func(t *testing.T, mockStore *mock_store.MockStore) { - mockStoreExpectApp1(mockStore) - }, - args: args{ - k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), - metadata: instanceBackupMetadata{ - backupName: "app-1-17332487841234", - backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), - kotsadmNamespace: "kotsadm", - backupStorageLocationNamespace: "kotsadm-backups", - apps: map[string]appInstanceBackupMetadata{ - "app-1": { - app: app1, - kotsKinds: kotsKinds, - parentSequence: 1, - }, - }, - isScheduled: true, - ec: nil, - }, - hasAppBackup: false, - }, - assert: func(t *testing.T, got *velerov1.Backup, err error) { - require.NoError(t, err) - if assert.Contains(t, got.Labels, "replicated.com/backup-name") { - assert.Equal(t, "app-1-17332487841234", got.Labels["replicated.com/backup-name"]) - } - }, - }, { name: "KOTSADM_TARGET_NAMESPACE should be added to includedNamespaces", setup: func(t *testing.T, mockStore *mock_store.MockStore) { @@ -2293,9 +2260,6 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) assert.Contains(t, got.Spec.IncludedNamespaces, "include-namespace-1") - if assert.Contains(t, got.Annotations, "replicated.com/backup-type") { - assert.Equal(t, "legacy", got.Annotations["replicated.com/backup-type"]) - } }, }, { @@ -2326,9 +2290,78 @@ func Test_getInfrastructureInstanceBackupSpec(t *testing.T) { assert: func(t *testing.T, got *velerov1.Backup, err error) { require.NoError(t, err) assert.NotContains(t, got.Spec.IncludedNamespaces, "include-namespace-1") + }, + }, + { + name: "should not add improved dr metadata when not using improved dr", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + + mockStoreExpectApp1(mockStore) + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppBackup: false, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + assert.NotContains(t, got.Labels, "replicated.com/backup-name") + assert.NotContains(t, got.Annotations, "replicated.com/backup-type") + assert.NotContains(t, got.Annotations, "replicated.com/backup-count") + }, + }, + { + name: "should add improved dr metadata when not using improved dr", + setup: func(t *testing.T, mockStore *mock_store.MockStore) { + t.Setenv("EMBEDDED_CLUSTER_ID", "embedded-cluster-id") + t.Setenv("EMBEDDED_CLUSTER_VERSION", "embedded-cluster-version") + }, + args: args{ + k8sClient: fake.NewSimpleClientset(kotsadmSts, registryCredsSecret), + metadata: instanceBackupMetadata{ + backupName: "app-1-17332487841234", + backupReqestedAt: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), + kotsadmNamespace: "kotsadm", + backupStorageLocationNamespace: "kotsadm-backups", + apps: map[string]appInstanceBackupMetadata{ + "app-1": { + app: app1, + kotsKinds: kotsKinds, + parentSequence: 1, + }, + }, + isScheduled: true, + ec: ecMeta, + }, + hasAppBackup: true, + }, + assert: func(t *testing.T, got *velerov1.Backup, err error) { + require.NoError(t, err) + if assert.Contains(t, got.Labels, "replicated.com/backup-name") { + assert.Equal(t, "app-1-17332487841234", got.Labels["replicated.com/backup-name"]) + } if assert.Contains(t, got.Annotations, "replicated.com/backup-type") { assert.Equal(t, "infra", got.Annotations["replicated.com/backup-type"]) } + if assert.Contains(t, got.Annotations, "replicated.com/backup-count") { + assert.Equal(t, "2", got.Annotations["replicated.com/backup-count"]) + } }, }, { diff --git a/pkg/kotsadmsnapshot/restore.go b/pkg/kotsadmsnapshot/restore.go index 97c1d20f0c..6fe82be98e 100644 --- a/pkg/kotsadmsnapshot/restore.go +++ b/pkg/kotsadmsnapshot/restore.go @@ -16,14 +16,13 @@ import ( velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" velerolabel "github.com/vmware-tanzu/velero/pkg/label" - "go.uber.org/zap" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/utils/pointer" ) -func GetRestore(ctx context.Context, kotsadmNamespace string, snapshotName string) (*velerov1.Restore, error) { +func GetRestore(ctx context.Context, kotsadmNamespace string, restoreID string) (*velerov1.Restore, error) { cfg, err := k8sutil.GetClusterConfig() if err != nil { return nil, errors.Wrap(err, "failed to get cluster config") @@ -49,7 +48,7 @@ func GetRestore(ctx context.Context, kotsadmNamespace string, snapshotName strin veleroNamespace := bsl.Namespace - restore, err := veleroClient.Restores(veleroNamespace).Get(ctx, snapshotName, metav1.GetOptions{}) + restore, err := veleroClient.Restores(veleroNamespace).Get(ctx, restoreID, metav1.GetOptions{}) if err != nil { if kuberneteserrors.IsNotFound(err) { return nil, nil @@ -60,11 +59,10 @@ func GetRestore(ctx context.Context, kotsadmNamespace string, snapshotName strin return restore, nil } -func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snapshotName string, appSlug string) error { +func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, backupID string, appSlug string) error { // Reference https://github.com/vmware-tanzu/velero/blob/42b612645863c2b3e451b447f9bf798295dd7dba/pkg/cmd/cli/restore/create.go#L222 - logger.Debug("creating restore", - zap.String("snapshotName", snapshotName)) + logger.Debugf("Creating restore for backup %s", backupID) cfg, err := k8sutil.GetClusterConfig() if err != nil { @@ -92,7 +90,7 @@ func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snap veleroNamespace := bsl.Namespace // get the backup - backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, snapshotName, metav1.GetOptions{}) + backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, backupID, metav1.GetOptions{}) if err != nil { return errors.Wrap(err, "failed to find backup") } @@ -100,20 +98,24 @@ func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snap restore := &velerov1.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, - Name: snapshotName, // restore name same as snapshot name + Name: backupID, // restore name same as snapshot name }, Spec: velerov1.RestoreSpec{ - BackupName: snapshotName, + BackupName: backupID, RestorePVs: pointer.Bool(true), IncludeClusterResources: pointer.Bool(true), }, } - if backup.Annotations["kots.io/instance"] == "true" { + if IsInstanceBackup(*backup) { + if GetInstanceBackupType(*backup) != types.InstanceBackupTypeLegacy { + return errors.New("only legacy type instance backups are restorable") + } + // only restore app-specific objects - restore.ObjectMeta.Name = fmt.Sprintf("%s.%s", snapshotName, appSlug) + restore.ObjectMeta.Name = fmt.Sprintf("%s.%s", backupID, appSlug) restore.ObjectMeta.Annotations = map[string]string{ - "kots.io/instance": "true", + types.InstanceBackupAnnotation: "true", } restore.Spec.LabelSelector = &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -130,7 +132,7 @@ func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snap return nil } -func DeleteRestore(ctx context.Context, kotsadmNamespace string, snapshotName string) error { +func DeleteRestore(ctx context.Context, kotsadmNamespace string, restoreID string) error { cfg, err := k8sutil.GetClusterConfig() if err != nil { return errors.Wrap(err, "failed to get cluster config") @@ -153,15 +155,15 @@ func DeleteRestore(ctx context.Context, kotsadmNamespace string, snapshotName st veleroNamespace := bsl.Namespace - err = veleroClient.Restores(veleroNamespace).Delete(ctx, snapshotName, metav1.DeleteOptions{}) + err = veleroClient.Restores(veleroNamespace).Delete(ctx, restoreID, metav1.DeleteOptions{}) if err != nil && !strings.Contains(err.Error(), "not found") { - return errors.Wrapf(err, "failed to delete restore %s", snapshotName) + return errors.Wrapf(err, "failed to delete restore %s", restoreID) } return nil } -func GetRestoreDetails(ctx context.Context, kotsadmNamespace string, restoreName string) (*types.RestoreDetail, error) { +func GetRestoreDetails(ctx context.Context, kotsadmNamespace string, restoreID string) (*types.RestoreDetail, error) { cfg, err := k8sutil.GetClusterConfig() if err != nil { return nil, errors.Wrap(err, "failed to get cluster config") @@ -187,7 +189,7 @@ func GetRestoreDetails(ctx context.Context, kotsadmNamespace string, restoreName veleroNamespace := backendStorageLocation.Namespace - restore, err := veleroClient.Restores(veleroNamespace).Get(ctx, restoreName, metav1.GetOptions{}) + restore, err := veleroClient.Restores(veleroNamespace).Get(ctx, restoreID, metav1.GetOptions{}) if err != nil { return nil, errors.Wrap(err, "failed to get restore") } diff --git a/pkg/kotsadmsnapshot/types/types.go b/pkg/kotsadmsnapshot/types/types.go index d23cc379d6..7e67ed15ae 100644 --- a/pkg/kotsadmsnapshot/types/types.go +++ b/pkg/kotsadmsnapshot/types/types.go @@ -6,6 +6,29 @@ import ( velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) +const ( + // InstanceBackupNameLabel is the label used to store the name of the backup for an instance + // backup. + InstanceBackupNameLabel = "replicated.com/backup-name" + // InstanceBackupTypeAnnotation is the annotation used to store the type of backup for an + // instance backup. + InstanceBackupTypeAnnotation = "replicated.com/backup-type" + // InstanceBackupCountAnnotation is the annotation used to store the expected number of backups + // for an instance backup. + InstanceBackupCountAnnotation = "replicated.com/backup-count" + + // InstanceBackupTypeInfra indicates that the backup is of type infrastructure. + InstanceBackupTypeInfra = "infra" + // InstanceBackupTypeApp indicates that the backup is of type application. + InstanceBackupTypeApp = "app" + // InstanceBackupTypeLegacy indicates that the backup is of type legacy (infra + app). + InstanceBackupTypeLegacy = "legacy" + + // InstanceBackupAnnotation is the annotation used to indicate that a backup is an instance + // backup. + InstanceBackupAnnotation = "kots.io/instance" +) + type App struct { Slug string `json:"slug"` Sequence int64 `json:"sequence"` diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 2e750b5ef8..0d414d3321 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -603,7 +603,7 @@ func (o *Operator) handleUndeployCompleted(a *apptypes.App) error { if err != nil { return errors.Wrap(err, "failed to get backup") } - if backup.Annotations["kots.io/instance"] == "true" { + if snapshot.IsInstanceBackup(*backup) { restoreName = fmt.Sprintf("%s.%s", snapshotName, a.Slug) } @@ -643,7 +643,7 @@ func (o *Operator) checkRestoreComplete(a *apptypes.App, restore *velerov1.Resto } var sequence int64 = 0 - if backupAnnotations["kots.io/instance"] == "true" { + if snapshot.IsInstanceBackup(*backup) { b, ok := backupAnnotations["kots.io/apps-sequences"] if !ok || b == "" { return errors.New("instance backup is missing apps sequences annotation") diff --git a/pkg/snapshot/backup.go b/pkg/snapshot/backup.go index 0ef92bfa41..26f47e91b5 100644 --- a/pkg/snapshot/backup.go +++ b/pkg/snapshot/backup.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "os" "time" @@ -12,6 +12,7 @@ import ( "github.com/pkg/errors" "github.com/replicatedhq/kots/pkg/auth" "github.com/replicatedhq/kots/pkg/k8sutil" + snapshottypes "github.com/replicatedhq/kots/pkg/kotsadmsnapshot/types" "github.com/replicatedhq/kots/pkg/logger" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" @@ -104,7 +105,7 @@ func CreateInstanceBackup(ctx context.Context, options CreateInstanceBackupOptio } defer resp.Body.Close() - respBody, err := ioutil.ReadAll(resp.Body) + respBody, err := io.ReadAll(resp.Body) if err != nil { log.FinishSpinnerWithError() return nil, errors.Wrap(err, "failed to read server response") @@ -174,7 +175,7 @@ func ListInstanceBackups(ctx context.Context, options ListInstanceBackupsOptions backups := []velerov1.Backup{} for _, backup := range b { - if backup.Annotations["kots.io/instance"] != "true" { + if backup.Annotations[snapshottypes.InstanceBackupAnnotation] != "true" { continue } diff --git a/pkg/snapshot/restore.go b/pkg/snapshot/restore.go index af3c438f3b..bb61fed4b9 100644 --- a/pkg/snapshot/restore.go +++ b/pkg/snapshot/restore.go @@ -72,10 +72,14 @@ func RestoreInstanceBackup(ctx context.Context, options RestoreInstanceBackupOpt } // make sure this is an instance backup - if backup.Annotations["kots.io/instance"] != "true" { + if backup.Annotations[snapshottypes.InstanceBackupAnnotation] != "true" { return errors.Wrap(err, "backup provided is not an instance backup") } + if getInstanceBackupType(*backup) != snapshottypes.InstanceBackupTypeLegacy { + return errors.New("only legacy type instance backups are restorable") + } + kotsadmImage, ok := backup.Annotations["kots.io/kotsadm-image"] if !ok { return errors.Wrap(err, "failed to find kotsadm image annotation") @@ -123,15 +127,15 @@ func RestoreInstanceBackup(ctx context.Context, options RestoreInstanceBackupOpt restore := &velerov1.Restore{ ObjectMeta: metav1.ObjectMeta{ Namespace: veleroNamespace, - Name: fmt.Sprintf("%s.kotsadm", options.BackupName), + Name: fmt.Sprintf("%s.kotsadm", backup.Name), Annotations: map[string]string{ - "kots.io/instance": "true", - "kots.io/kotsadm-image": kotsadmImage, - "kots.io/kotsadm-deploy-namespace": kotsadmNamespace, + snapshottypes.InstanceBackupAnnotation: "true", + "kots.io/kotsadm-image": kotsadmImage, + "kots.io/kotsadm-deploy-namespace": kotsadmNamespace, }, }, Spec: velerov1.RestoreSpec{ - BackupName: options.BackupName, + BackupName: backup.Name, LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ kotsadmtypes.KotsadmKey: kotsadmtypes.KotsadmLabelValue, // restoring applications is in a separate step after kotsadm spins up @@ -200,7 +204,7 @@ func RestoreInstanceBackup(ctx context.Context, options RestoreInstanceBackupOpt } // initiate kotsadm applications restore - err = initiateKotsadmApplicationsRestore(options.BackupName, kotsadmNamespace, kotsadmPodName, log) + err = initiateKotsadmApplicationsRestore(backup.Name, kotsadmNamespace, kotsadmPodName, log) if err != nil { log.FinishSpinnerWithError() return errors.Wrap(err, "failed to restore kotsadm applications") @@ -208,7 +212,7 @@ func RestoreInstanceBackup(ctx context.Context, options RestoreInstanceBackupOpt if options.WaitForApps { // wait for applications restore to finish - err = waitForKotsadmApplicationsRestore(options.BackupName, kotsadmNamespace, kotsadmPodName, log) + err = waitForKotsadmApplicationsRestore(backup.Name, kotsadmNamespace, kotsadmPodName, log) if err != nil { if _, ok := errors.Cause(err).(*kotsadmtypes.ErrorAppsRestore); ok { log.FinishSpinnerWithError() @@ -282,7 +286,7 @@ func ListInstanceRestores(ctx context.Context, options ListInstanceRestoresOptio restores := []velerov1.Restore{} for _, restore := range r.Items { - if restore.Annotations["kots.io/instance"] != "true" { + if restore.Annotations[snapshottypes.InstanceBackupAnnotation] != "true" { continue } @@ -328,7 +332,7 @@ func waitForVeleroRestoreCompleted(ctx context.Context, veleroNamespace string, } } -func initiateKotsadmApplicationsRestore(backupName string, kotsadmNamespace string, kotsadmPodName string, log *logger.CLILogger) error { +func initiateKotsadmApplicationsRestore(backupID string, kotsadmNamespace string, kotsadmPodName string, log *logger.CLILogger) error { getPodName := func() (string, error) { return kotsadmPodName, nil } @@ -361,7 +365,7 @@ func initiateKotsadmApplicationsRestore(backupName string, kotsadmNamespace stri return errors.Wrap(err, "failed to get kotsadm auth slug") } - url := fmt.Sprintf("http://localhost:%d/api/v1/snapshot/%s/restore-apps", localPort, backupName) + url := fmt.Sprintf("http://localhost:%d/api/v1/snapshot/%s/restore-apps", localPort, backupID) requestPayload := map[string]interface{}{ "restoreAll": true, @@ -390,7 +394,7 @@ func initiateKotsadmApplicationsRestore(backupName string, kotsadmNamespace stri return nil } -func waitForKotsadmApplicationsRestore(backupName string, kotsadmNamespace string, kotsadmPodName string, log *logger.CLILogger) error { +func waitForKotsadmApplicationsRestore(backupID string, kotsadmNamespace string, kotsadmPodName string, log *logger.CLILogger) error { getPodName := func() (string, error) { return kotsadmPodName, nil } @@ -423,7 +427,7 @@ func waitForKotsadmApplicationsRestore(backupName string, kotsadmNamespace strin return errors.Wrap(err, "failed to get kotsadm auth slug") } - url := fmt.Sprintf("http://localhost:%d/api/v1/snapshot/%s/apps-restore-status", localPort, backupName) + url := fmt.Sprintf("http://localhost:%d/api/v1/snapshot/%s/apps-restore-status", localPort, backupID) for { requestPayload := map[string]interface{}{ @@ -501,3 +505,11 @@ func waitForKotsadmApplicationsRestore(backupName string, kotsadmNamespace strin time.Sleep(time.Second * 2) } } + +// getInstanceBackupType returns the type of the backup from the velero backup object annotation. +func getInstanceBackupType(veleroBackup velerov1.Backup) string { + if val, ok := veleroBackup.GetAnnotations()[snapshottypes.InstanceBackupTypeAnnotation]; ok { + return val + } + return snapshottypes.InstanceBackupTypeLegacy +}