From dca8bf29bcd98b233fc08133cafc10dbbd648716 Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Thu, 16 Nov 2023 21:21:21 +0000 Subject: [PATCH 01/10] refactor FindBackupStoreLocation to accept interfaces --- pkg/handlers/snapshot_logs.go | 29 ++++++- pkg/kotsadm/minio.go | 24 ++++-- pkg/kotsadmsnapshot/backup.go | 137 +++++++++++++++++++-------------- pkg/kotsadmsnapshot/restore.go | 81 +++++++++++-------- pkg/snapshot/filesystem_lvp.go | 27 +++++-- pkg/snapshot/store.go | 40 +++++----- 6 files changed, 217 insertions(+), 121 deletions(-) diff --git a/pkg/handlers/snapshot_logs.go b/pkg/handlers/snapshot_logs.go index 91c2b1f7e7..512688ed4c 100644 --- a/pkg/handlers/snapshot_logs.go +++ b/pkg/handlers/snapshot_logs.go @@ -6,17 +6,44 @@ import ( "github.com/gorilla/mux" "github.com/pkg/errors" + "github.com/replicatedhq/kots/pkg/k8sutil" snapshot "github.com/replicatedhq/kots/pkg/kotsadmsnapshot" "github.com/replicatedhq/kots/pkg/logger" kotssnapshot "github.com/replicatedhq/kots/pkg/snapshot" "github.com/replicatedhq/kots/pkg/util" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" + "k8s.io/client-go/kubernetes" ) func (h *Handler) DownloadSnapshotLogs(w http.ResponseWriter, r *http.Request) { backupName := mux.Vars(r)["backup"] - bsl, err := kotssnapshot.FindBackupStoreLocation(r.Context(), util.PodNamespace) + cfg, err := k8sutil.GetClusterConfig() + if err != nil { + err = errors.Wrap(err, "failed to get cluster config") + logger.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + err = errors.Wrap(err, "failed to create clientset") + logger.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + err = errors.Wrap(err, "failed to create velero clientset") + logger.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + bsl, err := kotssnapshot.FindBackupStoreLocation(r.Context(), clientset, veleroClient, util.PodNamespace) if err != nil { err = errors.Wrap(err, "failed to find backup store location") logger.Error(err) diff --git a/pkg/kotsadm/minio.go b/pkg/kotsadm/minio.go index f342825c9d..7385b989e9 100644 --- a/pkg/kotsadm/minio.go +++ b/pkg/kotsadm/minio.go @@ -14,6 +14,7 @@ import ( "github.com/replicatedhq/kots/pkg/logger" "github.com/replicatedhq/kots/pkg/snapshot" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" corev1 "k8s.io/api/core/v1" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -180,7 +181,22 @@ func MigrateExistingMinioFilesystemDeployments(log *logger.CLILogger, deployOpti } veleroNamespace := veleroStatus.Namespace - bsl, err := snapshot.FindBackupStoreLocation(context.TODO(), deployOptions.Namespace) + cfg, err := k8sutil.GetClusterConfig() + if err != nil { + return errors.Wrap(err, "failed to get cluster config") + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return errors.Wrap(err, "failed to create clientset") + } + + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return errors.Wrap(err, "failed to create velero clientset") + } + + bsl, err := snapshot.FindBackupStoreLocation(context.TODO(), clientset, veleroClient, deployOptions.Namespace) if err != nil { return errors.Wrap(err, "failed to find backupstoragelocations") } @@ -226,10 +242,6 @@ func MigrateExistingMinioFilesystemDeployments(log *logger.CLILogger, deployOpti } // Add the config map to configure the new plugin - clientset, err := k8sutil.GetClientset() - if err != nil { - return errors.Wrap(err, "failed to get k8s clientset") - } fsDeployOptions := &snapshot.FileSystemDeployOptions{ Namespace: deployOptions.Namespace, IsOpenShift: k8sutil.IsOpenShift(clientset), @@ -250,7 +262,7 @@ func MigrateExistingMinioFilesystemDeployments(log *logger.CLILogger, deployOpti success := false defer func() { if !success { - err := snapshot.RevertToMinioFS(context.TODO(), deployOptions.Namespace, veleroNamespace, previousBsl) + err := snapshot.RevertToMinioFS(context.TODO(), clientset, veleroClient, deployOptions.Namespace, veleroNamespace, previousBsl) if err != nil { log.Error(errors.Wrap(err, "Could not restore minio backup storage location")) return diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index b5e8b826e4..779661c7c9 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -69,8 +69,23 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b return nil, errors.Wrap(err, "failed to get app version archive") } + cfg, err := k8sutil.GetClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to get cluster config") + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create clientset") + } + + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + kotsadmNamespace := util.PodNamespace - kotsadmVeleroBackendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + kotsadmVeleroBackendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } @@ -160,26 +175,11 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b } } - clientset, err := k8sutil.GetClientset() - if err != nil { - return nil, errors.Wrap(err, "failed to create k8s clientset") - } - err = excludeShutdownPodsFromBackup(ctx, clientset, veleroBackup) if err != nil { logger.Error(errors.Wrap(err, "failed to exclude shutdown pods from backup")) } - cfg, err := k8sutil.GetClusterConfig() - if err != nil { - return nil, errors.Wrap(err, "failed to get cluster config") - } - - veleroClient, err := veleroclientv1.NewForConfig(cfg) - if err != nil { - return nil, errors.Wrap(err, "failed to create clientset") - } - backup, err := veleroClient.Backups(kotsadmVeleroBackendStorageLocation.Namespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) if err != nil { return nil, errors.Wrap(err, "failed to create velero backup") @@ -191,9 +191,14 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstream, isScheduled bool) (*velerov1.Backup, error) { logger.Debug("creating instance backup") - clientset, err := k8sutil.GetClientset() + cfg, err := k8sutil.GetClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to get cluster config") + } + + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { - return nil, errors.Wrap(err, "failed to create k8s clientset") + return nil, errors.Wrap(err, "failed to create clientset") } isKurl, err := kurl.IsKurl(clientset) @@ -310,7 +315,12 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre backupHooks.Resources = append(backupHooks.Resources, veleroBackup.Spec.Hooks.Resources...) } - kotsadmVeleroBackendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + kotsadmVeleroBackendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } @@ -398,16 +408,6 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre logger.Error(errors.Wrap(err, "failed to exclude shutdown pods from backup")) } - cfg, err := k8sutil.GetClusterConfig() - if err != nil { - return nil, errors.Wrap(err, "failed to get cluster config") - } - - veleroClient, err := veleroclientv1.NewForConfig(cfg) - if err != nil { - return nil, errors.Wrap(err, "failed to create velero clientset") - } - backup, err := veleroClient.Backups(kotsadmVeleroBackendStorageLocation.Namespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) if err != nil { return nil, errors.Wrap(err, "failed to create velero backup") @@ -422,12 +422,17 @@ func ListBackupsForApp(ctx context.Context, kotsadmNamespace string, appID strin return nil, errors.Wrap(err, "failed to get cluster config") } - veleroClient, err := veleroclientv1.NewForConfig(cfg) + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { return nil, errors.Wrap(err, "failed to create clientset") } - backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } @@ -550,12 +555,17 @@ func ListInstanceBackups(ctx context.Context, kotsadmNamespace string) ([]*types return nil, errors.Wrap(err, "failed to get cluster config") } - veleroClient, err := veleroclientv1.NewForConfig(cfg) + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { return nil, errors.Wrap(err, "failed to create clientset") } - backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } @@ -717,7 +727,22 @@ func getSnapshotVolumeSummary(ctx context.Context, veleroBackup *velerov1.Backup } func GetBackup(ctx context.Context, kotsadmNamespace string, snapshotName string) (*velerov1.Backup, error) { - bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + cfg, err := k8sutil.GetClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to get cluster config") + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create clientset") + } + + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to get velero namespace") } @@ -728,26 +753,31 @@ func GetBackup(ctx context.Context, kotsadmNamespace string, snapshotName string veleroNamespace := bsl.Namespace // get the backup - cfg, err := k8sutil.GetClusterConfig() + backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, snapshotName, metav1.GetOptions{}) if err != nil { - return nil, errors.Wrap(err, "failed to get cluster config") + return nil, errors.Wrap(err, "failed to get backup") } - veleroClient, err := veleroclientv1.NewForConfig(cfg) + return backup, nil +} + +func DeleteBackup(ctx context.Context, kotsadmNamespace string, snapshotName string) error { + cfg, err := k8sutil.GetClusterConfig() if err != nil { - return nil, errors.Wrap(err, "failed to create clientset") + return errors.Wrap(err, "failed to get cluster config") } - backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, snapshotName, metav1.GetOptions{}) + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { - return nil, errors.Wrap(err, "failed to get backup") + return errors.Wrap(err, "failed to create clientset") } - return backup, nil -} + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return errors.Wrap(err, "failed to create velero clientset") + } -func DeleteBackup(ctx context.Context, kotsadmNamespace string, snapshotName string) error { - bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return errors.Wrap(err, "failed to get velero namespace") } @@ -766,16 +796,6 @@ func DeleteBackup(ctx context.Context, kotsadmNamespace string, snapshotName str }, } - cfg, err := k8sutil.GetClusterConfig() - if err != nil { - return errors.Wrap(err, "failed to get cluster config") - } - - veleroClient, err := veleroclientv1.NewForConfig(cfg) - if err != nil { - return errors.Wrap(err, "failed to create clientset") - } - _, err = veleroClient.DeleteBackupRequests(veleroNamespace).Create(context.TODO(), veleroDeleteBackupRequest, metav1.CreateOptions{}) if err != nil { return errors.Wrap(err, "failed to create delete backup request") @@ -820,12 +840,17 @@ func GetBackupDetail(ctx context.Context, kotsadmNamespace string, backupName st return nil, errors.Wrap(err, "failed to get cluster config") } - veleroClient, err := veleroclientv1.NewForConfig(cfg) + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { return nil, errors.Wrap(err, "failed to create clientset") } - backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } diff --git a/pkg/kotsadmsnapshot/restore.go b/pkg/kotsadmsnapshot/restore.go index e924cf06e2..97c1d20f0c 100644 --- a/pkg/kotsadmsnapshot/restore.go +++ b/pkg/kotsadmsnapshot/restore.go @@ -19,29 +19,35 @@ import ( "go.uber.org/zap" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "k8s.io/utils/pointer" ) func GetRestore(ctx context.Context, kotsadmNamespace string, snapshotName string) (*velerov1.Restore, error) { - bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + cfg, err := k8sutil.GetClusterConfig() if err != nil { - return nil, errors.Wrap(err, "failed to get velero namespace") - } - if bsl == nil { - return nil, errors.New("no backup store location found") + return nil, errors.Wrap(err, "failed to get cluster config") } - veleroNamespace := bsl.Namespace - - cfg, err := k8sutil.GetClusterConfig() + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { - return nil, errors.Wrap(err, "failed to get cluster config") + return nil, errors.Wrap(err, "failed to create clientset") } veleroClient, err := veleroclientv1.NewForConfig(cfg) if err != nil { - return nil, errors.Wrap(err, "failed to create clientset") + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) + if err != nil { + return nil, errors.Wrap(err, "failed to get velero namespace") } + if bsl == nil { + return nil, errors.New("no backup store location found") + } + + veleroNamespace := bsl.Namespace restore, err := veleroClient.Restores(veleroNamespace).Get(ctx, snapshotName, metav1.GetOptions{}) if err != nil { @@ -60,27 +66,32 @@ func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snap logger.Debug("creating restore", zap.String("snapshotName", snapshotName)) - bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + cfg, err := k8sutil.GetClusterConfig() if err != nil { - return errors.Wrap(err, "failed to get velero namespace") - } - if bsl == nil { - return errors.New("no backup store location found") + return errors.Wrap(err, "failed to get cluster config") } - veleroNamespace := bsl.Namespace - - // get the backup - cfg, err := k8sutil.GetClusterConfig() + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { - return errors.Wrap(err, "failed to get cluster config") + return errors.Wrap(err, "failed to create clientset") } veleroClient, err := veleroclientv1.NewForConfig(cfg) if err != nil { - return errors.Wrap(err, "failed to create clientset") + return errors.Wrap(err, "failed to create velero clientset") + } + + bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) + if err != nil { + return errors.Wrap(err, "failed to get velero namespace") + } + if bsl == nil { + return errors.New("no backup store location found") } + veleroNamespace := bsl.Namespace + + // get the backup backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, snapshotName, metav1.GetOptions{}) if err != nil { return errors.Wrap(err, "failed to find backup") @@ -120,23 +131,28 @@ func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snap } func DeleteRestore(ctx context.Context, kotsadmNamespace string, snapshotName string) error { - bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + cfg, err := k8sutil.GetClusterConfig() if err != nil { - return errors.Wrap(err, "failed to get velero namespace") + return errors.Wrap(err, "failed to get cluster config") } - veleroNamespace := bsl.Namespace - - cfg, err := k8sutil.GetClusterConfig() + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { - return errors.Wrap(err, "failed to get cluster config") + return errors.Wrap(err, "failed to create clientset") } veleroClient, err := veleroclientv1.NewForConfig(cfg) if err != nil { - return errors.Wrap(err, "failed to create clientset") + return errors.Wrap(err, "failed to create velero clientset") + } + + bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) + if err != nil { + return errors.Wrap(err, "failed to get velero namespace") } + veleroNamespace := bsl.Namespace + err = veleroClient.Restores(veleroNamespace).Delete(ctx, snapshotName, metav1.DeleteOptions{}) if err != nil && !strings.Contains(err.Error(), "not found") { return errors.Wrapf(err, "failed to delete restore %s", snapshotName) @@ -151,12 +167,17 @@ func GetRestoreDetails(ctx context.Context, kotsadmNamespace string, restoreName return nil, errors.Wrap(err, "failed to get cluster config") } - veleroClient, err := veleroclientv1.NewForConfig(cfg) + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { return nil, errors.Wrap(err, "failed to create clientset") } - backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } diff --git a/pkg/snapshot/filesystem_lvp.go b/pkg/snapshot/filesystem_lvp.go index 7210a0436e..55ff0e179e 100644 --- a/pkg/snapshot/filesystem_lvp.go +++ b/pkg/snapshot/filesystem_lvp.go @@ -11,6 +11,7 @@ import ( kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" types "github.com/replicatedhq/kots/pkg/snapshot/types" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" corev1 "k8s.io/api/core/v1" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -45,7 +46,22 @@ func ValidateFileSystemDeployment(ctx context.Context, clientset kubernetes.Inte } func GetCurrentLvpFileSystemConfig(ctx context.Context, namespace string) (*types.FileSystemConfig, error) { - bsl, err := FindBackupStoreLocation(ctx, namespace) + cfg, err := k8sutil.GetClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to get cluster config") + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create clientset") + } + + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + bsl, err := FindBackupStoreLocation(ctx, clientset, veleroClient, namespace) if err != nil { return nil, errors.Wrap(err, "failed to find velero backup storage location") } @@ -70,11 +86,6 @@ func GetCurrentLvpFileSystemConfig(ctx context.Context, namespace string) (*type } // backup storage location does not exist, get file system config from the config map - clientset, err := k8sutil.GetClientset() - if err != nil { - return nil, errors.Wrap(err, "failed to get k8s clientset") - } - fileSystemConfigMap, err := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, FileSystemLVPConfigMapName, metav1.GetOptions{}) if err != nil { if kuberneteserrors.IsNotFound(err) { @@ -103,8 +114,8 @@ func GetCurrentLvpFileSystemConfig(ctx context.Context, namespace string) (*type // RevertToMinioFS will apply the spec of the previous BSL to the current one and then update. // Used for recovery during a failed migration from Minio to LVP. -func RevertToMinioFS(ctx context.Context, kotsadmNamespace, veleroNamespace string, previousBsl *velerov1api.BackupStorageLocation) error { - bsl, err := FindBackupStoreLocation(context.TODO(), kotsadmNamespace) +func RevertToMinioFS(ctx context.Context, clientset kubernetes.Interface, veleroClient veleroclientv1.VeleroV1Interface, kotsadmNamespace, veleroNamespace string, previousBsl *velerov1api.BackupStorageLocation) error { + bsl, err := FindBackupStoreLocation(context.TODO(), clientset, veleroClient, kotsadmNamespace) if err != nil { return errors.Wrap(err, "failed to find backupstoragelocations") } diff --git a/pkg/snapshot/store.go b/pkg/snapshot/store.go index 11d090ee0f..17ddf21e84 100644 --- a/pkg/snapshot/store.go +++ b/pkg/snapshot/store.go @@ -427,7 +427,12 @@ func upsertGlobalStore(ctx context.Context, store *types.Store, kotsadmNamespace return nil, errors.Wrap(err, "failed to create clientset") } - bsl, err := FindBackupStoreLocation(ctx, kotsadmNamespace) + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + bsl, err := FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } @@ -867,13 +872,23 @@ func updateLvpFileSystemStore(ctx context.Context, store *types.Store, bsl *vele // GetGlobalStore will return the global store from the current backup storage location // or will find it, if the param is nil func GetGlobalStore(ctx context.Context, kotsadmNamespace string, bsl *velerov1.BackupStorageLocation) (*types.Store, error) { - clientset, err := k8sutil.GetClientset() + cfg, err := k8sutil.GetClusterConfig() if err != nil { - return nil, errors.Wrap(err, "failed to get k8s clientset") + return nil, errors.Wrap(err, "failed to get cluster config") + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create clientset") + } + + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") } if bsl == nil { - bsl, err = FindBackupStoreLocation(ctx, kotsadmNamespace) + bsl, err = FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } @@ -1055,17 +1070,7 @@ func mapAWSBackupStorageLocationToStore(kotsadmVeleroBackendStorageLocation *vel // FindBackupStoreLocation will find the backup storage location used by velero // kotsadmNamespace is only required in minimal rbac installations. if empty, cluster scope privileges will be needed to detect and validate velero -func FindBackupStoreLocation(ctx context.Context, kotsadmNamespace string) (*velerov1.BackupStorageLocation, error) { - cfg, err := k8sutil.GetClusterConfig() - if err != nil { - return nil, errors.Wrap(err, "failed to get cluster config") - } - - clientset, err := kubernetes.NewForConfig(cfg) - if err != nil { - return nil, errors.Wrap(err, "failed to create clientset") - } - +func FindBackupStoreLocation(ctx context.Context, clientset kubernetes.Interface, veleroClient veleroclientv1.VeleroV1Interface, kotsadmNamespace string) (*velerov1.BackupStorageLocation, error) { veleroNamespace, err := DetectVeleroNamespace(ctx, clientset, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to detect velero namespace") @@ -1075,11 +1080,6 @@ func FindBackupStoreLocation(ctx context.Context, kotsadmNamespace string) (*vel return nil, nil } - veleroClient, err := veleroclientv1.NewForConfig(cfg) - if err != nil { - return nil, errors.Wrap(err, "failed to create velero clientset") - } - backupStorageLocations, err := veleroClient.BackupStorageLocations(veleroNamespace).List(ctx, metav1.ListOptions{}) if err != nil { if kuberneteserrors.IsNotFound(err) { From 7d23c76d8acc5283092c8bee4eb33c44751a82fa Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Thu, 16 Nov 2023 21:35:32 +0000 Subject: [PATCH 02/10] snapshot provider, schedule, and retention policy --- pkg/api/reporting/types/types.go | 37 +++++++++++++-------- pkg/reporting/app.go | 56 ++++++++++++++++++++++++++++++-- pkg/reporting/app_airgap.go | 3 ++ pkg/reporting/instance_report.go | 3 ++ pkg/reporting/util.go | 4 +++ 5 files changed, 87 insertions(+), 16 deletions(-) diff --git a/pkg/api/reporting/types/types.go b/pkg/api/reporting/types/types.go index ec8ac91333..966481bebe 100644 --- a/pkg/api/reporting/types/types.go +++ b/pkg/api/reporting/types/types.go @@ -2,20 +2,23 @@ package types // This type is mimicked in the instance_report table. type ReportingInfo struct { - InstanceID string `json:"instance_id"` - ClusterID string `json:"cluster_id"` - Downstream DownstreamInfo `json:"downstream"` - AppStatus string `json:"app_status"` - IsKurl bool `json:"is_kurl"` - KurlNodeCountTotal int `json:"kurl_node_count_total"` - KurlNodeCountReady int `json:"kurl_node_count_ready"` - K8sVersion string `json:"k8s_version"` - K8sDistribution string `json:"k8s_distribution"` - UserAgent string `json:"user_agent"` - KOTSInstallID string `json:"kots_install_id"` - KURLInstallID string `json:"kurl_install_id"` - IsGitOpsEnabled bool `json:"is_gitops_enabled"` - GitOpsProvider string `json:"gitops_provider"` + InstanceID string `json:"instance_id"` + ClusterID string `json:"cluster_id"` + Downstream DownstreamInfo `json:"downstream"` + AppStatus string `json:"app_status"` + IsKurl bool `json:"is_kurl"` + KurlNodeCountTotal int `json:"kurl_node_count_total"` + KurlNodeCountReady int `json:"kurl_node_count_ready"` + K8sVersion string `json:"k8s_version"` + K8sDistribution string `json:"k8s_distribution"` + UserAgent string `json:"user_agent"` + KOTSInstallID string `json:"kots_install_id"` + KURLInstallID string `json:"kurl_install_id"` + IsGitOpsEnabled bool `json:"is_gitops_enabled"` + GitOpsProvider string `json:"gitops_provider"` + SnapshotProvider string `json:"snapshot_provider"` + SnapshotSchedule string `json:"snapshot_schedule"` + SnapshotRetentionPolicy string `json:"snapshot_retention_policy"` } type DownstreamInfo struct { @@ -31,6 +34,12 @@ type DownstreamInfo struct { NativeHelmInstalls int `json:"native_helm_installs"` } +type SnapshotReport struct { + Provider string + Schedule string + RetentionPolicy string +} + // This type is mimicked in the preflight_report table. type PreflightStatus struct { InstanceID string `json:"instance_id"` diff --git a/pkg/reporting/app.go b/pkg/reporting/app.go index 744e8f35ce..0ebb0f3a91 100644 --- a/pkg/reporting/app.go +++ b/pkg/reporting/app.go @@ -19,10 +19,12 @@ import ( "github.com/replicatedhq/kots/pkg/kotsutil" "github.com/replicatedhq/kots/pkg/kurl" "github.com/replicatedhq/kots/pkg/logger" + "github.com/replicatedhq/kots/pkg/snapshot" "github.com/replicatedhq/kots/pkg/store" "github.com/replicatedhq/kots/pkg/util" troubleshootpreflight "github.com/replicatedhq/troubleshoot/pkg/preflight" "github.com/segmentio/ksuid" + veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" helmrelease "helm.sh/helm/v3/pkg/release" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -33,6 +35,12 @@ var ( clusterID string // set when in Helm managed mode ) +type SnapshotReport struct { + Provider string + Schedule string + RetentionPolicy string +} + func Init() error { if util.IsHelmManaged() { err := initFromHelm() @@ -185,9 +193,14 @@ func GetReportingInfo(appID string) *types.ReportingInfo { UserAgent: buildversion.GetUserAgent(), } - clientset, err := k8sutil.GetClientset() + cfg, err := k8sutil.GetClusterConfig() if err != nil { - logger.Debugf(errors.Wrap(err, "failed to get kubernetes clientset").Error()) + logger.Debugf("failed to get cluster config: %v", err.Error()) + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + logger.Debugf("failed to get clientset: %v", err.Error()) } if util.IsHelmManaged() { @@ -248,6 +261,23 @@ func GetReportingInfo(appID string) *types.ReportingInfo { } r.IsGitOpsEnabled, r.GitOpsProvider = getGitOpsReport(clientset, appID, r.ClusterID) + + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + logger.Debugf("failed to get velero client: %v", err.Error()) + } + + if clientset != nil && veleroClient != nil { + report, err := getSnapshotReport(store.GetStore(), clientset, veleroClient, appID) + if err != nil { + logger.Debugf("failed to get snapshot report: %v", err.Error()) + } else { + r.SnapshotProvider = report.Provider + r.SnapshotSchedule = report.Schedule + r.SnapshotRetentionPolicy = report.RetentionPolicy + } + } + return &r } @@ -325,3 +355,25 @@ func getGitOpsReport(clientset kubernetes.Interface, appID string, clusterID str } return false, "" } + +func getSnapshotReport(kotsStore store.Store, clientset kubernetes.Interface, veleroClient veleroclientv1.VeleroV1Interface, appID string) (*SnapshotReport, error) { + report := &SnapshotReport{} + + bsl, err := snapshot.FindBackupStoreLocation(context.TODO(), clientset, veleroClient, util.PodNamespace) + if err != nil { + return nil, errors.Wrap(err, "failed to find backup store location") + } + if bsl == nil { + return nil, errors.New("no backup store location found") + } + report.Provider = bsl.Spec.Provider + + app, err := kotsStore.GetApp(appID) + if err != nil { + return nil, errors.Wrap(err, "failed to get app") + } + report.Schedule = app.SnapshotSchedule + report.RetentionPolicy = app.SnapshotTTL + + return report, nil +} diff --git a/pkg/reporting/app_airgap.go b/pkg/reporting/app_airgap.go index ca94895d9c..32ede797de 100644 --- a/pkg/reporting/app_airgap.go +++ b/pkg/reporting/app_airgap.go @@ -69,6 +69,9 @@ func BuildInstanceReport(licenseID string, reportingInfo *types.ReportingInfo) * KurlInstallID: reportingInfo.KURLInstallID, IsGitOpsEnabled: reportingInfo.IsGitOpsEnabled, GitOpsProvider: reportingInfo.GitOpsProvider, + SnapshotProvider: reportingInfo.SnapshotProvider, + SnapshotSchedule: reportingInfo.SnapshotSchedule, + SnapshotRetentionPolicy: reportingInfo.SnapshotRetentionPolicy, DownstreamChannelID: reportingInfo.Downstream.ChannelID, DownstreamChannelSequence: downstreamSequence, DownstreamChannelName: reportingInfo.Downstream.ChannelName, diff --git a/pkg/reporting/instance_report.go b/pkg/reporting/instance_report.go index 92ef1d23b4..0c234ab465 100644 --- a/pkg/reporting/instance_report.go +++ b/pkg/reporting/instance_report.go @@ -29,6 +29,9 @@ type InstanceReportEvent struct { KurlInstallID string `json:"kurl_install_id,omitempty"` IsGitOpsEnabled bool `json:"is_gitops_enabled"` GitOpsProvider string `json:"gitops_provider"` + SnapshotProvider string `json:"snapshot_provider"` + SnapshotSchedule string `json:"snapshot_schedule"` + SnapshotRetentionPolicy string `json:"snapshot_retention_policy"` DownstreamChannelID string `json:"downstream_channel_id,omitempty"` DownstreamChannelSequence uint64 `json:"downstream_channel_sequence,omitempty"` DownstreamChannelName string `json:"downstream_channel_name,omitempty"` diff --git a/pkg/reporting/util.go b/pkg/reporting/util.go index e9da1ba89f..6cf1eb832e 100644 --- a/pkg/reporting/util.go +++ b/pkg/reporting/util.go @@ -68,6 +68,10 @@ func GetReportingInfoHeaders(reportingInfo *types.ReportingInfo) map[string]stri headers["X-Replicated-IsGitOpsEnabled"] = strconv.FormatBool(reportingInfo.IsGitOpsEnabled) headers["X-Replicated-GitOpsProvider"] = reportingInfo.GitOpsProvider + headers["X-Replicated-SnapshotProvider"] = reportingInfo.SnapshotProvider + headers["X-Replicated-SnapshotSchedule"] = reportingInfo.SnapshotSchedule + headers["X-Replicated-SnapshotRetentionPolicy"] = reportingInfo.SnapshotRetentionPolicy + if reportingInfo.K8sDistribution != "" { headers["X-Replicated-K8sDistribution"] = reportingInfo.K8sDistribution } From 17e0a007db68bdde01575516baac50a120472f83 Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Fri, 17 Nov 2023 17:01:12 +0000 Subject: [PATCH 03/10] full and partial schedule and ttl --- pkg/api/reporting/types/types.go | 6 +++-- pkg/reporting/app.go | 38 +++++++++++++++++++++++++------- pkg/reporting/app_airgap.go | 6 +++-- pkg/reporting/instance_report.go | 6 +++-- pkg/reporting/util.go | 6 +++-- 5 files changed, 46 insertions(+), 16 deletions(-) diff --git a/pkg/api/reporting/types/types.go b/pkg/api/reporting/types/types.go index 966481bebe..8f0b857f2a 100644 --- a/pkg/api/reporting/types/types.go +++ b/pkg/api/reporting/types/types.go @@ -17,8 +17,10 @@ type ReportingInfo struct { IsGitOpsEnabled bool `json:"is_gitops_enabled"` GitOpsProvider string `json:"gitops_provider"` SnapshotProvider string `json:"snapshot_provider"` - SnapshotSchedule string `json:"snapshot_schedule"` - SnapshotRetentionPolicy string `json:"snapshot_retention_policy"` + SnapshotFullSchedule string `json:"snapshot_full_schedule"` + SnapshotFullTTL string `json:"snapshot_full_ttl"` + SnapshotPartialSchedule string `json:"snapshot_partial_schedule"` + SnapshotPartialTTL string `json:"snapshot_partial_ttl"` } type DownstreamInfo struct { diff --git a/pkg/reporting/app.go b/pkg/reporting/app.go index 0ebb0f3a91..bc9dc5c39f 100644 --- a/pkg/reporting/app.go +++ b/pkg/reporting/app.go @@ -11,6 +11,7 @@ import ( "path/filepath" "github.com/pkg/errors" + downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types" "github.com/replicatedhq/kots/pkg/api/reporting/types" "github.com/replicatedhq/kots/pkg/buildversion" "github.com/replicatedhq/kots/pkg/gitops" @@ -37,8 +38,10 @@ var ( type SnapshotReport struct { Provider string - Schedule string - RetentionPolicy string + FullSchedule string + FullTTL string + PartialSchedule string + PartialTTL string } func Init() error { @@ -268,13 +271,15 @@ func GetReportingInfo(appID string) *types.ReportingInfo { } if clientset != nil && veleroClient != nil { - report, err := getSnapshotReport(store.GetStore(), clientset, veleroClient, appID) + report, err := getSnapshotReport(store.GetStore(), clientset, veleroClient, appID, clusterID) if err != nil { logger.Debugf("failed to get snapshot report: %v", err.Error()) } else { r.SnapshotProvider = report.Provider - r.SnapshotSchedule = report.Schedule - r.SnapshotRetentionPolicy = report.RetentionPolicy + r.SnapshotFullSchedule = report.FullSchedule + r.SnapshotFullTTL = report.FullTTL + r.SnapshotPartialSchedule = report.PartialSchedule + r.SnapshotPartialTTL = report.PartialTTL } } @@ -356,7 +361,7 @@ func getGitOpsReport(clientset kubernetes.Interface, appID string, clusterID str return false, "" } -func getSnapshotReport(kotsStore store.Store, clientset kubernetes.Interface, veleroClient veleroclientv1.VeleroV1Interface, appID string) (*SnapshotReport, error) { +func getSnapshotReport(kotsStore store.Store, clientset kubernetes.Interface, veleroClient veleroclientv1.VeleroV1Interface, appID string, clusterID string) (*SnapshotReport, error) { report := &SnapshotReport{} bsl, err := snapshot.FindBackupStoreLocation(context.TODO(), clientset, veleroClient, util.PodNamespace) @@ -368,12 +373,29 @@ func getSnapshotReport(kotsStore store.Store, clientset kubernetes.Interface, ve } report.Provider = bsl.Spec.Provider + clusters, err := kotsStore.ListClusters() + if err != nil { + return nil, errors.Wrap(err, "failed to list clusters") + } + var downstream *downstreamtypes.Downstream + for _, cluster := range clusters { + if cluster.ClusterID == clusterID { + downstream = cluster + break + } + } + if downstream == nil { + return nil, errors.New("no downstream found") + } + report.FullSchedule = downstream.SnapshotSchedule + report.FullTTL = downstream.SnapshotTTL + app, err := kotsStore.GetApp(appID) if err != nil { return nil, errors.Wrap(err, "failed to get app") } - report.Schedule = app.SnapshotSchedule - report.RetentionPolicy = app.SnapshotTTL + report.PartialSchedule = app.SnapshotSchedule + report.PartialTTL = app.SnapshotTTL return report, nil } diff --git a/pkg/reporting/app_airgap.go b/pkg/reporting/app_airgap.go index 32ede797de..d35eda2e6b 100644 --- a/pkg/reporting/app_airgap.go +++ b/pkg/reporting/app_airgap.go @@ -70,8 +70,10 @@ func BuildInstanceReport(licenseID string, reportingInfo *types.ReportingInfo) * IsGitOpsEnabled: reportingInfo.IsGitOpsEnabled, GitOpsProvider: reportingInfo.GitOpsProvider, SnapshotProvider: reportingInfo.SnapshotProvider, - SnapshotSchedule: reportingInfo.SnapshotSchedule, - SnapshotRetentionPolicy: reportingInfo.SnapshotRetentionPolicy, + SnapshotFullSchedule: reportingInfo.SnapshotFullSchedule, + SnapshotFullTTL: reportingInfo.SnapshotFullTTL, + SnapshotPartialSchedule: reportingInfo.SnapshotPartialSchedule, + SnapshotPartialTTL: reportingInfo.SnapshotPartialTTL, DownstreamChannelID: reportingInfo.Downstream.ChannelID, DownstreamChannelSequence: downstreamSequence, DownstreamChannelName: reportingInfo.Downstream.ChannelName, diff --git a/pkg/reporting/instance_report.go b/pkg/reporting/instance_report.go index 0c234ab465..d83853871e 100644 --- a/pkg/reporting/instance_report.go +++ b/pkg/reporting/instance_report.go @@ -30,8 +30,10 @@ type InstanceReportEvent struct { IsGitOpsEnabled bool `json:"is_gitops_enabled"` GitOpsProvider string `json:"gitops_provider"` SnapshotProvider string `json:"snapshot_provider"` - SnapshotSchedule string `json:"snapshot_schedule"` - SnapshotRetentionPolicy string `json:"snapshot_retention_policy"` + SnapshotFullSchedule string `json:"snapshot_full_schedule"` + SnapshotFullTTL string `json:"snapshot_full_ttl"` + SnapshotPartialSchedule string `json:"snapshot_partial_schedule"` + SnapshotPartialTTL string `json:"snapshot_partial_ttl"` DownstreamChannelID string `json:"downstream_channel_id,omitempty"` DownstreamChannelSequence uint64 `json:"downstream_channel_sequence,omitempty"` DownstreamChannelName string `json:"downstream_channel_name,omitempty"` diff --git a/pkg/reporting/util.go b/pkg/reporting/util.go index 6cf1eb832e..b359217cfd 100644 --- a/pkg/reporting/util.go +++ b/pkg/reporting/util.go @@ -69,8 +69,10 @@ func GetReportingInfoHeaders(reportingInfo *types.ReportingInfo) map[string]stri headers["X-Replicated-GitOpsProvider"] = reportingInfo.GitOpsProvider headers["X-Replicated-SnapshotProvider"] = reportingInfo.SnapshotProvider - headers["X-Replicated-SnapshotSchedule"] = reportingInfo.SnapshotSchedule - headers["X-Replicated-SnapshotRetentionPolicy"] = reportingInfo.SnapshotRetentionPolicy + headers["X-Replicated-SnapshotFullSchedule"] = reportingInfo.SnapshotFullSchedule + headers["X-Replicated-SnapshotFullTTL"] = reportingInfo.SnapshotFullTTL + headers["X-Replicated-SnapshotPartialSchedule"] = reportingInfo.SnapshotPartialSchedule + headers["X-Replicated-SnapshotPartialTTL"] = reportingInfo.SnapshotPartialTTL if reportingInfo.K8sDistribution != "" { headers["X-Replicated-K8sDistribution"] = reportingInfo.K8sDistribution From 036c44273b5f29de1861ff97b28ec0770342d5fb Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Mon, 20 Nov 2023 17:06:13 +0000 Subject: [PATCH 04/10] fix cluster id for snapshot reporting --- pkg/reporting/app.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/reporting/app.go b/pkg/reporting/app.go index bc9dc5c39f..5e3fc35049 100644 --- a/pkg/reporting/app.go +++ b/pkg/reporting/app.go @@ -6,6 +6,7 @@ import ( "context" "encoding/base64" "encoding/json" + "fmt" "io/ioutil" "os" "path/filepath" @@ -271,7 +272,7 @@ func GetReportingInfo(appID string) *types.ReportingInfo { } if clientset != nil && veleroClient != nil { - report, err := getSnapshotReport(store.GetStore(), clientset, veleroClient, appID, clusterID) + report, err := getSnapshotReport(store.GetStore(), clientset, veleroClient, appID, r.ClusterID) if err != nil { logger.Debugf("failed to get snapshot report: %v", err.Error()) } else { @@ -385,7 +386,7 @@ func getSnapshotReport(kotsStore store.Store, clientset kubernetes.Interface, ve } } if downstream == nil { - return nil, errors.New("no downstream found") + return nil, fmt.Errorf("cluster %s not found", clusterID) } report.FullSchedule = downstream.SnapshotSchedule report.FullTTL = downstream.SnapshotTTL From 1090e613c549d8c485e348ffc2b2605c925401c9 Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Mon, 20 Nov 2023 21:35:58 +0000 Subject: [PATCH 05/10] remove unecessary clientset instantiation --- pkg/snapshot/velero.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/snapshot/velero.go b/pkg/snapshot/velero.go index 1f0e852952..b92f3bf625 100644 --- a/pkg/snapshot/velero.go +++ b/pkg/snapshot/velero.go @@ -249,11 +249,6 @@ func DetectVeleroNamespace(ctx context.Context, clientset kubernetes.Interface, veleroNamespace = TryGetVeleroNamespaceFromConfigMap(ctx, clientset, kotsadmNamespace) } - clientset, err := k8sutil.GetClientset() - if err != nil { - return "", errors.Wrap(err, "failed to get k8s clientset") - } - deployments, err := clientset.AppsV1().Deployments(veleroNamespace).List(ctx, metav1.ListOptions{}) if kuberneteserrors.IsNotFound(err) { return "", nil From b0fcd6c1399b02e84acd335b994de7b949761e8f Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Mon, 20 Nov 2023 21:36:12 +0000 Subject: [PATCH 06/10] unit tests for snapshot report --- pkg/reporting/app_test.go | 207 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 pkg/reporting/app_test.go diff --git a/pkg/reporting/app_test.go b/pkg/reporting/app_test.go new file mode 100644 index 0000000000..60af007b4a --- /dev/null +++ b/pkg/reporting/app_test.go @@ -0,0 +1,207 @@ +package reporting + +import ( + "errors" + "reflect" + "testing" + + "github.com/golang/mock/gomock" + downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types" + apptypes "github.com/replicatedhq/kots/pkg/app/types" + "github.com/replicatedhq/kots/pkg/store" + mock_store "github.com/replicatedhq/kots/pkg/store/mock" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerofake "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake" + veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" +) + +func Test_getSnapshotReport(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockStore := mock_store.NewMockStore(ctrl) + + t.Setenv("POD_NAMESPACE", "default") + + testVeleroNamespace := "velero" + veleroNamespaceConfigmap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm-velero-namespace", + }, + Data: map[string]string{ + "veleroNamespace": testVeleroNamespace, + }, + } + veleroDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "velero", + Namespace: testVeleroNamespace, + }, + } + + testAppID := "test-app-id" + testClusterID := "test-cluster-id" + + type args struct { + kotsStore store.Store + clientset kubernetes.Interface + veleroClient veleroclientv1.VeleroV1Interface + appID string + clusterID string + } + tests := []struct { + name string + args args + mockStoreExpectations func() + want *SnapshotReport + wantErr bool + }{ + { + name: "happy path with schedule and ttl", + args: args{ + kotsStore: mockStore, + clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), + veleroClient: velerofake.NewSimpleClientset( + backupStorageLocationWithProvider(testVeleroNamespace, "aws"), + ).VeleroV1(), + appID: testAppID, + clusterID: testClusterID, + }, + mockStoreExpectations: func() { + mockStore.EXPECT().ListClusters().Return([]*downstreamtypes.Downstream{ + { + ClusterID: testClusterID, + SnapshotSchedule: "0 0 * * *", + SnapshotTTL: "720h", + }, + }, nil) + mockStore.EXPECT().GetApp(testAppID).Return(&apptypes.App{ + ID: testAppID, + SnapshotSchedule: "0 0 * * MON", + SnapshotTTL: "168h", + }, nil) + }, + want: &SnapshotReport{ + Provider: "aws", + FullSchedule: "0 0 * * *", + FullTTL: "720h", + PartialSchedule: "0 0 * * MON", + PartialTTL: "168h", + }, + }, + { + name: "happy path with default ttl only", + args: args{ + kotsStore: mockStore, + clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), + veleroClient: velerofake.NewSimpleClientset( + backupStorageLocationWithProvider(testVeleroNamespace, "aws"), + ).VeleroV1(), + appID: testAppID, + clusterID: testClusterID, + }, + mockStoreExpectations: func() { + mockStore.EXPECT().ListClusters().Return([]*downstreamtypes.Downstream{ + { + ClusterID: testClusterID, + SnapshotSchedule: "", + SnapshotTTL: "720h", + }, + }, nil) + mockStore.EXPECT().GetApp(testAppID).Return(&apptypes.App{ + ID: testAppID, + SnapshotSchedule: "", + SnapshotTTL: "720h", + }, nil) + }, + want: &SnapshotReport{ + Provider: "aws", + FullSchedule: "", + FullTTL: "720h", + PartialSchedule: "", + PartialTTL: "720h", + }, + }, + { + name: "no backup storage location", + args: args{ + kotsStore: mockStore, + clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), + veleroClient: velerofake.NewSimpleClientset().VeleroV1(), + appID: testAppID, + clusterID: testClusterID, + }, + mockStoreExpectations: func() {}, + wantErr: true, + }, + { + name: "failed to list clusters", + args: args{ + kotsStore: mockStore, + clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), + veleroClient: velerofake.NewSimpleClientset( + backupStorageLocationWithProvider(testVeleroNamespace, "aws"), + ).VeleroV1(), + appID: testAppID, + clusterID: testClusterID, + }, + mockStoreExpectations: func() { + mockStore.EXPECT().ListClusters().Return(nil, errors.New("failed to list clusters")) + }, + wantErr: true, + }, + { + name: "failed to get app", + args: args{ + kotsStore: mockStore, + clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), + veleroClient: velerofake.NewSimpleClientset( + backupStorageLocationWithProvider(testVeleroNamespace, "aws"), + ).VeleroV1(), + appID: testAppID, + clusterID: testClusterID, + }, + mockStoreExpectations: func() { + mockStore.EXPECT().ListClusters().Return([]*downstreamtypes.Downstream{ + { + ClusterID: testClusterID, + SnapshotSchedule: "0 0 * * *", + SnapshotTTL: "720h", + }, + }, nil) + mockStore.EXPECT().GetApp(testAppID).Return(nil, errors.New("failed to get app")) + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.mockStoreExpectations() + got, err := getSnapshotReport(tt.args.kotsStore, tt.args.clientset, tt.args.veleroClient, tt.args.appID, tt.args.clusterID) + if (err != nil) != tt.wantErr { + t.Errorf("getSnapshotReport() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getSnapshotReport() = %v, want %v", got, tt.want) + } + }) + } +} + +func backupStorageLocationWithProvider(namespace string, provider string) *velerov1.BackupStorageLocation { + return &velerov1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: namespace, + }, + Spec: velerov1.BackupStorageLocationSpec{ + Provider: provider, + Default: true, + }, + } +} From 5b3b32e1cb305f99c36677541fdaba2d53fdc7f5 Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Mon, 20 Nov 2023 21:39:00 +0000 Subject: [PATCH 07/10] update airgap report test --- pkg/reporting/report_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/reporting/report_test.go b/pkg/reporting/report_test.go index af367d26b1..ec39743893 100644 --- a/pkg/reporting/report_test.go +++ b/pkg/reporting/report_test.go @@ -353,6 +353,11 @@ func createTestInstanceEvent(reportedAt int64) InstanceReportEvent { KurlInstallID: "test-kurl-install-id", IsGitOpsEnabled: true, GitOpsProvider: "test-gitops-provider", + SnapshotProvider: "test-snapshot-provider", + SnapshotFullSchedule: "0 0 * * *", + SnapshotFullTTL: "720h", + SnapshotPartialSchedule: "0 0 * * *", + SnapshotPartialTTL: "720h", DownstreamChannelID: "test-downstream-channel-id", DownstreamChannelSequence: 123, DownstreamChannelName: "test-downstream-channel-name", From 823c13a4a433f2ba7acfc67bcd5db27426ee12e5 Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Tue, 21 Nov 2023 21:09:14 +0000 Subject: [PATCH 08/10] address pr feedback --- pkg/reporting/app.go | 26 +++++------ pkg/reporting/app_test.go | 75 ++++++++------------------------ pkg/snapshot/store_test.go | 88 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 68 deletions(-) diff --git a/pkg/reporting/app.go b/pkg/reporting/app.go index 5e3fc35049..96d4b3cacc 100644 --- a/pkg/reporting/app.go +++ b/pkg/reporting/app.go @@ -26,6 +26,7 @@ import ( "github.com/replicatedhq/kots/pkg/util" troubleshootpreflight "github.com/replicatedhq/troubleshoot/pkg/preflight" "github.com/segmentio/ksuid" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" helmrelease "helm.sh/helm/v3/pkg/release" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -272,15 +273,20 @@ func GetReportingInfo(appID string) *types.ReportingInfo { } if clientset != nil && veleroClient != nil { - report, err := getSnapshotReport(store.GetStore(), clientset, veleroClient, appID, r.ClusterID) + bsl, err := snapshot.FindBackupStoreLocation(context.TODO(), clientset, veleroClient, util.PodNamespace) if err != nil { - logger.Debugf("failed to get snapshot report: %v", err.Error()) + logger.Debugf("failed to find backup store location: %v", err.Error()) } else { - r.SnapshotProvider = report.Provider - r.SnapshotFullSchedule = report.FullSchedule - r.SnapshotFullTTL = report.FullTTL - r.SnapshotPartialSchedule = report.PartialSchedule - r.SnapshotPartialTTL = report.PartialTTL + report, err := getSnapshotReport(store.GetStore(), bsl, appID, r.ClusterID) + if err != nil { + logger.Debugf("failed to get snapshot report: %v", err.Error()) + } else { + r.SnapshotProvider = report.Provider + r.SnapshotFullSchedule = report.FullSchedule + r.SnapshotFullTTL = report.FullTTL + r.SnapshotPartialSchedule = report.PartialSchedule + r.SnapshotPartialTTL = report.PartialTTL + } } } @@ -362,13 +368,9 @@ func getGitOpsReport(clientset kubernetes.Interface, appID string, clusterID str return false, "" } -func getSnapshotReport(kotsStore store.Store, clientset kubernetes.Interface, veleroClient veleroclientv1.VeleroV1Interface, appID string, clusterID string) (*SnapshotReport, error) { +func getSnapshotReport(kotsStore store.Store, bsl *velerov1.BackupStorageLocation, appID string, clusterID string) (*SnapshotReport, error) { report := &SnapshotReport{} - bsl, err := snapshot.FindBackupStoreLocation(context.TODO(), clientset, veleroClient, util.PodNamespace) - if err != nil { - return nil, errors.Wrap(err, "failed to find backup store location") - } if bsl == nil { return nil, errors.New("no backup store location found") } diff --git a/pkg/reporting/app_test.go b/pkg/reporting/app_test.go index 60af007b4a..1053a869bf 100644 --- a/pkg/reporting/app_test.go +++ b/pkg/reporting/app_test.go @@ -11,13 +11,7 @@ import ( "github.com/replicatedhq/kots/pkg/store" mock_store "github.com/replicatedhq/kots/pkg/store/mock" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - velerofake "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake" - veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/fake" ) func Test_getSnapshotReport(t *testing.T) { @@ -28,30 +22,25 @@ func Test_getSnapshotReport(t *testing.T) { t.Setenv("POD_NAMESPACE", "default") testVeleroNamespace := "velero" - veleroNamespaceConfigmap := &corev1.ConfigMap{ + testBsl := &velerov1.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ - Name: "kotsadm-velero-namespace", - }, - Data: map[string]string{ - "veleroNamespace": testVeleroNamespace, - }, - } - veleroDeployment := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "velero", + Name: "default", Namespace: testVeleroNamespace, }, + Spec: velerov1.BackupStorageLocationSpec{ + Provider: "aws", + Default: true, + }, } testAppID := "test-app-id" testClusterID := "test-cluster-id" type args struct { - kotsStore store.Store - clientset kubernetes.Interface - veleroClient veleroclientv1.VeleroV1Interface - appID string - clusterID string + kotsStore store.Store + bsl *velerov1.BackupStorageLocation + appID string + clusterID string } tests := []struct { name string @@ -64,10 +53,7 @@ func Test_getSnapshotReport(t *testing.T) { name: "happy path with schedule and ttl", args: args{ kotsStore: mockStore, - clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), - veleroClient: velerofake.NewSimpleClientset( - backupStorageLocationWithProvider(testVeleroNamespace, "aws"), - ).VeleroV1(), + bsl: testBsl, appID: testAppID, clusterID: testClusterID, }, @@ -97,10 +83,7 @@ func Test_getSnapshotReport(t *testing.T) { name: "happy path with default ttl only", args: args{ kotsStore: mockStore, - clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), - veleroClient: velerofake.NewSimpleClientset( - backupStorageLocationWithProvider(testVeleroNamespace, "aws"), - ).VeleroV1(), + bsl: testBsl, appID: testAppID, clusterID: testClusterID, }, @@ -129,11 +112,10 @@ func Test_getSnapshotReport(t *testing.T) { { name: "no backup storage location", args: args{ - kotsStore: mockStore, - clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), - veleroClient: velerofake.NewSimpleClientset().VeleroV1(), - appID: testAppID, - clusterID: testClusterID, + kotsStore: mockStore, + bsl: nil, + appID: testAppID, + clusterID: testClusterID, }, mockStoreExpectations: func() {}, wantErr: true, @@ -142,10 +124,7 @@ func Test_getSnapshotReport(t *testing.T) { name: "failed to list clusters", args: args{ kotsStore: mockStore, - clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), - veleroClient: velerofake.NewSimpleClientset( - backupStorageLocationWithProvider(testVeleroNamespace, "aws"), - ).VeleroV1(), + bsl: testBsl, appID: testAppID, clusterID: testClusterID, }, @@ -158,10 +137,7 @@ func Test_getSnapshotReport(t *testing.T) { name: "failed to get app", args: args{ kotsStore: mockStore, - clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), - veleroClient: velerofake.NewSimpleClientset( - backupStorageLocationWithProvider(testVeleroNamespace, "aws"), - ).VeleroV1(), + bsl: testBsl, appID: testAppID, clusterID: testClusterID, }, @@ -181,7 +157,7 @@ func Test_getSnapshotReport(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { tt.mockStoreExpectations() - got, err := getSnapshotReport(tt.args.kotsStore, tt.args.clientset, tt.args.veleroClient, tt.args.appID, tt.args.clusterID) + got, err := getSnapshotReport(tt.args.kotsStore, tt.args.bsl, tt.args.appID, tt.args.clusterID) if (err != nil) != tt.wantErr { t.Errorf("getSnapshotReport() error = %v, wantErr %v", err, tt.wantErr) return @@ -192,16 +168,3 @@ func Test_getSnapshotReport(t *testing.T) { }) } } - -func backupStorageLocationWithProvider(namespace string, provider string) *velerov1.BackupStorageLocation { - return &velerov1.BackupStorageLocation{ - ObjectMeta: metav1.ObjectMeta{ - Name: "default", - Namespace: namespace, - }, - Spec: velerov1.BackupStorageLocationSpec{ - Provider: provider, - Default: true, - }, - } -} diff --git a/pkg/snapshot/store_test.go b/pkg/snapshot/store_test.go index 37dfe64137..5443527401 100644 --- a/pkg/snapshot/store_test.go +++ b/pkg/snapshot/store_test.go @@ -2,13 +2,19 @@ package snapshot import ( "context" + "reflect" "testing" "github.com/replicatedhq/kots/pkg/snapshot/types" "github.com/stretchr/testify/require" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerofake "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake" + veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" testclient "k8s.io/client-go/kubernetes/fake" "k8s.io/utils/pointer" ) @@ -510,3 +516,85 @@ func Test_isMinioMigration(t *testing.T) { }) } } + +func TestFindBackupStoreLocation(t *testing.T) { + + testVeleroNamespace := "velero" + testBsl := &velerov1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: testVeleroNamespace, + }, + Spec: velerov1.BackupStorageLocationSpec{ + Provider: "aws", + Default: true, + }, + } + veleroNamespaceConfigmap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm-velero-namespace", + }, + Data: map[string]string{ + "veleroNamespace": testVeleroNamespace, + }, + } + veleroDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "velero", + Namespace: testVeleroNamespace, + }, + } + + type args struct { + clientset kubernetes.Interface + veleroClient veleroclientv1.VeleroV1Interface + kotsadmNamespace string + } + tests := []struct { + name string + args args + want *velerov1.BackupStorageLocation + wantErr bool + }{ + { + name: "backup store location found", + args: args{ + clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), + veleroClient: velerofake.NewSimpleClientset(testBsl).VeleroV1(), + kotsadmNamespace: "default", + }, + want: testBsl, + }, + { + name: "return nil if no backup store location found", + args: args{ + clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), + veleroClient: velerofake.NewSimpleClientset().VeleroV1(), + kotsadmNamespace: "default", + }, + want: nil, + }, + { + name: "return nil if no velero deployment found", + args: args{ + clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap), + veleroClient: velerofake.NewSimpleClientset(testBsl).VeleroV1(), + kotsadmNamespace: "default", + }, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + got, err := FindBackupStoreLocation(ctx, tt.args.clientset, tt.args.veleroClient, tt.args.kotsadmNamespace) + if (err != nil) != tt.wantErr { + t.Errorf("FindBackupStoreLocation() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("FindBackupStoreLocation() = %v, want %v", got, tt.want) + } + }) + } +} From 44939c25620bd3096fbca9a7c7a4f5fea86b2b9e Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Tue, 21 Nov 2023 21:14:14 +0000 Subject: [PATCH 09/10] clean up unit tests --- pkg/reporting/app_test.go | 2 -- pkg/snapshot/store_test.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/reporting/app_test.go b/pkg/reporting/app_test.go index 1053a869bf..1c1ca0a336 100644 --- a/pkg/reporting/app_test.go +++ b/pkg/reporting/app_test.go @@ -19,8 +19,6 @@ func Test_getSnapshotReport(t *testing.T) { defer ctrl.Finish() mockStore := mock_store.NewMockStore(ctrl) - t.Setenv("POD_NAMESPACE", "default") - testVeleroNamespace := "velero" testBsl := &velerov1.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/snapshot/store_test.go b/pkg/snapshot/store_test.go index 5443527401..84a000b520 100644 --- a/pkg/snapshot/store_test.go +++ b/pkg/snapshot/store_test.go @@ -518,7 +518,7 @@ func Test_isMinioMigration(t *testing.T) { } func TestFindBackupStoreLocation(t *testing.T) { - + t.Setenv("POD_NAMESPACE", "default") testVeleroNamespace := "velero" testBsl := &velerov1.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ From eacf50e07c85e10d24238b38041f2d70791822ad Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Wed, 22 Nov 2023 14:53:57 +0000 Subject: [PATCH 10/10] remove unused struct --- pkg/api/reporting/types/types.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/api/reporting/types/types.go b/pkg/api/reporting/types/types.go index 8f0b857f2a..bba8e48f8e 100644 --- a/pkg/api/reporting/types/types.go +++ b/pkg/api/reporting/types/types.go @@ -36,12 +36,6 @@ type DownstreamInfo struct { NativeHelmInstalls int `json:"native_helm_installs"` } -type SnapshotReport struct { - Provider string - Schedule string - RetentionPolicy string -} - // This type is mimicked in the preflight_report table. type PreflightStatus struct { InstanceID string `json:"instance_id"`