From 09f1c18ece846562ca705604cbf41ec2e619989e Mon Sep 17 00:00:00 2001 From: Craig O'Donnell Date: Wed, 22 Nov 2023 15:46:42 -0500 Subject: [PATCH] snapshot reporting (#4138) --- pkg/api/reporting/types/types.go | 33 +++--- pkg/handlers/snapshot_logs.go | 29 +++++- pkg/kotsadm/minio.go | 24 +++-- pkg/kotsadmsnapshot/backup.go | 137 ++++++++++++++----------- pkg/kotsadmsnapshot/restore.go | 81 +++++++++------ pkg/reporting/app.go | 81 ++++++++++++++- pkg/reporting/app_airgap.go | 5 + pkg/reporting/app_test.go | 168 +++++++++++++++++++++++++++++++ pkg/reporting/instance_report.go | 5 + pkg/reporting/report_test.go | 5 + pkg/reporting/util.go | 6 ++ pkg/snapshot/filesystem_lvp.go | 27 +++-- pkg/snapshot/store.go | 40 ++++---- pkg/snapshot/store_test.go | 88 ++++++++++++++++ pkg/snapshot/velero.go | 5 - 15 files changed, 592 insertions(+), 142 deletions(-) create mode 100644 pkg/reporting/app_test.go diff --git a/pkg/api/reporting/types/types.go b/pkg/api/reporting/types/types.go index ec8ac91333..bba8e48f8e 100644 --- a/pkg/api/reporting/types/types.go +++ b/pkg/api/reporting/types/types.go @@ -2,20 +2,25 @@ package types // This type is mimicked in the instance_report table. type ReportingInfo struct { - InstanceID string `json:"instance_id"` - ClusterID string `json:"cluster_id"` - Downstream DownstreamInfo `json:"downstream"` - AppStatus string `json:"app_status"` - IsKurl bool `json:"is_kurl"` - KurlNodeCountTotal int `json:"kurl_node_count_total"` - KurlNodeCountReady int `json:"kurl_node_count_ready"` - K8sVersion string `json:"k8s_version"` - K8sDistribution string `json:"k8s_distribution"` - UserAgent string `json:"user_agent"` - KOTSInstallID string `json:"kots_install_id"` - KURLInstallID string `json:"kurl_install_id"` - IsGitOpsEnabled bool `json:"is_gitops_enabled"` - GitOpsProvider string `json:"gitops_provider"` + InstanceID string `json:"instance_id"` + ClusterID string `json:"cluster_id"` + Downstream DownstreamInfo `json:"downstream"` + AppStatus string `json:"app_status"` + IsKurl bool `json:"is_kurl"` + KurlNodeCountTotal int `json:"kurl_node_count_total"` + KurlNodeCountReady int `json:"kurl_node_count_ready"` + K8sVersion string `json:"k8s_version"` + K8sDistribution string `json:"k8s_distribution"` + UserAgent string `json:"user_agent"` + KOTSInstallID string `json:"kots_install_id"` + KURLInstallID string `json:"kurl_install_id"` + IsGitOpsEnabled bool `json:"is_gitops_enabled"` + GitOpsProvider string `json:"gitops_provider"` + SnapshotProvider string `json:"snapshot_provider"` + SnapshotFullSchedule string `json:"snapshot_full_schedule"` + SnapshotFullTTL string `json:"snapshot_full_ttl"` + SnapshotPartialSchedule string `json:"snapshot_partial_schedule"` + SnapshotPartialTTL string `json:"snapshot_partial_ttl"` } type DownstreamInfo struct { diff --git a/pkg/handlers/snapshot_logs.go b/pkg/handlers/snapshot_logs.go index 91c2b1f7e7..512688ed4c 100644 --- a/pkg/handlers/snapshot_logs.go +++ b/pkg/handlers/snapshot_logs.go @@ -6,17 +6,44 @@ import ( "github.com/gorilla/mux" "github.com/pkg/errors" + "github.com/replicatedhq/kots/pkg/k8sutil" snapshot "github.com/replicatedhq/kots/pkg/kotsadmsnapshot" "github.com/replicatedhq/kots/pkg/logger" kotssnapshot "github.com/replicatedhq/kots/pkg/snapshot" "github.com/replicatedhq/kots/pkg/util" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" + "k8s.io/client-go/kubernetes" ) func (h *Handler) DownloadSnapshotLogs(w http.ResponseWriter, r *http.Request) { backupName := mux.Vars(r)["backup"] - bsl, err := kotssnapshot.FindBackupStoreLocation(r.Context(), util.PodNamespace) + cfg, err := k8sutil.GetClusterConfig() + if err != nil { + err = errors.Wrap(err, "failed to get cluster config") + logger.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + err = errors.Wrap(err, "failed to create clientset") + logger.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + err = errors.Wrap(err, "failed to create velero clientset") + logger.Error(err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + bsl, err := kotssnapshot.FindBackupStoreLocation(r.Context(), clientset, veleroClient, util.PodNamespace) if err != nil { err = errors.Wrap(err, "failed to find backup store location") logger.Error(err) diff --git a/pkg/kotsadm/minio.go b/pkg/kotsadm/minio.go index f342825c9d..7385b989e9 100644 --- a/pkg/kotsadm/minio.go +++ b/pkg/kotsadm/minio.go @@ -14,6 +14,7 @@ import ( "github.com/replicatedhq/kots/pkg/logger" "github.com/replicatedhq/kots/pkg/snapshot" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" corev1 "k8s.io/api/core/v1" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -180,7 +181,22 @@ func MigrateExistingMinioFilesystemDeployments(log *logger.CLILogger, deployOpti } veleroNamespace := veleroStatus.Namespace - bsl, err := snapshot.FindBackupStoreLocation(context.TODO(), deployOptions.Namespace) + cfg, err := k8sutil.GetClusterConfig() + if err != nil { + return errors.Wrap(err, "failed to get cluster config") + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return errors.Wrap(err, "failed to create clientset") + } + + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return errors.Wrap(err, "failed to create velero clientset") + } + + bsl, err := snapshot.FindBackupStoreLocation(context.TODO(), clientset, veleroClient, deployOptions.Namespace) if err != nil { return errors.Wrap(err, "failed to find backupstoragelocations") } @@ -226,10 +242,6 @@ func MigrateExistingMinioFilesystemDeployments(log *logger.CLILogger, deployOpti } // Add the config map to configure the new plugin - clientset, err := k8sutil.GetClientset() - if err != nil { - return errors.Wrap(err, "failed to get k8s clientset") - } fsDeployOptions := &snapshot.FileSystemDeployOptions{ Namespace: deployOptions.Namespace, IsOpenShift: k8sutil.IsOpenShift(clientset), @@ -250,7 +262,7 @@ func MigrateExistingMinioFilesystemDeployments(log *logger.CLILogger, deployOpti success := false defer func() { if !success { - err := snapshot.RevertToMinioFS(context.TODO(), deployOptions.Namespace, veleroNamespace, previousBsl) + err := snapshot.RevertToMinioFS(context.TODO(), clientset, veleroClient, deployOptions.Namespace, veleroNamespace, previousBsl) if err != nil { log.Error(errors.Wrap(err, "Could not restore minio backup storage location")) return diff --git a/pkg/kotsadmsnapshot/backup.go b/pkg/kotsadmsnapshot/backup.go index b5e8b826e4..779661c7c9 100644 --- a/pkg/kotsadmsnapshot/backup.go +++ b/pkg/kotsadmsnapshot/backup.go @@ -69,8 +69,23 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b return nil, errors.Wrap(err, "failed to get app version archive") } + cfg, err := k8sutil.GetClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to get cluster config") + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create clientset") + } + + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + kotsadmNamespace := util.PodNamespace - kotsadmVeleroBackendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + kotsadmVeleroBackendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } @@ -160,26 +175,11 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b } } - clientset, err := k8sutil.GetClientset() - if err != nil { - return nil, errors.Wrap(err, "failed to create k8s clientset") - } - err = excludeShutdownPodsFromBackup(ctx, clientset, veleroBackup) if err != nil { logger.Error(errors.Wrap(err, "failed to exclude shutdown pods from backup")) } - cfg, err := k8sutil.GetClusterConfig() - if err != nil { - return nil, errors.Wrap(err, "failed to get cluster config") - } - - veleroClient, err := veleroclientv1.NewForConfig(cfg) - if err != nil { - return nil, errors.Wrap(err, "failed to create clientset") - } - backup, err := veleroClient.Backups(kotsadmVeleroBackendStorageLocation.Namespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) if err != nil { return nil, errors.Wrap(err, "failed to create velero backup") @@ -191,9 +191,14 @@ func CreateApplicationBackup(ctx context.Context, a *apptypes.App, isScheduled b func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstream, isScheduled bool) (*velerov1.Backup, error) { logger.Debug("creating instance backup") - clientset, err := k8sutil.GetClientset() + cfg, err := k8sutil.GetClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to get cluster config") + } + + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { - return nil, errors.Wrap(err, "failed to create k8s clientset") + return nil, errors.Wrap(err, "failed to create clientset") } isKurl, err := kurl.IsKurl(clientset) @@ -310,7 +315,12 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre backupHooks.Resources = append(backupHooks.Resources, veleroBackup.Spec.Hooks.Resources...) } - kotsadmVeleroBackendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + kotsadmVeleroBackendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } @@ -398,16 +408,6 @@ func CreateInstanceBackup(ctx context.Context, cluster *downstreamtypes.Downstre logger.Error(errors.Wrap(err, "failed to exclude shutdown pods from backup")) } - cfg, err := k8sutil.GetClusterConfig() - if err != nil { - return nil, errors.Wrap(err, "failed to get cluster config") - } - - veleroClient, err := veleroclientv1.NewForConfig(cfg) - if err != nil { - return nil, errors.Wrap(err, "failed to create velero clientset") - } - backup, err := veleroClient.Backups(kotsadmVeleroBackendStorageLocation.Namespace).Create(ctx, veleroBackup, metav1.CreateOptions{}) if err != nil { return nil, errors.Wrap(err, "failed to create velero backup") @@ -422,12 +422,17 @@ func ListBackupsForApp(ctx context.Context, kotsadmNamespace string, appID strin return nil, errors.Wrap(err, "failed to get cluster config") } - veleroClient, err := veleroclientv1.NewForConfig(cfg) + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { return nil, errors.Wrap(err, "failed to create clientset") } - backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } @@ -550,12 +555,17 @@ func ListInstanceBackups(ctx context.Context, kotsadmNamespace string) ([]*types return nil, errors.Wrap(err, "failed to get cluster config") } - veleroClient, err := veleroclientv1.NewForConfig(cfg) + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { return nil, errors.Wrap(err, "failed to create clientset") } - backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } @@ -717,7 +727,22 @@ func getSnapshotVolumeSummary(ctx context.Context, veleroBackup *velerov1.Backup } func GetBackup(ctx context.Context, kotsadmNamespace string, snapshotName string) (*velerov1.Backup, error) { - bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + cfg, err := k8sutil.GetClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to get cluster config") + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create clientset") + } + + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to get velero namespace") } @@ -728,26 +753,31 @@ func GetBackup(ctx context.Context, kotsadmNamespace string, snapshotName string veleroNamespace := bsl.Namespace // get the backup - cfg, err := k8sutil.GetClusterConfig() + backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, snapshotName, metav1.GetOptions{}) if err != nil { - return nil, errors.Wrap(err, "failed to get cluster config") + return nil, errors.Wrap(err, "failed to get backup") } - veleroClient, err := veleroclientv1.NewForConfig(cfg) + return backup, nil +} + +func DeleteBackup(ctx context.Context, kotsadmNamespace string, snapshotName string) error { + cfg, err := k8sutil.GetClusterConfig() if err != nil { - return nil, errors.Wrap(err, "failed to create clientset") + return errors.Wrap(err, "failed to get cluster config") } - backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, snapshotName, metav1.GetOptions{}) + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { - return nil, errors.Wrap(err, "failed to get backup") + return errors.Wrap(err, "failed to create clientset") } - return backup, nil -} + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return errors.Wrap(err, "failed to create velero clientset") + } -func DeleteBackup(ctx context.Context, kotsadmNamespace string, snapshotName string) error { - bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return errors.Wrap(err, "failed to get velero namespace") } @@ -766,16 +796,6 @@ func DeleteBackup(ctx context.Context, kotsadmNamespace string, snapshotName str }, } - cfg, err := k8sutil.GetClusterConfig() - if err != nil { - return errors.Wrap(err, "failed to get cluster config") - } - - veleroClient, err := veleroclientv1.NewForConfig(cfg) - if err != nil { - return errors.Wrap(err, "failed to create clientset") - } - _, err = veleroClient.DeleteBackupRequests(veleroNamespace).Create(context.TODO(), veleroDeleteBackupRequest, metav1.CreateOptions{}) if err != nil { return errors.Wrap(err, "failed to create delete backup request") @@ -820,12 +840,17 @@ func GetBackupDetail(ctx context.Context, kotsadmNamespace string, backupName st return nil, errors.Wrap(err, "failed to get cluster config") } - veleroClient, err := veleroclientv1.NewForConfig(cfg) + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { return nil, errors.Wrap(err, "failed to create clientset") } - backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } diff --git a/pkg/kotsadmsnapshot/restore.go b/pkg/kotsadmsnapshot/restore.go index e924cf06e2..97c1d20f0c 100644 --- a/pkg/kotsadmsnapshot/restore.go +++ b/pkg/kotsadmsnapshot/restore.go @@ -19,29 +19,35 @@ import ( "go.uber.org/zap" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" "k8s.io/utils/pointer" ) func GetRestore(ctx context.Context, kotsadmNamespace string, snapshotName string) (*velerov1.Restore, error) { - bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + cfg, err := k8sutil.GetClusterConfig() if err != nil { - return nil, errors.Wrap(err, "failed to get velero namespace") - } - if bsl == nil { - return nil, errors.New("no backup store location found") + return nil, errors.Wrap(err, "failed to get cluster config") } - veleroNamespace := bsl.Namespace - - cfg, err := k8sutil.GetClusterConfig() + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { - return nil, errors.Wrap(err, "failed to get cluster config") + return nil, errors.Wrap(err, "failed to create clientset") } veleroClient, err := veleroclientv1.NewForConfig(cfg) if err != nil { - return nil, errors.Wrap(err, "failed to create clientset") + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) + if err != nil { + return nil, errors.Wrap(err, "failed to get velero namespace") } + if bsl == nil { + return nil, errors.New("no backup store location found") + } + + veleroNamespace := bsl.Namespace restore, err := veleroClient.Restores(veleroNamespace).Get(ctx, snapshotName, metav1.GetOptions{}) if err != nil { @@ -60,27 +66,32 @@ func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snap logger.Debug("creating restore", zap.String("snapshotName", snapshotName)) - bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + cfg, err := k8sutil.GetClusterConfig() if err != nil { - return errors.Wrap(err, "failed to get velero namespace") - } - if bsl == nil { - return errors.New("no backup store location found") + return errors.Wrap(err, "failed to get cluster config") } - veleroNamespace := bsl.Namespace - - // get the backup - cfg, err := k8sutil.GetClusterConfig() + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { - return errors.Wrap(err, "failed to get cluster config") + return errors.Wrap(err, "failed to create clientset") } veleroClient, err := veleroclientv1.NewForConfig(cfg) if err != nil { - return errors.Wrap(err, "failed to create clientset") + return errors.Wrap(err, "failed to create velero clientset") + } + + bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) + if err != nil { + return errors.Wrap(err, "failed to get velero namespace") + } + if bsl == nil { + return errors.New("no backup store location found") } + veleroNamespace := bsl.Namespace + + // get the backup backup, err := veleroClient.Backups(veleroNamespace).Get(ctx, snapshotName, metav1.GetOptions{}) if err != nil { return errors.Wrap(err, "failed to find backup") @@ -120,23 +131,28 @@ func CreateApplicationRestore(ctx context.Context, kotsadmNamespace string, snap } func DeleteRestore(ctx context.Context, kotsadmNamespace string, snapshotName string) error { - bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + cfg, err := k8sutil.GetClusterConfig() if err != nil { - return errors.Wrap(err, "failed to get velero namespace") + return errors.Wrap(err, "failed to get cluster config") } - veleroNamespace := bsl.Namespace - - cfg, err := k8sutil.GetClusterConfig() + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { - return errors.Wrap(err, "failed to get cluster config") + return errors.Wrap(err, "failed to create clientset") } veleroClient, err := veleroclientv1.NewForConfig(cfg) if err != nil { - return errors.Wrap(err, "failed to create clientset") + return errors.Wrap(err, "failed to create velero clientset") + } + + bsl, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) + if err != nil { + return errors.Wrap(err, "failed to get velero namespace") } + veleroNamespace := bsl.Namespace + err = veleroClient.Restores(veleroNamespace).Delete(ctx, snapshotName, metav1.DeleteOptions{}) if err != nil && !strings.Contains(err.Error(), "not found") { return errors.Wrapf(err, "failed to delete restore %s", snapshotName) @@ -151,12 +167,17 @@ func GetRestoreDetails(ctx context.Context, kotsadmNamespace string, restoreName return nil, errors.Wrap(err, "failed to get cluster config") } - veleroClient, err := veleroclientv1.NewForConfig(cfg) + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { return nil, errors.Wrap(err, "failed to create clientset") } - backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, kotsadmNamespace) + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + backendStorageLocation, err := kotssnapshot.FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } diff --git a/pkg/reporting/app.go b/pkg/reporting/app.go index 744e8f35ce..96d4b3cacc 100644 --- a/pkg/reporting/app.go +++ b/pkg/reporting/app.go @@ -6,11 +6,13 @@ import ( "context" "encoding/base64" "encoding/json" + "fmt" "io/ioutil" "os" "path/filepath" "github.com/pkg/errors" + downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types" "github.com/replicatedhq/kots/pkg/api/reporting/types" "github.com/replicatedhq/kots/pkg/buildversion" "github.com/replicatedhq/kots/pkg/gitops" @@ -19,10 +21,13 @@ import ( "github.com/replicatedhq/kots/pkg/kotsutil" "github.com/replicatedhq/kots/pkg/kurl" "github.com/replicatedhq/kots/pkg/logger" + "github.com/replicatedhq/kots/pkg/snapshot" "github.com/replicatedhq/kots/pkg/store" "github.com/replicatedhq/kots/pkg/util" troubleshootpreflight "github.com/replicatedhq/troubleshoot/pkg/preflight" "github.com/segmentio/ksuid" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" helmrelease "helm.sh/helm/v3/pkg/release" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -33,6 +38,14 @@ var ( clusterID string // set when in Helm managed mode ) +type SnapshotReport struct { + Provider string + FullSchedule string + FullTTL string + PartialSchedule string + PartialTTL string +} + func Init() error { if util.IsHelmManaged() { err := initFromHelm() @@ -185,9 +198,14 @@ func GetReportingInfo(appID string) *types.ReportingInfo { UserAgent: buildversion.GetUserAgent(), } - clientset, err := k8sutil.GetClientset() + cfg, err := k8sutil.GetClusterConfig() + if err != nil { + logger.Debugf("failed to get cluster config: %v", err.Error()) + } + + clientset, err := kubernetes.NewForConfig(cfg) if err != nil { - logger.Debugf(errors.Wrap(err, "failed to get kubernetes clientset").Error()) + logger.Debugf("failed to get clientset: %v", err.Error()) } if util.IsHelmManaged() { @@ -248,6 +266,30 @@ func GetReportingInfo(appID string) *types.ReportingInfo { } r.IsGitOpsEnabled, r.GitOpsProvider = getGitOpsReport(clientset, appID, r.ClusterID) + + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + logger.Debugf("failed to get velero client: %v", err.Error()) + } + + if clientset != nil && veleroClient != nil { + bsl, err := snapshot.FindBackupStoreLocation(context.TODO(), clientset, veleroClient, util.PodNamespace) + if err != nil { + logger.Debugf("failed to find backup store location: %v", err.Error()) + } else { + report, err := getSnapshotReport(store.GetStore(), bsl, appID, r.ClusterID) + if err != nil { + logger.Debugf("failed to get snapshot report: %v", err.Error()) + } else { + r.SnapshotProvider = report.Provider + r.SnapshotFullSchedule = report.FullSchedule + r.SnapshotFullTTL = report.FullTTL + r.SnapshotPartialSchedule = report.PartialSchedule + r.SnapshotPartialTTL = report.PartialTTL + } + } + } + return &r } @@ -325,3 +367,38 @@ func getGitOpsReport(clientset kubernetes.Interface, appID string, clusterID str } return false, "" } + +func getSnapshotReport(kotsStore store.Store, bsl *velerov1.BackupStorageLocation, appID string, clusterID string) (*SnapshotReport, error) { + report := &SnapshotReport{} + + if bsl == nil { + return nil, errors.New("no backup store location found") + } + report.Provider = bsl.Spec.Provider + + clusters, err := kotsStore.ListClusters() + if err != nil { + return nil, errors.Wrap(err, "failed to list clusters") + } + var downstream *downstreamtypes.Downstream + for _, cluster := range clusters { + if cluster.ClusterID == clusterID { + downstream = cluster + break + } + } + if downstream == nil { + return nil, fmt.Errorf("cluster %s not found", clusterID) + } + report.FullSchedule = downstream.SnapshotSchedule + report.FullTTL = downstream.SnapshotTTL + + app, err := kotsStore.GetApp(appID) + if err != nil { + return nil, errors.Wrap(err, "failed to get app") + } + report.PartialSchedule = app.SnapshotSchedule + report.PartialTTL = app.SnapshotTTL + + return report, nil +} diff --git a/pkg/reporting/app_airgap.go b/pkg/reporting/app_airgap.go index ca94895d9c..d35eda2e6b 100644 --- a/pkg/reporting/app_airgap.go +++ b/pkg/reporting/app_airgap.go @@ -69,6 +69,11 @@ func BuildInstanceReport(licenseID string, reportingInfo *types.ReportingInfo) * KurlInstallID: reportingInfo.KURLInstallID, IsGitOpsEnabled: reportingInfo.IsGitOpsEnabled, GitOpsProvider: reportingInfo.GitOpsProvider, + SnapshotProvider: reportingInfo.SnapshotProvider, + SnapshotFullSchedule: reportingInfo.SnapshotFullSchedule, + SnapshotFullTTL: reportingInfo.SnapshotFullTTL, + SnapshotPartialSchedule: reportingInfo.SnapshotPartialSchedule, + SnapshotPartialTTL: reportingInfo.SnapshotPartialTTL, DownstreamChannelID: reportingInfo.Downstream.ChannelID, DownstreamChannelSequence: downstreamSequence, DownstreamChannelName: reportingInfo.Downstream.ChannelName, diff --git a/pkg/reporting/app_test.go b/pkg/reporting/app_test.go new file mode 100644 index 0000000000..1c1ca0a336 --- /dev/null +++ b/pkg/reporting/app_test.go @@ -0,0 +1,168 @@ +package reporting + +import ( + "errors" + "reflect" + "testing" + + "github.com/golang/mock/gomock" + downstreamtypes "github.com/replicatedhq/kots/pkg/api/downstream/types" + apptypes "github.com/replicatedhq/kots/pkg/app/types" + "github.com/replicatedhq/kots/pkg/store" + mock_store "github.com/replicatedhq/kots/pkg/store/mock" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Test_getSnapshotReport(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockStore := mock_store.NewMockStore(ctrl) + + testVeleroNamespace := "velero" + testBsl := &velerov1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: testVeleroNamespace, + }, + Spec: velerov1.BackupStorageLocationSpec{ + Provider: "aws", + Default: true, + }, + } + + testAppID := "test-app-id" + testClusterID := "test-cluster-id" + + type args struct { + kotsStore store.Store + bsl *velerov1.BackupStorageLocation + appID string + clusterID string + } + tests := []struct { + name string + args args + mockStoreExpectations func() + want *SnapshotReport + wantErr bool + }{ + { + name: "happy path with schedule and ttl", + args: args{ + kotsStore: mockStore, + bsl: testBsl, + appID: testAppID, + clusterID: testClusterID, + }, + mockStoreExpectations: func() { + mockStore.EXPECT().ListClusters().Return([]*downstreamtypes.Downstream{ + { + ClusterID: testClusterID, + SnapshotSchedule: "0 0 * * *", + SnapshotTTL: "720h", + }, + }, nil) + mockStore.EXPECT().GetApp(testAppID).Return(&apptypes.App{ + ID: testAppID, + SnapshotSchedule: "0 0 * * MON", + SnapshotTTL: "168h", + }, nil) + }, + want: &SnapshotReport{ + Provider: "aws", + FullSchedule: "0 0 * * *", + FullTTL: "720h", + PartialSchedule: "0 0 * * MON", + PartialTTL: "168h", + }, + }, + { + name: "happy path with default ttl only", + args: args{ + kotsStore: mockStore, + bsl: testBsl, + appID: testAppID, + clusterID: testClusterID, + }, + mockStoreExpectations: func() { + mockStore.EXPECT().ListClusters().Return([]*downstreamtypes.Downstream{ + { + ClusterID: testClusterID, + SnapshotSchedule: "", + SnapshotTTL: "720h", + }, + }, nil) + mockStore.EXPECT().GetApp(testAppID).Return(&apptypes.App{ + ID: testAppID, + SnapshotSchedule: "", + SnapshotTTL: "720h", + }, nil) + }, + want: &SnapshotReport{ + Provider: "aws", + FullSchedule: "", + FullTTL: "720h", + PartialSchedule: "", + PartialTTL: "720h", + }, + }, + { + name: "no backup storage location", + args: args{ + kotsStore: mockStore, + bsl: nil, + appID: testAppID, + clusterID: testClusterID, + }, + mockStoreExpectations: func() {}, + wantErr: true, + }, + { + name: "failed to list clusters", + args: args{ + kotsStore: mockStore, + bsl: testBsl, + appID: testAppID, + clusterID: testClusterID, + }, + mockStoreExpectations: func() { + mockStore.EXPECT().ListClusters().Return(nil, errors.New("failed to list clusters")) + }, + wantErr: true, + }, + { + name: "failed to get app", + args: args{ + kotsStore: mockStore, + bsl: testBsl, + appID: testAppID, + clusterID: testClusterID, + }, + mockStoreExpectations: func() { + mockStore.EXPECT().ListClusters().Return([]*downstreamtypes.Downstream{ + { + ClusterID: testClusterID, + SnapshotSchedule: "0 0 * * *", + SnapshotTTL: "720h", + }, + }, nil) + mockStore.EXPECT().GetApp(testAppID).Return(nil, errors.New("failed to get app")) + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt.mockStoreExpectations() + got, err := getSnapshotReport(tt.args.kotsStore, tt.args.bsl, tt.args.appID, tt.args.clusterID) + if (err != nil) != tt.wantErr { + t.Errorf("getSnapshotReport() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getSnapshotReport() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/reporting/instance_report.go b/pkg/reporting/instance_report.go index 92ef1d23b4..d83853871e 100644 --- a/pkg/reporting/instance_report.go +++ b/pkg/reporting/instance_report.go @@ -29,6 +29,11 @@ type InstanceReportEvent struct { KurlInstallID string `json:"kurl_install_id,omitempty"` IsGitOpsEnabled bool `json:"is_gitops_enabled"` GitOpsProvider string `json:"gitops_provider"` + SnapshotProvider string `json:"snapshot_provider"` + SnapshotFullSchedule string `json:"snapshot_full_schedule"` + SnapshotFullTTL string `json:"snapshot_full_ttl"` + SnapshotPartialSchedule string `json:"snapshot_partial_schedule"` + SnapshotPartialTTL string `json:"snapshot_partial_ttl"` DownstreamChannelID string `json:"downstream_channel_id,omitempty"` DownstreamChannelSequence uint64 `json:"downstream_channel_sequence,omitempty"` DownstreamChannelName string `json:"downstream_channel_name,omitempty"` diff --git a/pkg/reporting/report_test.go b/pkg/reporting/report_test.go index af367d26b1..ec39743893 100644 --- a/pkg/reporting/report_test.go +++ b/pkg/reporting/report_test.go @@ -353,6 +353,11 @@ func createTestInstanceEvent(reportedAt int64) InstanceReportEvent { KurlInstallID: "test-kurl-install-id", IsGitOpsEnabled: true, GitOpsProvider: "test-gitops-provider", + SnapshotProvider: "test-snapshot-provider", + SnapshotFullSchedule: "0 0 * * *", + SnapshotFullTTL: "720h", + SnapshotPartialSchedule: "0 0 * * *", + SnapshotPartialTTL: "720h", DownstreamChannelID: "test-downstream-channel-id", DownstreamChannelSequence: 123, DownstreamChannelName: "test-downstream-channel-name", diff --git a/pkg/reporting/util.go b/pkg/reporting/util.go index e9da1ba89f..b359217cfd 100644 --- a/pkg/reporting/util.go +++ b/pkg/reporting/util.go @@ -68,6 +68,12 @@ func GetReportingInfoHeaders(reportingInfo *types.ReportingInfo) map[string]stri headers["X-Replicated-IsGitOpsEnabled"] = strconv.FormatBool(reportingInfo.IsGitOpsEnabled) headers["X-Replicated-GitOpsProvider"] = reportingInfo.GitOpsProvider + headers["X-Replicated-SnapshotProvider"] = reportingInfo.SnapshotProvider + headers["X-Replicated-SnapshotFullSchedule"] = reportingInfo.SnapshotFullSchedule + headers["X-Replicated-SnapshotFullTTL"] = reportingInfo.SnapshotFullTTL + headers["X-Replicated-SnapshotPartialSchedule"] = reportingInfo.SnapshotPartialSchedule + headers["X-Replicated-SnapshotPartialTTL"] = reportingInfo.SnapshotPartialTTL + if reportingInfo.K8sDistribution != "" { headers["X-Replicated-K8sDistribution"] = reportingInfo.K8sDistribution } diff --git a/pkg/snapshot/filesystem_lvp.go b/pkg/snapshot/filesystem_lvp.go index 7210a0436e..55ff0e179e 100644 --- a/pkg/snapshot/filesystem_lvp.go +++ b/pkg/snapshot/filesystem_lvp.go @@ -11,6 +11,7 @@ import ( kotsadmtypes "github.com/replicatedhq/kots/pkg/kotsadm/types" types "github.com/replicatedhq/kots/pkg/snapshot/types" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" corev1 "k8s.io/api/core/v1" kuberneteserrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -45,7 +46,22 @@ func ValidateFileSystemDeployment(ctx context.Context, clientset kubernetes.Inte } func GetCurrentLvpFileSystemConfig(ctx context.Context, namespace string) (*types.FileSystemConfig, error) { - bsl, err := FindBackupStoreLocation(ctx, namespace) + cfg, err := k8sutil.GetClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to get cluster config") + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create clientset") + } + + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + bsl, err := FindBackupStoreLocation(ctx, clientset, veleroClient, namespace) if err != nil { return nil, errors.Wrap(err, "failed to find velero backup storage location") } @@ -70,11 +86,6 @@ func GetCurrentLvpFileSystemConfig(ctx context.Context, namespace string) (*type } // backup storage location does not exist, get file system config from the config map - clientset, err := k8sutil.GetClientset() - if err != nil { - return nil, errors.Wrap(err, "failed to get k8s clientset") - } - fileSystemConfigMap, err := clientset.CoreV1().ConfigMaps(namespace).Get(ctx, FileSystemLVPConfigMapName, metav1.GetOptions{}) if err != nil { if kuberneteserrors.IsNotFound(err) { @@ -103,8 +114,8 @@ func GetCurrentLvpFileSystemConfig(ctx context.Context, namespace string) (*type // RevertToMinioFS will apply the spec of the previous BSL to the current one and then update. // Used for recovery during a failed migration from Minio to LVP. -func RevertToMinioFS(ctx context.Context, kotsadmNamespace, veleroNamespace string, previousBsl *velerov1api.BackupStorageLocation) error { - bsl, err := FindBackupStoreLocation(context.TODO(), kotsadmNamespace) +func RevertToMinioFS(ctx context.Context, clientset kubernetes.Interface, veleroClient veleroclientv1.VeleroV1Interface, kotsadmNamespace, veleroNamespace string, previousBsl *velerov1api.BackupStorageLocation) error { + bsl, err := FindBackupStoreLocation(context.TODO(), clientset, veleroClient, kotsadmNamespace) if err != nil { return errors.Wrap(err, "failed to find backupstoragelocations") } diff --git a/pkg/snapshot/store.go b/pkg/snapshot/store.go index 11d090ee0f..17ddf21e84 100644 --- a/pkg/snapshot/store.go +++ b/pkg/snapshot/store.go @@ -427,7 +427,12 @@ func upsertGlobalStore(ctx context.Context, store *types.Store, kotsadmNamespace return nil, errors.Wrap(err, "failed to create clientset") } - bsl, err := FindBackupStoreLocation(ctx, kotsadmNamespace) + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") + } + + bsl, err := FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } @@ -867,13 +872,23 @@ func updateLvpFileSystemStore(ctx context.Context, store *types.Store, bsl *vele // GetGlobalStore will return the global store from the current backup storage location // or will find it, if the param is nil func GetGlobalStore(ctx context.Context, kotsadmNamespace string, bsl *velerov1.BackupStorageLocation) (*types.Store, error) { - clientset, err := k8sutil.GetClientset() + cfg, err := k8sutil.GetClusterConfig() if err != nil { - return nil, errors.Wrap(err, "failed to get k8s clientset") + return nil, errors.Wrap(err, "failed to get cluster config") + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create clientset") + } + + veleroClient, err := veleroclientv1.NewForConfig(cfg) + if err != nil { + return nil, errors.Wrap(err, "failed to create velero clientset") } if bsl == nil { - bsl, err = FindBackupStoreLocation(ctx, kotsadmNamespace) + bsl, err = FindBackupStoreLocation(ctx, clientset, veleroClient, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to find backupstoragelocations") } @@ -1055,17 +1070,7 @@ func mapAWSBackupStorageLocationToStore(kotsadmVeleroBackendStorageLocation *vel // FindBackupStoreLocation will find the backup storage location used by velero // kotsadmNamespace is only required in minimal rbac installations. if empty, cluster scope privileges will be needed to detect and validate velero -func FindBackupStoreLocation(ctx context.Context, kotsadmNamespace string) (*velerov1.BackupStorageLocation, error) { - cfg, err := k8sutil.GetClusterConfig() - if err != nil { - return nil, errors.Wrap(err, "failed to get cluster config") - } - - clientset, err := kubernetes.NewForConfig(cfg) - if err != nil { - return nil, errors.Wrap(err, "failed to create clientset") - } - +func FindBackupStoreLocation(ctx context.Context, clientset kubernetes.Interface, veleroClient veleroclientv1.VeleroV1Interface, kotsadmNamespace string) (*velerov1.BackupStorageLocation, error) { veleroNamespace, err := DetectVeleroNamespace(ctx, clientset, kotsadmNamespace) if err != nil { return nil, errors.Wrap(err, "failed to detect velero namespace") @@ -1075,11 +1080,6 @@ func FindBackupStoreLocation(ctx context.Context, kotsadmNamespace string) (*vel return nil, nil } - veleroClient, err := veleroclientv1.NewForConfig(cfg) - if err != nil { - return nil, errors.Wrap(err, "failed to create velero clientset") - } - backupStorageLocations, err := veleroClient.BackupStorageLocations(veleroNamespace).List(ctx, metav1.ListOptions{}) if err != nil { if kuberneteserrors.IsNotFound(err) { diff --git a/pkg/snapshot/store_test.go b/pkg/snapshot/store_test.go index 37dfe64137..84a000b520 100644 --- a/pkg/snapshot/store_test.go +++ b/pkg/snapshot/store_test.go @@ -2,13 +2,19 @@ package snapshot import ( "context" + "reflect" "testing" "github.com/replicatedhq/kots/pkg/snapshot/types" "github.com/stretchr/testify/require" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerofake "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake" + veleroclientv1 "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" testclient "k8s.io/client-go/kubernetes/fake" "k8s.io/utils/pointer" ) @@ -510,3 +516,85 @@ func Test_isMinioMigration(t *testing.T) { }) } } + +func TestFindBackupStoreLocation(t *testing.T) { + t.Setenv("POD_NAMESPACE", "default") + testVeleroNamespace := "velero" + testBsl := &velerov1.BackupStorageLocation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: testVeleroNamespace, + }, + Spec: velerov1.BackupStorageLocationSpec{ + Provider: "aws", + Default: true, + }, + } + veleroNamespaceConfigmap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kotsadm-velero-namespace", + }, + Data: map[string]string{ + "veleroNamespace": testVeleroNamespace, + }, + } + veleroDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "velero", + Namespace: testVeleroNamespace, + }, + } + + type args struct { + clientset kubernetes.Interface + veleroClient veleroclientv1.VeleroV1Interface + kotsadmNamespace string + } + tests := []struct { + name string + args args + want *velerov1.BackupStorageLocation + wantErr bool + }{ + { + name: "backup store location found", + args: args{ + clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), + veleroClient: velerofake.NewSimpleClientset(testBsl).VeleroV1(), + kotsadmNamespace: "default", + }, + want: testBsl, + }, + { + name: "return nil if no backup store location found", + args: args{ + clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap, veleroDeployment), + veleroClient: velerofake.NewSimpleClientset().VeleroV1(), + kotsadmNamespace: "default", + }, + want: nil, + }, + { + name: "return nil if no velero deployment found", + args: args{ + clientset: fake.NewSimpleClientset(veleroNamespaceConfigmap), + veleroClient: velerofake.NewSimpleClientset(testBsl).VeleroV1(), + kotsadmNamespace: "default", + }, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + got, err := FindBackupStoreLocation(ctx, tt.args.clientset, tt.args.veleroClient, tt.args.kotsadmNamespace) + if (err != nil) != tt.wantErr { + t.Errorf("FindBackupStoreLocation() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("FindBackupStoreLocation() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/snapshot/velero.go b/pkg/snapshot/velero.go index 1f0e852952..b92f3bf625 100644 --- a/pkg/snapshot/velero.go +++ b/pkg/snapshot/velero.go @@ -249,11 +249,6 @@ func DetectVeleroNamespace(ctx context.Context, clientset kubernetes.Interface, veleroNamespace = TryGetVeleroNamespaceFromConfigMap(ctx, clientset, kotsadmNamespace) } - clientset, err := k8sutil.GetClientset() - if err != nil { - return "", errors.Wrap(err, "failed to get k8s clientset") - } - deployments, err := clientset.AppsV1().Deployments(veleroNamespace).List(ctx, metav1.ListOptions{}) if kuberneteserrors.IsNotFound(err) { return "", nil