From e533ec0596b3a9e1678c9c7328a6fa987f95a994 Mon Sep 17 00:00:00 2001 From: Archit Sharma Date: Sun, 8 Oct 2023 20:55:17 +0000 Subject: [PATCH] feat: add velero analyzer (#806) Signed-off-by: Archit Sharma --- pkg/analyze/analyzer.go | 21 ++ pkg/analyze/longhorn.go | 19 -- pkg/analyze/velero.go | 195 ++++++++++++++++++ .../troubleshoot/v1beta2/analyzer_shared.go | 8 + pkg/supportbundle/test/velero.yaml | 2 + 5 files changed, 226 insertions(+), 19 deletions(-) create mode 100644 pkg/analyze/velero.go diff --git a/pkg/analyze/analyzer.go b/pkg/analyze/analyzer.go index 0d6f4e681..a95b12a2c 100644 --- a/pkg/analyze/analyzer.go +++ b/pkg/analyze/analyzer.go @@ -1,16 +1,20 @@ package analyzer import ( + "bufio" + "bytes" "context" "encoding/json" "fmt" "reflect" "strconv" + "strings" "github.com/pkg/errors" troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" "github.com/replicatedhq/troubleshoot/pkg/constants" "github.com/replicatedhq/troubleshoot/pkg/multitype" + "github.com/replicatedhq/troubleshoot/pkg/redact" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" @@ -226,6 +230,8 @@ func getAnalyzer(analyzer *troubleshootv1beta2.Analyze) Analyzer { return &AnalyzeRedis{analyzer: analyzer.Redis} case analyzer.CephStatus != nil: return &AnalyzeCephStatus{analyzer: analyzer.CephStatus} + case analyzer.Velero != nil: + return &AnalyzeVelero{analyzer: analyzer.Velero} case analyzer.Longhorn != nil: return &AnalyzeLonghorn{analyzer: analyzer.Longhorn} case analyzer.RegistryImages != nil: @@ -265,3 +271,18 @@ func DedupAnalyzers(allAnalyzers []*troubleshootv1beta2.Analyze) []*troubleshoot } return finalAnalyzers } + +func stripRedactedLines(yaml []byte) []byte { + buf := bytes.NewBuffer(yaml) + scanner := bufio.NewScanner(buf) + + out := []byte{} + + for scanner.Scan() { + line := strings.ReplaceAll(scanner.Text(), redact.MASK_TEXT, "HIDDEN") + out = append(out, []byte(line)...) + out = append(out, '\n') + } + + return out +} diff --git a/pkg/analyze/longhorn.go b/pkg/analyze/longhorn.go index a9e824452..09e28b2bf 100644 --- a/pkg/analyze/longhorn.go +++ b/pkg/analyze/longhorn.go @@ -1,19 +1,15 @@ package analyzer import ( - "bufio" - "bytes" "fmt" "path/filepath" "reflect" - "strings" "github.com/pkg/errors" troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" "github.com/replicatedhq/troubleshoot/pkg/collect" longhornv1beta1 "github.com/replicatedhq/troubleshoot/pkg/longhorn/apis/longhorn/v1beta1" longhorntypes "github.com/replicatedhq/troubleshoot/pkg/longhorn/types" - "github.com/replicatedhq/troubleshoot/pkg/redact" "gopkg.in/yaml.v2" ) @@ -241,21 +237,6 @@ func analyzeLonghornEngine(engine *longhornv1beta1.Engine) *AnalyzeResult { return result } -func stripRedactedLines(yaml []byte) []byte { - buf := bytes.NewBuffer(yaml) - scanner := bufio.NewScanner(buf) - - out := []byte{} - - for scanner.Scan() { - line := strings.ReplaceAll(scanner.Text(), redact.MASK_TEXT, "HIDDEN") - out = append(out, []byte(line)...) - out = append(out, '\n') - } - - return out -} - func analyzeLonghornReplicaChecksums(volumeName string, checksums []map[string]string) *AnalyzeResult { result := &AnalyzeResult{ Title: fmt.Sprintf("Longhorn Volume Replica Corruption: %s", volumeName), diff --git a/pkg/analyze/velero.go b/pkg/analyze/velero.go new file mode 100644 index 000000000..dc9b7bd5f --- /dev/null +++ b/pkg/analyze/velero.go @@ -0,0 +1,195 @@ +package analyzer + +import ( + "fmt" + "path/filepath" + + "github.com/pkg/errors" + troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" + "github.com/replicatedhq/troubleshoot/pkg/collect" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "gopkg.in/yaml.v2" +) + +type AnalyzeVelero struct { + analyzer *troubleshootv1beta2.VeleroAnalyze +} + +func (a *AnalyzeVelero) Title() string { + title := a.analyzer.CheckName + if title == "" { + title = "Velero" + } + + return title +} + +func (a *AnalyzeVelero) IsExcluded() (bool, error) { + return isExcluded(a.analyzer.Exclude) +} + +func (a *AnalyzeVelero) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) { + results, err := a.veleroStatus(a.analyzer, getFile, findFiles) + if err != nil { + return nil, err + } + for i := range results { + results[i].Strict = a.analyzer.Strict.BoolOrDefaultFalse() + } + return results, nil +} + +func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze, getFileContents getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) { + ns := collect.DefaultVeleroNamespace + if analyzer.Namespace != "" { + ns = analyzer.Namespace + } + + excludeFiles := []string{} + + // get backups.velero.io + backupsDir := collect.GetVeleroBackupsDirectory(ns) + backupsGlob := filepath.Join(backupsDir, "*") + backupsYaml, err := findFiles(backupsGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero backups files under %s", backupsDir) + } + backups := []*velerov1.Backup{} + for key, backupYaml := range backupsYaml { + backup := &velerov1.Backup{} + err := yaml.Unmarshal(backupYaml, backup) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal backup yaml from %s", key) + } + backups = append(backups, backup) + } + // fmt.Printf("\n..found %d backups\n", len(backups)) + + // get backuprepositories.velero.io + backupRpositoriesDir := collect.GetVeleroBackupRepositoriesDirectory(ns) + backupRepositoriesGlob := filepath.Join(backupRpositoriesDir, "*") + backupRepositoriesYaml, err := findFiles(backupRepositoriesGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero backup repositories files under %s", backupRpositoriesDir) + } + backupRepositories := []*velerov1.BackupRepository{} + for key, backupRepositoryYaml := range backupRepositoriesYaml { + backupRepository := &velerov1.BackupRepository{} + err := yaml.Unmarshal(backupRepositoryYaml, backupRepository) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal backup repository yaml from %s", key) + } + backupRepositories = append(backupRepositories, backupRepository) + } + + results := []*AnalyzeResult{} + + results = append(results, analyzeBackups(backups)...) + + // get restores.velero.io + // restoresDir := collect.GetVeleroRestoresDirectory(ns) + + // return print backup files found + // return nil, fmt.Errorf("found %d backups, %d backup repositories", len(backups), len(backupRepositories)) + results = append(results, analyzeBackupRepositories(backupRepositories)...) + + return aggregateResults(results), nil +} + +func analyzeBackups(backups []*velerov1.Backup) []*AnalyzeResult { + results := []*AnalyzeResult{} + + failedPhases := map[velerov1.BackupPhase]bool{ + velerov1.BackupPhaseFailed: true, + velerov1.BackupPhasePartiallyFailed: true, + velerov1.BackupPhaseFailedValidation: true, + velerov1.BackupPhaseFinalizingPartiallyFailed: true, + velerov1.BackupPhaseWaitingForPluginOperationsPartiallyFailed: true, + } + + for _, backup := range backups { + + if failedPhases[backup.Status.Phase] { + result := &AnalyzeResult{ + Title: fmt.Sprintf("Backup %s", backup.Name), + } + result.IsFail = true + // result.Strict = true + result.Message = fmt.Sprintf("Backup %s phase is %s", backup.Name, backup.Status.Phase) + results = append(results, result) + + } + // else if backup.Status.Phase == velerov1.BackupPhaseCompleted { + // result.IsPass = true + // // result.Strict = true + // } else { + // // may indicate phases like: + // // - velerov1.BackupPhaseWaitingForPluginOperations + // // - velerov1.BackupPhaseFinalizing + // result.IsWarn = true + // } + + } + + results = append(results, &AnalyzeResult{ + Title: "Velero Backups count", + IsPass: true, + Message: fmt.Sprintf("Found %d backups", len(backups)), + }) + + return results +} + +func analyzeBackupRepositories(backupRepositories []*velerov1.BackupRepository) []*AnalyzeResult { + + results := []*AnalyzeResult{} + + backupRepositoriesResult := &AnalyzeResult{ + Title: "At least 1 Velero Backup Repository configured", + } + if len(backupRepositories) == 0 { + backupRepositoriesResult.IsFail = true + backupRepositoriesResult.Message = "No backup repositories configured" + } else { + for _, backupRepository := range backupRepositories { + + if backupRepository.Status.Phase == velerov1.BackupRepositoryPhaseNotReady { + result := &AnalyzeResult{ + Title: fmt.Sprintf("Backup Repository %s", backupRepository.Name), + } + result.Message = fmt.Sprintf("Backup Repository [%s] is in phase NotReady", backupRepository.Name) + result.IsWarn = true + results = append(results, result) + // result.Strict = false + } + } + backupRepositoriesResult.IsPass = true + backupRepositoriesResult.Message = fmt.Sprintf("Found %d configured backup repositories", len(backupRepositories)) + } + results = append(results, backupRepositoriesResult) + + return results + +} + +func aggregateResults(results []*AnalyzeResult) []*AnalyzeResult { + out := []*AnalyzeResult{} + resultPass := false + for _, result := range results { + if result.IsPass { + resultPass = true + // continue + } + out = append(out, result) + } + + if resultPass && len(out) == 0 { + out = append(out, &AnalyzeResult{ + Title: "Velero Status", + IsPass: true, + Message: "Backups and CRDs are healthy", + }) + } + + return out +} diff --git a/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go b/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go index 4cb171d30..a206ad7bd 100644 --- a/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go +++ b/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go @@ -186,6 +186,13 @@ type CephStatusAnalyze struct { Namespace string `json:"namespace" yaml:"namespace"` } +type VeleroAnalyze struct { + AnalyzeMeta `json:",inline" yaml:",inline"` + Outcomes []*Outcome `json:"outcomes" yaml:"outcomes"` + CollectorName string `json:"collectorName,omitempty" yaml:"collectorName,omitempty"` + Namespace string `json:"namespace" yaml:"namespace"` +} + type LonghornAnalyze struct { AnalyzeMeta `json:",inline" yaml:",inline"` Outcomes []*Outcome `json:"outcomes" yaml:"outcomes"` @@ -245,6 +252,7 @@ type Analyze struct { Mysql *DatabaseAnalyze `json:"mysql,omitempty" yaml:"mysql,omitempty"` Redis *DatabaseAnalyze `json:"redis,omitempty" yaml:"redis,omitempty"` CephStatus *CephStatusAnalyze `json:"cephStatus,omitempty" yaml:"cephStatus,omitempty"` + Velero *VeleroAnalyze `json:"velero,omitempty" yaml:"velero,omitempty"` Longhorn *LonghornAnalyze `json:"longhorn,omitempty" yaml:"longhorn,omitempty"` RegistryImages *RegistryImagesAnalyze `json:"registryImages,omitempty" yaml:"registryImages,omitempty"` WeaveReport *WeaveReportAnalyze `json:"weaveReport,omitempty" yaml:"weaveReport,omitempty"` diff --git a/pkg/supportbundle/test/velero.yaml b/pkg/supportbundle/test/velero.yaml index 74551be0f..93c97f568 100644 --- a/pkg/supportbundle/test/velero.yaml +++ b/pkg/supportbundle/test/velero.yaml @@ -5,3 +5,5 @@ metadata: spec: collectors: - velero: {} + analyzers: + - velero: {} \ No newline at end of file