diff --git a/config/crds/troubleshoot.sh_analyzers.yaml b/config/crds/troubleshoot.sh_analyzers.yaml index ae1e62ffe..4d506d1d9 100644 --- a/config/crds/troubleshoot.sh_analyzers.yaml +++ b/config/crds/troubleshoot.sh_analyzers.yaml @@ -1489,6 +1489,58 @@ spec: required: - outcomes type: object + velero: + properties: + annotations: + additionalProperties: + type: string + type: object + checkName: + type: string + collectorName: + type: string + exclude: + type: BoolString + namespace: + type: string + outcomes: + items: + properties: + fail: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + pass: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + warn: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + type: object + type: array + strict: + type: BoolString + required: + - namespace + - outcomes + type: object weaveReport: properties: annotations: diff --git a/config/crds/troubleshoot.sh_preflights.yaml b/config/crds/troubleshoot.sh_preflights.yaml index cd2392f6c..08f1b2252 100644 --- a/config/crds/troubleshoot.sh_preflights.yaml +++ b/config/crds/troubleshoot.sh_preflights.yaml @@ -1489,6 +1489,58 @@ spec: required: - outcomes type: object + velero: + properties: + annotations: + additionalProperties: + type: string + type: object + checkName: + type: string + collectorName: + type: string + exclude: + type: BoolString + namespace: + type: string + outcomes: + items: + properties: + fail: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + pass: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + warn: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + type: object + type: array + strict: + type: BoolString + required: + - namespace + - outcomes + type: object weaveReport: properties: annotations: diff --git a/config/crds/troubleshoot.sh_supportbundles.yaml b/config/crds/troubleshoot.sh_supportbundles.yaml index 861c708b5..e93d157d7 100644 --- a/config/crds/troubleshoot.sh_supportbundles.yaml +++ b/config/crds/troubleshoot.sh_supportbundles.yaml @@ -1520,6 +1520,58 @@ spec: required: - outcomes type: object + velero: + properties: + annotations: + additionalProperties: + type: string + type: object + checkName: + type: string + collectorName: + type: string + exclude: + type: BoolString + namespace: + type: string + outcomes: + items: + properties: + fail: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + pass: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + warn: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + type: object + type: array + strict: + type: BoolString + required: + - namespace + - outcomes + type: object weaveReport: properties: annotations: diff --git a/pkg/analyze/analyzer.go b/pkg/analyze/analyzer.go index 0d6f4e681..a95b12a2c 100644 --- a/pkg/analyze/analyzer.go +++ b/pkg/analyze/analyzer.go @@ -1,16 +1,20 @@ package analyzer import ( + "bufio" + "bytes" "context" "encoding/json" "fmt" "reflect" "strconv" + "strings" "github.com/pkg/errors" troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" "github.com/replicatedhq/troubleshoot/pkg/constants" "github.com/replicatedhq/troubleshoot/pkg/multitype" + "github.com/replicatedhq/troubleshoot/pkg/redact" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" @@ -226,6 +230,8 @@ func getAnalyzer(analyzer *troubleshootv1beta2.Analyze) Analyzer { return &AnalyzeRedis{analyzer: analyzer.Redis} case analyzer.CephStatus != nil: return &AnalyzeCephStatus{analyzer: analyzer.CephStatus} + case analyzer.Velero != nil: + return &AnalyzeVelero{analyzer: analyzer.Velero} case analyzer.Longhorn != nil: return &AnalyzeLonghorn{analyzer: analyzer.Longhorn} case analyzer.RegistryImages != nil: @@ -265,3 +271,18 @@ func DedupAnalyzers(allAnalyzers []*troubleshootv1beta2.Analyze) []*troubleshoot } return finalAnalyzers } + +func stripRedactedLines(yaml []byte) []byte { + buf := bytes.NewBuffer(yaml) + scanner := bufio.NewScanner(buf) + + out := []byte{} + + for scanner.Scan() { + line := strings.ReplaceAll(scanner.Text(), redact.MASK_TEXT, "HIDDEN") + out = append(out, []byte(line)...) + out = append(out, '\n') + } + + return out +} diff --git a/pkg/analyze/longhorn.go b/pkg/analyze/longhorn.go index a9e824452..09e28b2bf 100644 --- a/pkg/analyze/longhorn.go +++ b/pkg/analyze/longhorn.go @@ -1,19 +1,15 @@ package analyzer import ( - "bufio" - "bytes" "fmt" "path/filepath" "reflect" - "strings" "github.com/pkg/errors" troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" "github.com/replicatedhq/troubleshoot/pkg/collect" longhornv1beta1 "github.com/replicatedhq/troubleshoot/pkg/longhorn/apis/longhorn/v1beta1" longhorntypes "github.com/replicatedhq/troubleshoot/pkg/longhorn/types" - "github.com/replicatedhq/troubleshoot/pkg/redact" "gopkg.in/yaml.v2" ) @@ -241,21 +237,6 @@ func analyzeLonghornEngine(engine *longhornv1beta1.Engine) *AnalyzeResult { return result } -func stripRedactedLines(yaml []byte) []byte { - buf := bytes.NewBuffer(yaml) - scanner := bufio.NewScanner(buf) - - out := []byte{} - - for scanner.Scan() { - line := strings.ReplaceAll(scanner.Text(), redact.MASK_TEXT, "HIDDEN") - out = append(out, []byte(line)...) - out = append(out, '\n') - } - - return out -} - func analyzeLonghornReplicaChecksums(volumeName string, checksums []map[string]string) *AnalyzeResult { result := &AnalyzeResult{ Title: fmt.Sprintf("Longhorn Volume Replica Corruption: %s", volumeName), diff --git a/pkg/analyze/velero.go b/pkg/analyze/velero.go new file mode 100644 index 000000000..dc9b7bd5f --- /dev/null +++ b/pkg/analyze/velero.go @@ -0,0 +1,195 @@ +package analyzer + +import ( + "fmt" + "path/filepath" + + "github.com/pkg/errors" + troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" + "github.com/replicatedhq/troubleshoot/pkg/collect" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "gopkg.in/yaml.v2" +) + +type AnalyzeVelero struct { + analyzer *troubleshootv1beta2.VeleroAnalyze +} + +func (a *AnalyzeVelero) Title() string { + title := a.analyzer.CheckName + if title == "" { + title = "Velero" + } + + return title +} + +func (a *AnalyzeVelero) IsExcluded() (bool, error) { + return isExcluded(a.analyzer.Exclude) +} + +func (a *AnalyzeVelero) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) { + results, err := a.veleroStatus(a.analyzer, getFile, findFiles) + if err != nil { + return nil, err + } + for i := range results { + results[i].Strict = a.analyzer.Strict.BoolOrDefaultFalse() + } + return results, nil +} + +func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze, getFileContents getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) { + ns := collect.DefaultVeleroNamespace + if analyzer.Namespace != "" { + ns = analyzer.Namespace + } + + excludeFiles := []string{} + + // get backups.velero.io + backupsDir := collect.GetVeleroBackupsDirectory(ns) + backupsGlob := filepath.Join(backupsDir, "*") + backupsYaml, err := findFiles(backupsGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero backups files under %s", backupsDir) + } + backups := []*velerov1.Backup{} + for key, backupYaml := range backupsYaml { + backup := &velerov1.Backup{} + err := yaml.Unmarshal(backupYaml, backup) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal backup yaml from %s", key) + } + backups = append(backups, backup) + } + // fmt.Printf("\n..found %d backups\n", len(backups)) + + // get backuprepositories.velero.io + backupRpositoriesDir := collect.GetVeleroBackupRepositoriesDirectory(ns) + backupRepositoriesGlob := filepath.Join(backupRpositoriesDir, "*") + backupRepositoriesYaml, err := findFiles(backupRepositoriesGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero backup repositories files under %s", backupRpositoriesDir) + } + backupRepositories := []*velerov1.BackupRepository{} + for key, backupRepositoryYaml := range backupRepositoriesYaml { + backupRepository := &velerov1.BackupRepository{} + err := yaml.Unmarshal(backupRepositoryYaml, backupRepository) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal backup repository yaml from %s", key) + } + backupRepositories = append(backupRepositories, backupRepository) + } + + results := []*AnalyzeResult{} + + results = append(results, analyzeBackups(backups)...) + + // get restores.velero.io + // restoresDir := collect.GetVeleroRestoresDirectory(ns) + + // return print backup files found + // return nil, fmt.Errorf("found %d backups, %d backup repositories", len(backups), len(backupRepositories)) + results = append(results, analyzeBackupRepositories(backupRepositories)...) + + return aggregateResults(results), nil +} + +func analyzeBackups(backups []*velerov1.Backup) []*AnalyzeResult { + results := []*AnalyzeResult{} + + failedPhases := map[velerov1.BackupPhase]bool{ + velerov1.BackupPhaseFailed: true, + velerov1.BackupPhasePartiallyFailed: true, + velerov1.BackupPhaseFailedValidation: true, + velerov1.BackupPhaseFinalizingPartiallyFailed: true, + velerov1.BackupPhaseWaitingForPluginOperationsPartiallyFailed: true, + } + + for _, backup := range backups { + + if failedPhases[backup.Status.Phase] { + result := &AnalyzeResult{ + Title: fmt.Sprintf("Backup %s", backup.Name), + } + result.IsFail = true + // result.Strict = true + result.Message = fmt.Sprintf("Backup %s phase is %s", backup.Name, backup.Status.Phase) + results = append(results, result) + + } + // else if backup.Status.Phase == velerov1.BackupPhaseCompleted { + // result.IsPass = true + // // result.Strict = true + // } else { + // // may indicate phases like: + // // - velerov1.BackupPhaseWaitingForPluginOperations + // // - velerov1.BackupPhaseFinalizing + // result.IsWarn = true + // } + + } + + results = append(results, &AnalyzeResult{ + Title: "Velero Backups count", + IsPass: true, + Message: fmt.Sprintf("Found %d backups", len(backups)), + }) + + return results +} + +func analyzeBackupRepositories(backupRepositories []*velerov1.BackupRepository) []*AnalyzeResult { + + results := []*AnalyzeResult{} + + backupRepositoriesResult := &AnalyzeResult{ + Title: "At least 1 Velero Backup Repository configured", + } + if len(backupRepositories) == 0 { + backupRepositoriesResult.IsFail = true + backupRepositoriesResult.Message = "No backup repositories configured" + } else { + for _, backupRepository := range backupRepositories { + + if backupRepository.Status.Phase == velerov1.BackupRepositoryPhaseNotReady { + result := &AnalyzeResult{ + Title: fmt.Sprintf("Backup Repository %s", backupRepository.Name), + } + result.Message = fmt.Sprintf("Backup Repository [%s] is in phase NotReady", backupRepository.Name) + result.IsWarn = true + results = append(results, result) + // result.Strict = false + } + } + backupRepositoriesResult.IsPass = true + backupRepositoriesResult.Message = fmt.Sprintf("Found %d configured backup repositories", len(backupRepositories)) + } + results = append(results, backupRepositoriesResult) + + return results + +} + +func aggregateResults(results []*AnalyzeResult) []*AnalyzeResult { + out := []*AnalyzeResult{} + resultPass := false + for _, result := range results { + if result.IsPass { + resultPass = true + // continue + } + out = append(out, result) + } + + if resultPass && len(out) == 0 { + out = append(out, &AnalyzeResult{ + Title: "Velero Status", + IsPass: true, + Message: "Backups and CRDs are healthy", + }) + } + + return out +} diff --git a/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go b/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go index 4cb171d30..a206ad7bd 100644 --- a/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go +++ b/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go @@ -186,6 +186,13 @@ type CephStatusAnalyze struct { Namespace string `json:"namespace" yaml:"namespace"` } +type VeleroAnalyze struct { + AnalyzeMeta `json:",inline" yaml:",inline"` + Outcomes []*Outcome `json:"outcomes" yaml:"outcomes"` + CollectorName string `json:"collectorName,omitempty" yaml:"collectorName,omitempty"` + Namespace string `json:"namespace" yaml:"namespace"` +} + type LonghornAnalyze struct { AnalyzeMeta `json:",inline" yaml:",inline"` Outcomes []*Outcome `json:"outcomes" yaml:"outcomes"` @@ -245,6 +252,7 @@ type Analyze struct { Mysql *DatabaseAnalyze `json:"mysql,omitempty" yaml:"mysql,omitempty"` Redis *DatabaseAnalyze `json:"redis,omitempty" yaml:"redis,omitempty"` CephStatus *CephStatusAnalyze `json:"cephStatus,omitempty" yaml:"cephStatus,omitempty"` + Velero *VeleroAnalyze `json:"velero,omitempty" yaml:"velero,omitempty"` Longhorn *LonghornAnalyze `json:"longhorn,omitempty" yaml:"longhorn,omitempty"` RegistryImages *RegistryImagesAnalyze `json:"registryImages,omitempty" yaml:"registryImages,omitempty"` WeaveReport *WeaveReportAnalyze `json:"weaveReport,omitempty" yaml:"weaveReport,omitempty"` diff --git a/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go b/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go index 3c02191de..f113eb652 100644 --- a/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go +++ b/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go @@ -169,6 +169,11 @@ func (in *Analyze) DeepCopyInto(out *Analyze) { *out = new(CephStatusAnalyze) (*in).DeepCopyInto(*out) } + if in.Velero != nil { + in, out := &in.Velero, &out.Velero + *out = new(VeleroAnalyze) + (*in).DeepCopyInto(*out) + } if in.Longhorn != nil { in, out := &in.Longhorn, &out.Longhorn *out = new(LonghornAnalyze) @@ -4675,6 +4680,33 @@ func (in *Velero) DeepCopy() *Velero { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VeleroAnalyze) DeepCopyInto(out *VeleroAnalyze) { + *out = *in + in.AnalyzeMeta.DeepCopyInto(&out.AnalyzeMeta) + if in.Outcomes != nil { + in, out := &in.Outcomes, &out.Outcomes + *out = make([]*Outcome, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Outcome) + (*in).DeepCopyInto(*out) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VeleroAnalyze. +func (in *VeleroAnalyze) DeepCopy() *VeleroAnalyze { + if in == nil { + return nil + } + out := new(VeleroAnalyze) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WeaveReportAnalyze) DeepCopyInto(out *WeaveReportAnalyze) { *out = *in diff --git a/pkg/supportbundle/test/velero.yaml b/pkg/supportbundle/test/velero.yaml index 74551be0f..93c97f568 100644 --- a/pkg/supportbundle/test/velero.yaml +++ b/pkg/supportbundle/test/velero.yaml @@ -5,3 +5,5 @@ metadata: spec: collectors: - velero: {} + analyzers: + - velero: {} \ No newline at end of file diff --git a/schemas/analyzer-troubleshoot-v1beta2.json b/schemas/analyzer-troubleshoot-v1beta2.json index 023101929..c07e25235 100644 --- a/schemas/analyzer-troubleshoot-v1beta2.json +++ b/schemas/analyzer-troubleshoot-v1beta2.json @@ -2255,6 +2255,86 @@ } } }, + "velero": { + "type": "object", + "required": [ + "namespace", + "outcomes" + ], + "properties": { + "annotations": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "checkName": { + "type": "string" + }, + "collectorName": { + "type": "string" + }, + "exclude": { + "oneOf": [{"type": "string"},{"type": "boolean"}] + }, + "namespace": { + "type": "string" + }, + "outcomes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "fail": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + }, + "pass": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + }, + "warn": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + } + } + } + }, + "strict": { + "oneOf": [{"type": "string"},{"type": "boolean"}] + } + } + }, "weaveReport": { "type": "object", "required": [ diff --git a/schemas/preflight-troubleshoot-v1beta2.json b/schemas/preflight-troubleshoot-v1beta2.json index 6c7601863..434247ba8 100644 --- a/schemas/preflight-troubleshoot-v1beta2.json +++ b/schemas/preflight-troubleshoot-v1beta2.json @@ -2255,6 +2255,86 @@ } } }, + "velero": { + "type": "object", + "required": [ + "namespace", + "outcomes" + ], + "properties": { + "annotations": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "checkName": { + "type": "string" + }, + "collectorName": { + "type": "string" + }, + "exclude": { + "oneOf": [{"type": "string"},{"type": "boolean"}] + }, + "namespace": { + "type": "string" + }, + "outcomes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "fail": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + }, + "pass": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + }, + "warn": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + } + } + } + }, + "strict": { + "oneOf": [{"type": "string"},{"type": "boolean"}] + } + } + }, "weaveReport": { "type": "object", "required": [ diff --git a/schemas/supportbundle-troubleshoot-v1beta2.json b/schemas/supportbundle-troubleshoot-v1beta2.json index cf65462c7..44eb75c98 100644 --- a/schemas/supportbundle-troubleshoot-v1beta2.json +++ b/schemas/supportbundle-troubleshoot-v1beta2.json @@ -2301,6 +2301,86 @@ } } }, + "velero": { + "type": "object", + "required": [ + "namespace", + "outcomes" + ], + "properties": { + "annotations": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "checkName": { + "type": "string" + }, + "collectorName": { + "type": "string" + }, + "exclude": { + "oneOf": [{"type": "string"},{"type": "boolean"}] + }, + "namespace": { + "type": "string" + }, + "outcomes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "fail": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + }, + "pass": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + }, + "warn": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + } + } + } + }, + "strict": { + "oneOf": [{"type": "string"},{"type": "boolean"}] + } + } + }, "weaveReport": { "type": "object", "required": [