From bb2ca4d2fb4a84268c5a7b4ce6c6b3c37a399391 Mon Sep 17 00:00:00 2001 From: Archit Sharma Date: Sun, 8 Oct 2023 20:55:17 +0000 Subject: [PATCH 1/9] feat: add velero analyzer (#806) * updated schemas * velero analyzer without collector Signed-off-by: Archit Sharma --- config/crds/troubleshoot.sh_analyzers.yaml | 52 ++ config/crds/troubleshoot.sh_preflights.yaml | 52 ++ .../crds/troubleshoot.sh_supportbundles.yaml | 52 ++ go.mod | 4 +- go.sum | 21 +- pkg/analyze/analyzer.go | 21 + pkg/analyze/longhorn.go | 19 - pkg/analyze/velero.go | 733 ++++++++++++++++++ .../troubleshoot/v1beta2/analyzer_shared.go | 8 + .../v1beta2/zz_generated.deepcopy.go | 32 + pkg/supportbundle/test/velero.yaml | 12 + schemas/analyzer-troubleshoot-v1beta2.json | 80 ++ schemas/preflight-troubleshoot-v1beta2.json | 80 ++ .../supportbundle-troubleshoot-v1beta2.json | 80 ++ 14 files changed, 1211 insertions(+), 35 deletions(-) create mode 100644 pkg/analyze/velero.go create mode 100644 pkg/supportbundle/test/velero.yaml diff --git a/config/crds/troubleshoot.sh_analyzers.yaml b/config/crds/troubleshoot.sh_analyzers.yaml index ae1e62ffe..4d506d1d9 100644 --- a/config/crds/troubleshoot.sh_analyzers.yaml +++ b/config/crds/troubleshoot.sh_analyzers.yaml @@ -1489,6 +1489,58 @@ spec: required: - outcomes type: object + velero: + properties: + annotations: + additionalProperties: + type: string + type: object + checkName: + type: string + collectorName: + type: string + exclude: + type: BoolString + namespace: + type: string + outcomes: + items: + properties: + fail: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + pass: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + warn: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + type: object + type: array + strict: + type: BoolString + required: + - namespace + - outcomes + type: object weaveReport: properties: annotations: diff --git a/config/crds/troubleshoot.sh_preflights.yaml b/config/crds/troubleshoot.sh_preflights.yaml index fbb0d7b4b..7b020c4c0 100644 --- a/config/crds/troubleshoot.sh_preflights.yaml +++ b/config/crds/troubleshoot.sh_preflights.yaml @@ -1489,6 +1489,58 @@ spec: required: - outcomes type: object + velero: + properties: + annotations: + additionalProperties: + type: string + type: object + checkName: + type: string + collectorName: + type: string + exclude: + type: BoolString + namespace: + type: string + outcomes: + items: + properties: + fail: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + pass: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + warn: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + type: object + type: array + strict: + type: BoolString + required: + - namespace + - outcomes + type: object weaveReport: properties: annotations: diff --git a/config/crds/troubleshoot.sh_supportbundles.yaml b/config/crds/troubleshoot.sh_supportbundles.yaml index c938845ae..efc6a70e3 100644 --- a/config/crds/troubleshoot.sh_supportbundles.yaml +++ b/config/crds/troubleshoot.sh_supportbundles.yaml @@ -1520,6 +1520,58 @@ spec: required: - outcomes type: object + velero: + properties: + annotations: + additionalProperties: + type: string + type: object + checkName: + type: string + collectorName: + type: string + exclude: + type: BoolString + namespace: + type: string + outcomes: + items: + properties: + fail: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + pass: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + warn: + properties: + message: + type: string + uri: + type: string + when: + type: string + type: object + type: object + type: array + strict: + type: BoolString + required: + - namespace + - outcomes + type: object weaveReport: properties: annotations: diff --git a/go.mod b/go.mod index 5eb76f66b..19f4c9d22 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/spf13/viper v1.17.0 github.com/stretchr/testify v1.8.4 github.com/tj/go-spin v1.1.0 + github.com/vmware-tanzu/velero v1.12.0 go.opentelemetry.io/otel v1.19.0 go.opentelemetry.io/otel/sdk v1.19.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 @@ -86,7 +87,6 @@ require ( github.com/mistifyio/go-zfs/v3 v3.0.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/onsi/ginkgo v1.14.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/rubenv/sql-migrate v1.3.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -123,7 +123,7 @@ require ( github.com/Microsoft/go-winio v0.6.0 // indirect github.com/Microsoft/hcsshim v0.10.0-rc.7 // indirect github.com/andybalholm/brotli v1.0.1 // indirect - github.com/aws/aws-sdk-go v1.44.198 // indirect + github.com/aws/aws-sdk-go v1.44.253 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/c9s/goprocinfo v0.0.0-20170724085704-0010a05ce49f // indirect diff --git a/go.sum b/go.sum index 180050de6..79bdb7d99 100644 --- a/go.sum +++ b/go.sum @@ -192,7 +192,7 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774 h1:SCbEWT58NSt7d2mcFdvxC9uyrdcTfvBbPLThhkDmXzg= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= -github.com/Azure/azure-sdk-for-go v56.3.0+incompatible h1:DmhwMrUIvpeoTDiWRDtNHqelNUd3Og8JCkrLHQK795c= +github.com/Azure/azure-sdk-for-go v67.2.0+incompatible h1:Uu/Ww6ernvPTrpq31kITVTIm/I5jlJ1wjtEH/bmSB2k= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 h1:/iHxaJhsFr0+xVFfbMr5vxz848jyiWuIEDhYq3y5odY= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= @@ -244,8 +244,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.44.198 h1:kgnvxQv4/kP5M0nbxBx0Ac0so9ndr9f8Ti0g+NmPQF8= -github.com/aws/aws-sdk-go v1.44.198/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.253 h1:iqDd0okcH4ShfFexz2zzf4VmeDFf6NOMm07pHnEb8iY= +github.com/aws/aws-sdk-go v1.44.253/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -800,19 +800,14 @@ github.com/nsf/termbox-go v0.0.0-20190121233118-02980233997d/go.mod h1:IuKpRQcYE github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nwaples/rardecode v1.1.2 h1:Cj0yZY6T1Zx1R7AhTbyGSALm44/Mmq+BAPc4B/p/d3M= github.com/nwaples/rardecode v1.1.2/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -1001,6 +996,8 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vladimirvivien/gexe v0.2.0 h1:nbdAQ6vbZ+ZNsolCgSVb9Fno60kzSuvtzVh6Ytqi/xY= github.com/vladimirvivien/gexe v0.2.0/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w= +github.com/vmware-tanzu/velero v1.12.0 h1:gN8PbQMYOAMv8OYE3RkIvxr6s6IoMS0Daxc+IQN0X5U= +github.com/vmware-tanzu/velero v1.12.0/go.mod h1:rY2UfdC2K9je9jtjnSBsZr8Zmg8hzePjG2W00Oe/CT4= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -1149,7 +1146,6 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -1246,14 +1242,12 @@ golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1268,7 +1262,6 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/analyze/analyzer.go b/pkg/analyze/analyzer.go index 0d6f4e681..a95b12a2c 100644 --- a/pkg/analyze/analyzer.go +++ b/pkg/analyze/analyzer.go @@ -1,16 +1,20 @@ package analyzer import ( + "bufio" + "bytes" "context" "encoding/json" "fmt" "reflect" "strconv" + "strings" "github.com/pkg/errors" troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" "github.com/replicatedhq/troubleshoot/pkg/constants" "github.com/replicatedhq/troubleshoot/pkg/multitype" + "github.com/replicatedhq/troubleshoot/pkg/redact" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" @@ -226,6 +230,8 @@ func getAnalyzer(analyzer *troubleshootv1beta2.Analyze) Analyzer { return &AnalyzeRedis{analyzer: analyzer.Redis} case analyzer.CephStatus != nil: return &AnalyzeCephStatus{analyzer: analyzer.CephStatus} + case analyzer.Velero != nil: + return &AnalyzeVelero{analyzer: analyzer.Velero} case analyzer.Longhorn != nil: return &AnalyzeLonghorn{analyzer: analyzer.Longhorn} case analyzer.RegistryImages != nil: @@ -265,3 +271,18 @@ func DedupAnalyzers(allAnalyzers []*troubleshootv1beta2.Analyze) []*troubleshoot } return finalAnalyzers } + +func stripRedactedLines(yaml []byte) []byte { + buf := bytes.NewBuffer(yaml) + scanner := bufio.NewScanner(buf) + + out := []byte{} + + for scanner.Scan() { + line := strings.ReplaceAll(scanner.Text(), redact.MASK_TEXT, "HIDDEN") + out = append(out, []byte(line)...) + out = append(out, '\n') + } + + return out +} diff --git a/pkg/analyze/longhorn.go b/pkg/analyze/longhorn.go index a9e824452..09e28b2bf 100644 --- a/pkg/analyze/longhorn.go +++ b/pkg/analyze/longhorn.go @@ -1,19 +1,15 @@ package analyzer import ( - "bufio" - "bytes" "fmt" "path/filepath" "reflect" - "strings" "github.com/pkg/errors" troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" "github.com/replicatedhq/troubleshoot/pkg/collect" longhornv1beta1 "github.com/replicatedhq/troubleshoot/pkg/longhorn/apis/longhorn/v1beta1" longhorntypes "github.com/replicatedhq/troubleshoot/pkg/longhorn/types" - "github.com/replicatedhq/troubleshoot/pkg/redact" "gopkg.in/yaml.v2" ) @@ -241,21 +237,6 @@ func analyzeLonghornEngine(engine *longhornv1beta1.Engine) *AnalyzeResult { return result } -func stripRedactedLines(yaml []byte) []byte { - buf := bytes.NewBuffer(yaml) - scanner := bufio.NewScanner(buf) - - out := []byte{} - - for scanner.Scan() { - line := strings.ReplaceAll(scanner.Text(), redact.MASK_TEXT, "HIDDEN") - out = append(out, []byte(line)...) - out = append(out, '\n') - } - - return out -} - func analyzeLonghornReplicaChecksums(volumeName string, checksums []map[string]string) *AnalyzeResult { result := &AnalyzeResult{ Title: fmt.Sprintf("Longhorn Volume Replica Corruption: %s", volumeName), diff --git a/pkg/analyze/velero.go b/pkg/analyze/velero.go new file mode 100644 index 000000000..01b6dd12e --- /dev/null +++ b/pkg/analyze/velero.go @@ -0,0 +1,733 @@ +package analyzer + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" + + "github.com/pkg/errors" + troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + // velerov1beta1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1beta1" +) + +const ( + DefaultVeleroNamespace = "velero" +) + +type AnalyzeVelero struct { + analyzer *troubleshootv1beta2.VeleroAnalyze +} + +func (a *AnalyzeVelero) Title() string { + title := a.analyzer.CheckName + if title == "" { + title = "Velero" + } + + return title +} + +func (a *AnalyzeVelero) IsExcluded() (bool, error) { + return isExcluded(a.analyzer.Exclude) +} + +func (a *AnalyzeVelero) Analyze(getFile getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) { + results, err := a.veleroStatus(a.analyzer, getFile, findFiles) + if err != nil { + return nil, err + } + for i := range results { + results[i].Strict = a.analyzer.Strict.BoolOrDefaultFalse() + } + return results, nil +} + +func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze, getFileContents getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) { + ns := DefaultVeleroNamespace + if analyzer.Namespace != "" { + ns = analyzer.Namespace + } + + excludeFiles := []string{} + + // get backuprepositories.velero.io + backupRepositoriesDir := GetVeleroBackupRepositoriesDirectory(ns) + fmt.Println(backupRepositoriesDir) + backupRepositoriesGlob := filepath.Join(backupRepositoriesDir, "*.json") + backupRepositoriesJson, err := findFiles(backupRepositoriesGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero backup repositories files under %s", backupRepositoriesDir) + } + backupRepositories := []*velerov1.BackupRepository{} + for key, backupRepositoryJson := range backupRepositoriesJson { + var backupRepositoryArray []*velerov1.BackupRepository + // backupRepository := &velerov1.BackupRepository{} + err := json.Unmarshal(backupRepositoryJson, &backupRepositoryArray) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal backup repository json from %s", key) + } + backupRepositories = append(backupRepositories, backupRepositoryArray...) + } + // print len of backupRepositories + fmt.Printf("\n..found %d velero backup repositories\n", len(backupRepositories)) + + // get backups.velero.io + backupsDir := GetVeleroBackupsDirectory(ns) + backupsGlob := filepath.Join(backupsDir, "*.json") + fmt.Println(backupsDir) + veleroJSONs, err := findFiles(backupsGlob, excludeFiles) + // print files for debugging + fmt.Printf("\n..found %d velero backup jsons\n", len(veleroJSONs)) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero backup files") + } + backups := []*velerov1.Backup{} + for _, veleroJSON := range veleroJSONs { + if err != nil { + return nil, errors.Wrapf(err, "failed to read velero backup file %s", veleroJSON) + } + var veleroBackups []*velerov1.Backup + err = json.Unmarshal(veleroJSON, &veleroBackups) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal velero backup file %s", veleroJSON) + } + backups = append(backups, veleroBackups...) + } + fmt.Printf("\n..found %d backups\n", len(backups)) + + // // velerov1.BackupRepositoryTypeRestic + // // // get resticrepositories.velero.io + // resticRepositoriesDir := GetVeleroResticRepositoriesDirectory(ns) + // resticRepositoriesGlob := filepath.Join(resticRepositoriesDir, "*.json") + // resticRepositoriesYaml, err := findFiles(resticRepositoriesGlob, excludeFiles) + // if err != nil { + // return nil, errors.Wrapf(err, "failed to find velero restic repositories files under %s", resticRepositoriesDir) + // } + // resticRepositories := []*velerov1beta1.ResticRepository{} + // for key, resticRepositoryYaml := range resticRepositoriesYaml { + // resticRepository := &velerov1beta1.ResticRepository{} + // err := json.Unmarshal(resticRepositoryYaml, resticRepository) + // if err != nil { + // return nil, errors.Wrapf(err, "failed to unmarshal restic repository json from %s", key) + // } + // resticRepositories = append(resticRepositories, resticRepository) + // } + + // get backupstoragelocations.velero.io + backupStorageLocationsDir := GetVeleroBackupStorageLocationsDirectory(ns) + backupStorageLocationsGlob := filepath.Join(backupStorageLocationsDir, "*.json") + backupStorageLocationsJson, err := findFiles(backupStorageLocationsGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero backup storage locations files under %s", backupStorageLocationsDir) + } + backupStorageLocations := []*velerov1.BackupStorageLocation{} + for key, backupStorageLocationYaml := range backupStorageLocationsJson { + var backupStorageLocationArray []*velerov1.BackupStorageLocation + err := json.Unmarshal(backupStorageLocationYaml, &backupStorageLocationArray) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal backup storage location json from %s", key) + } + backupStorageLocations = append(backupStorageLocations, backupStorageLocationArray...) + } + fmt.Printf("\n..found %d velero backup storage locations\n", len(backupStorageLocations)) + + // get deletebackuprequests.velero.io + deleteBackupRequestsDir := GetVeleroDeleteBackupRequestsDirectory(ns) + deleteBackupRequestsGlob := filepath.Join(deleteBackupRequestsDir, "*.json") + deleteBackupRequestsJson, err := findFiles(deleteBackupRequestsGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero delete backup requests files under %s", deleteBackupRequestsDir) + } + deleteBackupRequests := []*velerov1.DeleteBackupRequest{} + for key, deleteBackupRequestYaml := range deleteBackupRequestsJson { + var deleteBackupRequestArray []*velerov1.DeleteBackupRequest + err := json.Unmarshal(deleteBackupRequestYaml, &deleteBackupRequestArray) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal delete backup request json from %s", key) + } + deleteBackupRequests = append(deleteBackupRequests, deleteBackupRequestArray...) + } + fmt.Printf("\n..found %d velero delete backup requests\n", len(deleteBackupRequests)) + + // get downloadrequests.velero.io + // downloadRequestsDir := GetVeleroDownloadRequestsDirectory(ns) + // downloadRequestsGlob := filepath.Join(downloadRequestsDir, "*.json") + // downloadRequestsYaml, err := findFiles(downloadRequestsGlob, excludeFiles) + // if err != nil { + // return nil, errors.Wrapf(err, "failed to find velero download requests files under %s", downloadRequestsDir) + // } + // downloadRequests := []*velerov1.DownloadRequest{} + // for key, downloadRequestYaml := range downloadRequestsYaml { + // downloadRequest := &velerov1.DownloadRequest{} + // err := json.Unmarshal(downloadRequestYaml, downloadRequest) + // if err != nil { + // return nil, errors.Wrapf(err, "failed to unmarshal download request json from %s", key) + // } + // downloadRequests = append(downloadRequests, downloadRequest) + // } + + // get podvolumebackups.velero.io + podVolumeBackupsDir := GetVeleroPodVolumeBackupsDirectory(ns) + podVolumeBackupsGlob := filepath.Join(podVolumeBackupsDir, "*.json") + podVolumeBackupsJson, err := findFiles(podVolumeBackupsGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero pod volume backups files under %s", podVolumeBackupsDir) + } + podVolumeBackups := []*velerov1.PodVolumeBackup{} + for key, podVolumeBackupJson := range podVolumeBackupsJson { + var podVolumeBackupArray []*velerov1.PodVolumeBackup + err := json.Unmarshal(podVolumeBackupJson, &podVolumeBackupArray) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal pod volume backup json from %s", key) + } + podVolumeBackups = append(podVolumeBackups, podVolumeBackupArray...) + } + + // get podvolumerestores.velero.io + podVolumeRestoresDir := GetVeleroPodVolumeRestoresDirectory(ns) + podVolumeRestoresGlob := filepath.Join(podVolumeRestoresDir, "*.json") + podVolumeRestoresJson, err := findFiles(podVolumeRestoresGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero pod volume restores files under %s", podVolumeRestoresDir) + } + podVolumeRestores := []*velerov1.PodVolumeRestore{} + for key, podVolumeRestoreYaml := range podVolumeRestoresJson { + var podVolumeRestoreArray []*velerov1.PodVolumeRestore + err := json.Unmarshal(podVolumeRestoreYaml, &podVolumeRestoreArray) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal pod volume restore json from %s", key) + } + podVolumeRestores = append(podVolumeRestores, podVolumeRestoreArray...) + } + fmt.Println("FIRST----------") + + // get restores.velero.io + restoresDir := GetVeleroRestoresDirectory(ns) + restoresGlob := filepath.Join(restoresDir, "*.json") + restoresJson, err := findFiles(restoresGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero restores files under %s", restoresDir) + } + restores := []*velerov1.Restore{} + for key, restoreYaml := range restoresJson { + var restoreArray []*velerov1.Restore + err := json.Unmarshal(restoreYaml, &restoreArray) + if err != nil { + fmt.Println("ERROR----------", key, err) + return nil, errors.Wrapf(err, "failed to unmarshal restore json from %s", key) + } + restores = append(restores, restoreArray...) + } + fmt.Println("THIRD----------") + + // get schedules.velero.io + schedulesDir := GetVeleroSchedulesDirectory(ns) + schedulesGlob := filepath.Join(schedulesDir, "*.json") + schedulesYaml, err := findFiles(schedulesGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero schedules files under %s", schedulesDir) + } + schedules := []*velerov1.Schedule{} + for key, scheduleYaml := range schedulesYaml { + var scheduleArray []*velerov1.Schedule + err := json.Unmarshal(scheduleYaml, &scheduleArray) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal schedule json from %s", key) + } + schedules = append(schedules, scheduleArray...) + } + + // get serverstatusrequests.velero.io + serverStatusRequestsDir := GetVeleroServerStatusRequestsDirectory(ns) + serverStatusRequestsGlob := filepath.Join(serverStatusRequestsDir, "*.json") + serverStatusRequestsYaml, err := findFiles(serverStatusRequestsGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero server status requests files under %s", serverStatusRequestsDir) + } + serverStatusRequests := []*velerov1.ServerStatusRequest{} + for key, serverStatusRequestYaml := range serverStatusRequestsYaml { + var serverStatusRequestArray []*velerov1.ServerStatusRequest + err := json.Unmarshal(serverStatusRequestYaml, &serverStatusRequestArray) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal server status request json from %s", key) + } + serverStatusRequests = append(serverStatusRequests, serverStatusRequestArray...) + } + + // get volumesnapshotlocations.velero.io + volumeSnapshotLocationsDir := GetVeleroVolumeSnapshotLocationsDirectory(ns) + volumeSnapshotLocationsGlob := filepath.Join(volumeSnapshotLocationsDir, "*.json") + volumeSnapshotLocationsYaml, err := findFiles(volumeSnapshotLocationsGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero volume snapshot locations files under %s", volumeSnapshotLocationsDir) + } + volumeSnapshotLocations := []*velerov1.VolumeSnapshotLocation{} + for key, volumeSnapshotLocationYaml := range volumeSnapshotLocationsYaml { + var volumeSnapshotLocationArray []*velerov1.VolumeSnapshotLocation + err := json.Unmarshal(volumeSnapshotLocationYaml, &volumeSnapshotLocationArray) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal volume snapshot location json from %s", key) + } + volumeSnapshotLocations = append(volumeSnapshotLocations, volumeSnapshotLocationArray...) + } + + logsDir := GetVeleroLogsDirectory(ns) + fmt.Println(logsDir) + logsGlob := filepath.Join(logsDir, "node-agent*", "*.log") + logs, err := findFiles(logsGlob, excludeFiles) + + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero logs files under %s", logsDir) + } + + results := []*AnalyzeResult{} + results = append(results, analyzeLogs(logs)...) + results = append(results, analyzeBackupRepositories(backupRepositories)...) + results = append(results, analyzeBackups(backups)...) + // results = append(results, analyzeResticRepositories(resticRepositories)...) + results = append(results, analyzeBackupStorageLocations(backupStorageLocations)...) + results = append(results, analyzeDeleteBackupRequests(deleteBackupRequests)...) + // results = append(results, analyzeDownloadRequests(downloadRequests)...) + results = append(results, analyzePodVolumeBackups(podVolumeBackups)...) + results = append(results, analyzePodVolumeRestores(podVolumeRestores)...) + results = append(results, analyzeRestores(restores)...) + results = append(results, analyzeSchedules(schedules)...) + results = append(results, analyzeServerStatusRequests(serverStatusRequests)...) + results = append(results, analyzeVolumeSnapshotLocations(volumeSnapshotLocations)...) + + return aggregateResults(results), nil +} + +func analyzeBackupRepositories(backupRepositories []*velerov1.BackupRepository) []*AnalyzeResult { + results := []*AnalyzeResult{} + readyCount := 0 + backupRepositoriesResult := &AnalyzeResult{ + Title: "At least 1 Velero Backup Repository configured", + } + if len(backupRepositories) == 0 { + backupRepositoriesResult.IsFail = true + backupRepositoriesResult.Message = "No backup repositories configured" + } else { + for _, backupRepository := range backupRepositories { + if backupRepository.Status.Phase != velerov1.BackupRepositoryPhaseReady { + result := &AnalyzeResult{ + Title: fmt.Sprintf("Backup Repository %s", backupRepository.Name), + } + result.Message = fmt.Sprintf("Backup Repository [%s] is in phase %s", backupRepository.Name, backupRepository.Status.Phase) + result.IsWarn = true + results = append(results, result) + } else { + readyCount++ + } + } + if readyCount > 0 { + backupRepositoriesResult.IsPass = true + backupRepositoriesResult.Message = fmt.Sprintf("Found %d backup repositories configured and %d Ready", len(backupRepositories), readyCount) + } else { + backupRepositoriesResult.IsWarn = true + backupRepositoriesResult.Message = fmt.Sprintf("Found %d configured backup repositories, but none are ready", len(backupRepositories)) + } + } + results = append(results, backupRepositoriesResult) + + return results + +} + +func analyzeBackups(backups []*velerov1.Backup) []*AnalyzeResult { + results := []*AnalyzeResult{} + + failedPhases := map[velerov1.BackupPhase]bool{ + velerov1.BackupPhaseFailed: true, + velerov1.BackupPhasePartiallyFailed: true, + velerov1.BackupPhaseFailedValidation: true, + velerov1.BackupPhaseFinalizingPartiallyFailed: true, + velerov1.BackupPhaseWaitingForPluginOperationsPartiallyFailed: true, + } + + for _, backup := range backups { + + if failedPhases[backup.Status.Phase] { + result := &AnalyzeResult{ + Title: fmt.Sprintf("Backup %s", backup.Name), + } + result.IsFail = true + result.Message = fmt.Sprintf("Backup %s phase is %s", backup.Name, backup.Status.Phase) + results = append(results, result) + + } + } + if len(backups) > 0 { + results = append(results, &AnalyzeResult{ + Title: "Velero Backups", + IsPass: true, + Message: fmt.Sprintf("Found %d backups", len(backups)), + }) + } + return results +} + +func analyzeBackupStorageLocations(backupStorageLocations []*velerov1.BackupStorageLocation) []*AnalyzeResult { + results := []*AnalyzeResult{} + // atleast 1 backup storage location Phase Available + availableCount := 0 + bslResult := &AnalyzeResult{ + Title: "At least 1 Velero Backup Storage Location configured", + } + + if len(backupStorageLocations) == 0 { + bslResult.IsFail = true + bslResult.Message = "No backup storage locations configured" + } else { + for _, backupStorageLocation := range backupStorageLocations { + if backupStorageLocation.Status.Phase != velerov1.BackupStorageLocationPhaseAvailable { + result := &AnalyzeResult{ + Title: fmt.Sprintf("Backup Storage Location %s", backupStorageLocation.Name), + } + result.Message = fmt.Sprintf("Backup Storage Location [%s] is in phase %s", backupStorageLocation.Name, backupStorageLocation.Status.Phase) + result.IsWarn = true + results = append(results, result) + // result.Strict = false + } else { + availableCount++ + } + } + if availableCount > 0 { + bslResult.IsPass = true + bslResult.Message = fmt.Sprintf("Found %d backup storage locations configured and %d Available", len(backupStorageLocations), availableCount) + } else { + bslResult.IsWarn = true + bslResult.Message = fmt.Sprintf("Found %d configured backup storage locations, but none are available", len(backupStorageLocations)) + } + } + results = append(results, bslResult) + + return results +} + +func analyzeDeleteBackupRequests(deleteBackupRequests []*velerov1.DeleteBackupRequest) []*AnalyzeResult { + results := []*AnalyzeResult{} + // all in progress, new and processed + inProgressCount := 0 + if len(deleteBackupRequests) > 0 { + for _, deleteBackupRequest := range deleteBackupRequests { + if deleteBackupRequest.Status.Phase == velerov1.DeleteBackupRequestPhaseInProgress { + inProgressCount++ + } + } + if inProgressCount > 0 { + deleteBackupRequestsResult := &AnalyzeResult{ + Title: "Delete Backup Requests summary", + } + deleteBackupRequestsResult.IsWarn = true + deleteBackupRequestsResult.Message = fmt.Sprintf("Found %d delete backup requests in progress", inProgressCount) + results = append(results, deleteBackupRequestsResult) + } + } + + return results +} + +// func analyzeDownloadRequests(downloadRequests []*velerov1.DownloadRequest) []*AnalyzeResult { +// results := []*AnalyzeResult{} +// // all +// processedCount := 0 +// newCount := 0 +// if len(downloadRequests) > 0 { +// for _, downloadRequest := range downloadRequests { +// if downloadRequest.Status.Phase == velerov1.DownloadRequestPhaseProcessed { +// processedCount++ +// } else if downloadRequest.Status.Phase == velerov1.DownloadRequestPhaseNew { +// newCount++ +// } +// } +// if processedCount > 0 || newCount > 0 { +// downloadRequestsResult := &AnalyzeResult{ +// Title: "Download Requests summary", +// } +// downloadRequestsResult.IsPass = true +// downloadRequestsResult.Message = fmt.Sprintf("Found %d processed and %d new download requests", processedCount, newCount) +// results = append(results, downloadRequestsResult) +// } +// } + +// return results +// } + +func analyzePodVolumeBackups(podVolumeBackups []*velerov1.PodVolumeBackup) []*AnalyzeResult { + results := []*AnalyzeResult{} + failures := 0 + // isFail if any pod volume backup phase is Failed + if len(podVolumeBackups) > 0 { + // look for PodVolumeBackupPhaseFailed (only 1) + for _, podVolumeBackup := range podVolumeBackups { + if podVolumeBackup.Status.Phase == velerov1.PodVolumeBackupPhaseFailed { + result := &AnalyzeResult{ + Title: fmt.Sprintf("Pod Volume Backup %s", podVolumeBackup.Name), + } + result.IsFail = true + // result.Strict = true + result.Message = fmt.Sprintf("Pod Volume Backup %s phase is %s", podVolumeBackup.Name, podVolumeBackup.Status.Phase) + results = append(results, result) + failures++ + } + } + + if failures == 0 { + results = append(results, &AnalyzeResult{ + Title: "Pod Volume Backups", + IsPass: true, + Message: fmt.Sprintf("Found %d pod volume backups", len(podVolumeBackups)), + }) + } + } + + return results +} + +func analyzePodVolumeRestores(podVolumeRestores []*velerov1.PodVolumeRestore) []*AnalyzeResult { + results := []*AnalyzeResult{} + failures := 0 + + if len(podVolumeRestores) > 0 { + for _, podVolumeRestore := range podVolumeRestores { + if podVolumeRestore.Status.Phase == velerov1.PodVolumeRestorePhaseFailed { + result := &AnalyzeResult{ + Title: fmt.Sprintf("Pod Volume Restore %s", podVolumeRestore.Name), + } + result.IsFail = true + // result.Strict = true + result.Message = fmt.Sprintf("Pod Volume Restore %s phase is %s", podVolumeRestore.Name, podVolumeRestore.Status.Phase) + results = append(results, result) + failures++ + } + } + if failures == 0 { + results = append(results, &AnalyzeResult{ + Title: "Pod Volume Restores", + IsPass: true, + Message: fmt.Sprintf("Found %d pod volume restores", len(podVolumeRestores)), + }) + } + } + return results +} + +func analyzeRestores(restores []*velerov1.Restore) []*AnalyzeResult { + results := []*AnalyzeResult{} + failures := 0 + + if len(restores) > 0 { + + failedPhases := map[velerov1.RestorePhase]bool{ + velerov1.RestorePhaseFailed: true, + velerov1.RestorePhasePartiallyFailed: true, + velerov1.RestorePhaseFailedValidation: true, + velerov1.RestorePhaseWaitingForPluginOperationsPartiallyFailed: true, + } + + for _, restore := range restores { + if failedPhases[restore.Status.Phase] { + result := &AnalyzeResult{ + Title: fmt.Sprintf("Restore %s", restore.Name), + } + result.IsFail = true + // result.Strict = true + result.Message = fmt.Sprintf("Restore %s phase is %s", restore.Name, restore.Status.Phase) + results = append(results, result) + failures++ + } + // else if restore.Status.Phase == velerov1.RestorePhaseCompleted { + // result.IsPass = true + // // result.Strict = true + // } else { + // // may indicate phases like: + // // - velerov1.RestorePhaseWaitingForPluginOperations + // // - velerov1.RestorePhaseFinalizing + // result.IsWarn = true + // } + } + if failures == 0 { + results = append(results, &AnalyzeResult{ + Title: "Velero Restores", + IsPass: true, + Message: fmt.Sprintf("Found %d restores", len(restores)), + }) + } + } + + return results +} + +func analyzeSchedules(schedules []*velerov1.Schedule) []*AnalyzeResult { + results := []*AnalyzeResult{} + // check for velerov1.SchedulePhaseFailedValidation + failures := 0 + if len(schedules) > 0 { + for _, schedule := range schedules { + if schedule.Status.Phase == velerov1.SchedulePhaseFailedValidation { + result := &AnalyzeResult{ + Title: fmt.Sprintf("Schedule %s", schedule.Name), + } + result.IsFail = true + // result.Strict = true + result.Message = fmt.Sprintf("Schedule %s phase is %s", schedule.Name, schedule.Status.Phase) + results = append(results, result) + failures++ + } + } + if failures == 0 { + results = append(results, &AnalyzeResult{ + Title: "Velero Schedules", + IsPass: true, + Message: fmt.Sprintf("Found %d schedules", len(schedules)), + }) + } + } + return results +} + +func analyzeServerStatusRequests(serverStatusRequests []*velerov1.ServerStatusRequest) []*AnalyzeResult { + results := []*AnalyzeResult{} + // TODO + return results +} + +func analyzeVolumeSnapshotLocations(volumeSnapshotLocations []*velerov1.VolumeSnapshotLocation) []*AnalyzeResult { + results := []*AnalyzeResult{} + // fail on velerov1.VolumeSnapshotLocationPhaseUnavailable + failures := 0 + if len(volumeSnapshotLocations) > 0 { + for _, volumeSnapshotLocation := range volumeSnapshotLocations { + if volumeSnapshotLocation.Status.Phase == velerov1.VolumeSnapshotLocationPhaseUnavailable { + result := &AnalyzeResult{ + Title: fmt.Sprintf("Volume Snapshot Location %s", volumeSnapshotLocation.Name), + } + result.IsFail = true + // result.Strict = true + result.Message = fmt.Sprintf("Volume Snapshot Location %s phase is %s", volumeSnapshotLocation.Name, volumeSnapshotLocation.Status.Phase) + results = append(results, result) + failures++ + } + } + if failures == 0 { + results = append(results, &AnalyzeResult{ + Title: "Velero Volume Snapshot Locations", + IsPass: true, + Message: fmt.Sprintf("Found %d volume snapshot locations", len(volumeSnapshotLocations)), + }) + } + } + + return results +} + +func analyzeLogs(logs map[string][]byte) []*AnalyzeResult { + results := []*AnalyzeResult{} + if len(logs) > 0 { + // fileName + for _, logBytes := range logs { + logContent := string(logBytes) + result := &AnalyzeResult{ + Title: fmt.Sprintf("Velero logs for pod [node-agent] "), + } + if strings.Contains(logContent, "permission denied") { + result.IsFail = true + // result.Strict = true + result.Message = fmt.Sprintf("Found 'permission denied' in node-agent log file(s)") + results = append(results, result) + continue + } + + if strings.Contains(logContent, "error") || strings.Contains(logContent, "panic") || strings.Contains(logContent, "fatal") { + result.IsWarn = true + // result.Strict = false + result.Message = fmt.Sprintf("Found error|panic|fatal in node-agent log file(s)") + results = append(results, result) + } + } + + results = append(results, &AnalyzeResult{ + Title: "Velero Logs", + IsPass: true, + Message: fmt.Sprintf("Found %d log files", len(logs)), + }) + } + return results +} + +func aggregateResults(results []*AnalyzeResult) []*AnalyzeResult { + out := []*AnalyzeResult{} + resultFailed := false + for _, result := range results { + if result.IsFail { + resultFailed = true + // continue + } + out = append(out, result) + } + + if resultFailed == false { + out = append(out, &AnalyzeResult{ + Title: "Velero Status", + IsPass: true, + Message: "Backups and CRDs are healthy", + }) + } + + return out +} + +func GetVeleroBackupsDirectory(namespace string) string { + return fmt.Sprintf("cluster-resources/custom-resources/backups.velero.io") +} + +func GetVeleroBackupRepositoriesDirectory(namespace string) string { + return fmt.Sprintf("cluster-resources/custom-resources/backuprepositories.velero.io") +} + +func GetVeleroBackupStorageLocationsDirectory(namespace string) string { + return fmt.Sprintf("cluster-resources/custom-resources/backupstoragelocations.velero.io") +} + +func GetVeleroDeleteBackupRequestsDirectory(namespace string) string { + return fmt.Sprintf("cluster-resources/custom-resources/deletebackuprequests.velero.io") +} + +func GetVeleroDownloadRequestsDirectory(namespace string) string { + return fmt.Sprintf("cluster-resources/custom-resources/downloadrequests.velero.io") +} + +func GetVeleroLogsDirectory(namespace string) string { + return fmt.Sprintf("velero/logs") +} + +func GetVeleroPodVolumeBackupsDirectory(namespace string) string { + return fmt.Sprintf("cluster-resources/custom-resources/podvolumebackups.velero.io") +} + +func GetVeleroPodVolumeRestoresDirectory(namespace string) string { + return fmt.Sprintf("cluster-resources/custom-resources/podvolumerestores.velero.io") +} + +func GetVeleroRestoresDirectory(namespace string) string { + return fmt.Sprintf("cluster-resources/custom-resources/restores.velero.io") +} + +func GetVeleroSchedulesDirectory(namespace string) string { + return fmt.Sprintf("cluster-resources/custom-resources/schedules.velero.io") +} + +func GetVeleroServerStatusRequestsDirectory(namespace string) string { + return fmt.Sprintf("cluster-resources/custom-resources/serverstatusrequests.velero.io") +} + +func GetVeleroVolumeSnapshotLocationsDirectory(namespace string) string { + return fmt.Sprintf("cluster-resources/custom-resources/volumesnapshotlocations.velero.io") +} + +func GetVeleroResticRepositoriesDirectory(namespace string) string { + return fmt.Sprintf("cluster-resources/custom-resources/resticrepositories.velero.io") +} diff --git a/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go b/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go index 4cb171d30..a206ad7bd 100644 --- a/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go +++ b/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go @@ -186,6 +186,13 @@ type CephStatusAnalyze struct { Namespace string `json:"namespace" yaml:"namespace"` } +type VeleroAnalyze struct { + AnalyzeMeta `json:",inline" yaml:",inline"` + Outcomes []*Outcome `json:"outcomes" yaml:"outcomes"` + CollectorName string `json:"collectorName,omitempty" yaml:"collectorName,omitempty"` + Namespace string `json:"namespace" yaml:"namespace"` +} + type LonghornAnalyze struct { AnalyzeMeta `json:",inline" yaml:",inline"` Outcomes []*Outcome `json:"outcomes" yaml:"outcomes"` @@ -245,6 +252,7 @@ type Analyze struct { Mysql *DatabaseAnalyze `json:"mysql,omitempty" yaml:"mysql,omitempty"` Redis *DatabaseAnalyze `json:"redis,omitempty" yaml:"redis,omitempty"` CephStatus *CephStatusAnalyze `json:"cephStatus,omitempty" yaml:"cephStatus,omitempty"` + Velero *VeleroAnalyze `json:"velero,omitempty" yaml:"velero,omitempty"` Longhorn *LonghornAnalyze `json:"longhorn,omitempty" yaml:"longhorn,omitempty"` RegistryImages *RegistryImagesAnalyze `json:"registryImages,omitempty" yaml:"registryImages,omitempty"` WeaveReport *WeaveReportAnalyze `json:"weaveReport,omitempty" yaml:"weaveReport,omitempty"` diff --git a/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go b/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go index a5fe91e8d..9b8219a30 100644 --- a/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go +++ b/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go @@ -169,6 +169,11 @@ func (in *Analyze) DeepCopyInto(out *Analyze) { *out = new(CephStatusAnalyze) (*in).DeepCopyInto(*out) } + if in.Velero != nil { + in, out := &in.Velero, &out.Velero + *out = new(VeleroAnalyze) + (*in).DeepCopyInto(*out) + } if in.Longhorn != nil { in, out := &in.Longhorn, &out.Longhorn *out = new(LonghornAnalyze) @@ -4654,6 +4659,33 @@ func (in *UDPPortStatusAnalyze) DeepCopy() *UDPPortStatusAnalyze { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VeleroAnalyze) DeepCopyInto(out *VeleroAnalyze) { + *out = *in + in.AnalyzeMeta.DeepCopyInto(&out.AnalyzeMeta) + if in.Outcomes != nil { + in, out := &in.Outcomes, &out.Outcomes + *out = make([]*Outcome, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Outcome) + (*in).DeepCopyInto(*out) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VeleroAnalyze. +func (in *VeleroAnalyze) DeepCopy() *VeleroAnalyze { + if in == nil { + return nil + } + out := new(VeleroAnalyze) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WeaveReportAnalyze) DeepCopyInto(out *WeaveReportAnalyze) { *out = *in diff --git a/pkg/supportbundle/test/velero.yaml b/pkg/supportbundle/test/velero.yaml new file mode 100644 index 000000000..c684db427 --- /dev/null +++ b/pkg/supportbundle/test/velero.yaml @@ -0,0 +1,12 @@ +apiVersion: troubleshoot.sh/v1beta2 +kind: SupportBundle +metadata: + name: velero +spec: + collectors: + - logs: + namespace: velero + name: velero/logs + analyzers: + - velero: + strict: true diff --git a/schemas/analyzer-troubleshoot-v1beta2.json b/schemas/analyzer-troubleshoot-v1beta2.json index 023101929..c07e25235 100644 --- a/schemas/analyzer-troubleshoot-v1beta2.json +++ b/schemas/analyzer-troubleshoot-v1beta2.json @@ -2255,6 +2255,86 @@ } } }, + "velero": { + "type": "object", + "required": [ + "namespace", + "outcomes" + ], + "properties": { + "annotations": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "checkName": { + "type": "string" + }, + "collectorName": { + "type": "string" + }, + "exclude": { + "oneOf": [{"type": "string"},{"type": "boolean"}] + }, + "namespace": { + "type": "string" + }, + "outcomes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "fail": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + }, + "pass": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + }, + "warn": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + } + } + } + }, + "strict": { + "oneOf": [{"type": "string"},{"type": "boolean"}] + } + } + }, "weaveReport": { "type": "object", "required": [ diff --git a/schemas/preflight-troubleshoot-v1beta2.json b/schemas/preflight-troubleshoot-v1beta2.json index 4e2ee222b..fae3371c7 100644 --- a/schemas/preflight-troubleshoot-v1beta2.json +++ b/schemas/preflight-troubleshoot-v1beta2.json @@ -2255,6 +2255,86 @@ } } }, + "velero": { + "type": "object", + "required": [ + "namespace", + "outcomes" + ], + "properties": { + "annotations": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "checkName": { + "type": "string" + }, + "collectorName": { + "type": "string" + }, + "exclude": { + "oneOf": [{"type": "string"},{"type": "boolean"}] + }, + "namespace": { + "type": "string" + }, + "outcomes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "fail": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + }, + "pass": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + }, + "warn": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + } + } + } + }, + "strict": { + "oneOf": [{"type": "string"},{"type": "boolean"}] + } + } + }, "weaveReport": { "type": "object", "required": [ diff --git a/schemas/supportbundle-troubleshoot-v1beta2.json b/schemas/supportbundle-troubleshoot-v1beta2.json index c667cb53b..14767b014 100644 --- a/schemas/supportbundle-troubleshoot-v1beta2.json +++ b/schemas/supportbundle-troubleshoot-v1beta2.json @@ -2301,6 +2301,86 @@ } } }, + "velero": { + "type": "object", + "required": [ + "namespace", + "outcomes" + ], + "properties": { + "annotations": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "checkName": { + "type": "string" + }, + "collectorName": { + "type": "string" + }, + "exclude": { + "oneOf": [{"type": "string"},{"type": "boolean"}] + }, + "namespace": { + "type": "string" + }, + "outcomes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "fail": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + }, + "pass": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + }, + "warn": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "uri": { + "type": "string" + }, + "when": { + "type": "string" + } + } + } + } + } + }, + "strict": { + "oneOf": [{"type": "string"},{"type": "boolean"}] + } + } + }, "weaveReport": { "type": "object", "required": [ From 2043631a7c583e2638173be0610f33152c5a8b4c Mon Sep 17 00:00:00 2001 From: Archit Sharma Date: Thu, 19 Oct 2023 17:30:51 +0530 Subject: [PATCH 2/9] no debug Signed-off-by: Archit Sharma --- pkg/analyze/velero.go | 40 ++-------------------------------------- 1 file changed, 2 insertions(+), 38 deletions(-) diff --git a/pkg/analyze/velero.go b/pkg/analyze/velero.go index 01b6dd12e..21d077e40 100644 --- a/pkg/analyze/velero.go +++ b/pkg/analyze/velero.go @@ -54,7 +54,6 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze // get backuprepositories.velero.io backupRepositoriesDir := GetVeleroBackupRepositoriesDirectory(ns) - fmt.Println(backupRepositoriesDir) backupRepositoriesGlob := filepath.Join(backupRepositoriesDir, "*.json") backupRepositoriesJson, err := findFiles(backupRepositoriesGlob, excludeFiles) if err != nil { @@ -63,23 +62,17 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze backupRepositories := []*velerov1.BackupRepository{} for key, backupRepositoryJson := range backupRepositoriesJson { var backupRepositoryArray []*velerov1.BackupRepository - // backupRepository := &velerov1.BackupRepository{} err := json.Unmarshal(backupRepositoryJson, &backupRepositoryArray) if err != nil { return nil, errors.Wrapf(err, "failed to unmarshal backup repository json from %s", key) } backupRepositories = append(backupRepositories, backupRepositoryArray...) } - // print len of backupRepositories - fmt.Printf("\n..found %d velero backup repositories\n", len(backupRepositories)) // get backups.velero.io backupsDir := GetVeleroBackupsDirectory(ns) backupsGlob := filepath.Join(backupsDir, "*.json") - fmt.Println(backupsDir) veleroJSONs, err := findFiles(backupsGlob, excludeFiles) - // print files for debugging - fmt.Printf("\n..found %d velero backup jsons\n", len(veleroJSONs)) if err != nil { return nil, errors.Wrapf(err, "failed to find velero backup files") } @@ -95,10 +88,9 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze } backups = append(backups, veleroBackups...) } - fmt.Printf("\n..found %d backups\n", len(backups)) - // // velerov1.BackupRepositoryTypeRestic - // // // get resticrepositories.velero.io + // velerov1.BackupRepositoryTypeRestic + // get resticrepositories.velero.io // resticRepositoriesDir := GetVeleroResticRepositoriesDirectory(ns) // resticRepositoriesGlob := filepath.Join(resticRepositoriesDir, "*.json") // resticRepositoriesYaml, err := findFiles(resticRepositoriesGlob, excludeFiles) @@ -131,7 +123,6 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze } backupStorageLocations = append(backupStorageLocations, backupStorageLocationArray...) } - fmt.Printf("\n..found %d velero backup storage locations\n", len(backupStorageLocations)) // get deletebackuprequests.velero.io deleteBackupRequestsDir := GetVeleroDeleteBackupRequestsDirectory(ns) @@ -149,7 +140,6 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze } deleteBackupRequests = append(deleteBackupRequests, deleteBackupRequestArray...) } - fmt.Printf("\n..found %d velero delete backup requests\n", len(deleteBackupRequests)) // get downloadrequests.velero.io // downloadRequestsDir := GetVeleroDownloadRequestsDirectory(ns) @@ -201,7 +191,6 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze } podVolumeRestores = append(podVolumeRestores, podVolumeRestoreArray...) } - fmt.Println("FIRST----------") // get restores.velero.io restoresDir := GetVeleroRestoresDirectory(ns) @@ -215,12 +204,10 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze var restoreArray []*velerov1.Restore err := json.Unmarshal(restoreYaml, &restoreArray) if err != nil { - fmt.Println("ERROR----------", key, err) return nil, errors.Wrapf(err, "failed to unmarshal restore json from %s", key) } restores = append(restores, restoreArray...) } - fmt.Println("THIRD----------") // get schedules.velero.io schedulesDir := GetVeleroSchedulesDirectory(ns) @@ -274,7 +261,6 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze } logsDir := GetVeleroLogsDirectory(ns) - fmt.Println(logsDir) logsGlob := filepath.Join(logsDir, "node-agent*", "*.log") logs, err := findFiles(logsGlob, excludeFiles) @@ -371,7 +357,6 @@ func analyzeBackups(backups []*velerov1.Backup) []*AnalyzeResult { func analyzeBackupStorageLocations(backupStorageLocations []*velerov1.BackupStorageLocation) []*AnalyzeResult { results := []*AnalyzeResult{} - // atleast 1 backup storage location Phase Available availableCount := 0 bslResult := &AnalyzeResult{ Title: "At least 1 Velero Backup Storage Location configured", @@ -409,7 +394,6 @@ func analyzeBackupStorageLocations(backupStorageLocations []*velerov1.BackupStor func analyzeDeleteBackupRequests(deleteBackupRequests []*velerov1.DeleteBackupRequest) []*AnalyzeResult { results := []*AnalyzeResult{} - // all in progress, new and processed inProgressCount := 0 if len(deleteBackupRequests) > 0 { for _, deleteBackupRequest := range deleteBackupRequests { @@ -459,16 +443,13 @@ func analyzeDeleteBackupRequests(deleteBackupRequests []*velerov1.DeleteBackupRe func analyzePodVolumeBackups(podVolumeBackups []*velerov1.PodVolumeBackup) []*AnalyzeResult { results := []*AnalyzeResult{} failures := 0 - // isFail if any pod volume backup phase is Failed if len(podVolumeBackups) > 0 { - // look for PodVolumeBackupPhaseFailed (only 1) for _, podVolumeBackup := range podVolumeBackups { if podVolumeBackup.Status.Phase == velerov1.PodVolumeBackupPhaseFailed { result := &AnalyzeResult{ Title: fmt.Sprintf("Pod Volume Backup %s", podVolumeBackup.Name), } result.IsFail = true - // result.Strict = true result.Message = fmt.Sprintf("Pod Volume Backup %s phase is %s", podVolumeBackup.Name, podVolumeBackup.Status.Phase) results = append(results, result) failures++ @@ -534,20 +515,10 @@ func analyzeRestores(restores []*velerov1.Restore) []*AnalyzeResult { Title: fmt.Sprintf("Restore %s", restore.Name), } result.IsFail = true - // result.Strict = true result.Message = fmt.Sprintf("Restore %s phase is %s", restore.Name, restore.Status.Phase) results = append(results, result) failures++ } - // else if restore.Status.Phase == velerov1.RestorePhaseCompleted { - // result.IsPass = true - // // result.Strict = true - // } else { - // // may indicate phases like: - // // - velerov1.RestorePhaseWaitingForPluginOperations - // // - velerov1.RestorePhaseFinalizing - // result.IsWarn = true - // } } if failures == 0 { results = append(results, &AnalyzeResult{ @@ -563,7 +534,6 @@ func analyzeRestores(restores []*velerov1.Restore) []*AnalyzeResult { func analyzeSchedules(schedules []*velerov1.Schedule) []*AnalyzeResult { results := []*AnalyzeResult{} - // check for velerov1.SchedulePhaseFailedValidation failures := 0 if len(schedules) > 0 { for _, schedule := range schedules { @@ -572,7 +542,6 @@ func analyzeSchedules(schedules []*velerov1.Schedule) []*AnalyzeResult { Title: fmt.Sprintf("Schedule %s", schedule.Name), } result.IsFail = true - // result.Strict = true result.Message = fmt.Sprintf("Schedule %s phase is %s", schedule.Name, schedule.Status.Phase) results = append(results, result) failures++ @@ -597,7 +566,6 @@ func analyzeServerStatusRequests(serverStatusRequests []*velerov1.ServerStatusRe func analyzeVolumeSnapshotLocations(volumeSnapshotLocations []*velerov1.VolumeSnapshotLocation) []*AnalyzeResult { results := []*AnalyzeResult{} - // fail on velerov1.VolumeSnapshotLocationPhaseUnavailable failures := 0 if len(volumeSnapshotLocations) > 0 { for _, volumeSnapshotLocation := range volumeSnapshotLocations { @@ -627,7 +595,6 @@ func analyzeVolumeSnapshotLocations(volumeSnapshotLocations []*velerov1.VolumeSn func analyzeLogs(logs map[string][]byte) []*AnalyzeResult { results := []*AnalyzeResult{} if len(logs) > 0 { - // fileName for _, logBytes := range logs { logContent := string(logBytes) result := &AnalyzeResult{ @@ -635,7 +602,6 @@ func analyzeLogs(logs map[string][]byte) []*AnalyzeResult { } if strings.Contains(logContent, "permission denied") { result.IsFail = true - // result.Strict = true result.Message = fmt.Sprintf("Found 'permission denied' in node-agent log file(s)") results = append(results, result) continue @@ -643,7 +609,6 @@ func analyzeLogs(logs map[string][]byte) []*AnalyzeResult { if strings.Contains(logContent, "error") || strings.Contains(logContent, "panic") || strings.Contains(logContent, "fatal") { result.IsWarn = true - // result.Strict = false result.Message = fmt.Sprintf("Found error|panic|fatal in node-agent log file(s)") results = append(results, result) } @@ -664,7 +629,6 @@ func aggregateResults(results []*AnalyzeResult) []*AnalyzeResult { for _, result := range results { if result.IsFail { resultFailed = true - // continue } out = append(out, result) } From d70026ee4a0d7e49756b2fd0e27ae4474fd93f57 Mon Sep 17 00:00:00 2001 From: Archit Sharma Date: Thu, 19 Oct 2023 19:10:32 +0530 Subject: [PATCH 3/9] fix var names yaml -> json Signed-off-by: Archit Sharma --- pkg/analyze/velero.go | 46 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/pkg/analyze/velero.go b/pkg/analyze/velero.go index 21d077e40..e380c4655 100644 --- a/pkg/analyze/velero.go +++ b/pkg/analyze/velero.go @@ -93,14 +93,14 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze // get resticrepositories.velero.io // resticRepositoriesDir := GetVeleroResticRepositoriesDirectory(ns) // resticRepositoriesGlob := filepath.Join(resticRepositoriesDir, "*.json") - // resticRepositoriesYaml, err := findFiles(resticRepositoriesGlob, excludeFiles) + // resticRepositoriesJson, err := findFiles(resticRepositoriesGlob, excludeFiles) // if err != nil { // return nil, errors.Wrapf(err, "failed to find velero restic repositories files under %s", resticRepositoriesDir) // } // resticRepositories := []*velerov1beta1.ResticRepository{} - // for key, resticRepositoryYaml := range resticRepositoriesYaml { + // for key, resticRepositoryJson := range resticRepositoriesJson { // resticRepository := &velerov1beta1.ResticRepository{} - // err := json.Unmarshal(resticRepositoryYaml, resticRepository) + // err := json.Unmarshal(resticRepositoryJson, resticRepository) // if err != nil { // return nil, errors.Wrapf(err, "failed to unmarshal restic repository json from %s", key) // } @@ -115,9 +115,9 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze return nil, errors.Wrapf(err, "failed to find velero backup storage locations files under %s", backupStorageLocationsDir) } backupStorageLocations := []*velerov1.BackupStorageLocation{} - for key, backupStorageLocationYaml := range backupStorageLocationsJson { + for key, backupStorageLocationJson := range backupStorageLocationsJson { var backupStorageLocationArray []*velerov1.BackupStorageLocation - err := json.Unmarshal(backupStorageLocationYaml, &backupStorageLocationArray) + err := json.Unmarshal(backupStorageLocationJson, &backupStorageLocationArray) if err != nil { return nil, errors.Wrapf(err, "failed to unmarshal backup storage location json from %s", key) } @@ -132,9 +132,9 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze return nil, errors.Wrapf(err, "failed to find velero delete backup requests files under %s", deleteBackupRequestsDir) } deleteBackupRequests := []*velerov1.DeleteBackupRequest{} - for key, deleteBackupRequestYaml := range deleteBackupRequestsJson { + for key, deleteBackupRequestJson := range deleteBackupRequestsJson { var deleteBackupRequestArray []*velerov1.DeleteBackupRequest - err := json.Unmarshal(deleteBackupRequestYaml, &deleteBackupRequestArray) + err := json.Unmarshal(deleteBackupRequestJson, &deleteBackupRequestArray) if err != nil { return nil, errors.Wrapf(err, "failed to unmarshal delete backup request json from %s", key) } @@ -144,14 +144,14 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze // get downloadrequests.velero.io // downloadRequestsDir := GetVeleroDownloadRequestsDirectory(ns) // downloadRequestsGlob := filepath.Join(downloadRequestsDir, "*.json") - // downloadRequestsYaml, err := findFiles(downloadRequestsGlob, excludeFiles) + // downloadRequestsJson, err := findFiles(downloadRequestsGlob, excludeFiles) // if err != nil { // return nil, errors.Wrapf(err, "failed to find velero download requests files under %s", downloadRequestsDir) // } // downloadRequests := []*velerov1.DownloadRequest{} - // for key, downloadRequestYaml := range downloadRequestsYaml { + // for key, downloadRequestJson := range downloadRequestsJson { // downloadRequest := &velerov1.DownloadRequest{} - // err := json.Unmarshal(downloadRequestYaml, downloadRequest) + // err := json.Unmarshal(downloadRequestJson, downloadRequest) // if err != nil { // return nil, errors.Wrapf(err, "failed to unmarshal download request json from %s", key) // } @@ -183,9 +183,9 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze return nil, errors.Wrapf(err, "failed to find velero pod volume restores files under %s", podVolumeRestoresDir) } podVolumeRestores := []*velerov1.PodVolumeRestore{} - for key, podVolumeRestoreYaml := range podVolumeRestoresJson { + for key, podVolumeRestoreJson := range podVolumeRestoresJson { var podVolumeRestoreArray []*velerov1.PodVolumeRestore - err := json.Unmarshal(podVolumeRestoreYaml, &podVolumeRestoreArray) + err := json.Unmarshal(podVolumeRestoreJson, &podVolumeRestoreArray) if err != nil { return nil, errors.Wrapf(err, "failed to unmarshal pod volume restore json from %s", key) } @@ -200,9 +200,9 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze return nil, errors.Wrapf(err, "failed to find velero restores files under %s", restoresDir) } restores := []*velerov1.Restore{} - for key, restoreYaml := range restoresJson { + for key, restoreJson := range restoresJson { var restoreArray []*velerov1.Restore - err := json.Unmarshal(restoreYaml, &restoreArray) + err := json.Unmarshal(restoreJson, &restoreArray) if err != nil { return nil, errors.Wrapf(err, "failed to unmarshal restore json from %s", key) } @@ -212,14 +212,14 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze // get schedules.velero.io schedulesDir := GetVeleroSchedulesDirectory(ns) schedulesGlob := filepath.Join(schedulesDir, "*.json") - schedulesYaml, err := findFiles(schedulesGlob, excludeFiles) + schedulesJson, err := findFiles(schedulesGlob, excludeFiles) if err != nil { return nil, errors.Wrapf(err, "failed to find velero schedules files under %s", schedulesDir) } schedules := []*velerov1.Schedule{} - for key, scheduleYaml := range schedulesYaml { + for key, scheduleJson := range schedulesJson { var scheduleArray []*velerov1.Schedule - err := json.Unmarshal(scheduleYaml, &scheduleArray) + err := json.Unmarshal(scheduleJson, &scheduleArray) if err != nil { return nil, errors.Wrapf(err, "failed to unmarshal schedule json from %s", key) } @@ -229,14 +229,14 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze // get serverstatusrequests.velero.io serverStatusRequestsDir := GetVeleroServerStatusRequestsDirectory(ns) serverStatusRequestsGlob := filepath.Join(serverStatusRequestsDir, "*.json") - serverStatusRequestsYaml, err := findFiles(serverStatusRequestsGlob, excludeFiles) + serverStatusRequestsJson, err := findFiles(serverStatusRequestsGlob, excludeFiles) if err != nil { return nil, errors.Wrapf(err, "failed to find velero server status requests files under %s", serverStatusRequestsDir) } serverStatusRequests := []*velerov1.ServerStatusRequest{} - for key, serverStatusRequestYaml := range serverStatusRequestsYaml { + for key, serverStatusRequestJson := range serverStatusRequestsJson { var serverStatusRequestArray []*velerov1.ServerStatusRequest - err := json.Unmarshal(serverStatusRequestYaml, &serverStatusRequestArray) + err := json.Unmarshal(serverStatusRequestJson, &serverStatusRequestArray) if err != nil { return nil, errors.Wrapf(err, "failed to unmarshal server status request json from %s", key) } @@ -246,14 +246,14 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze // get volumesnapshotlocations.velero.io volumeSnapshotLocationsDir := GetVeleroVolumeSnapshotLocationsDirectory(ns) volumeSnapshotLocationsGlob := filepath.Join(volumeSnapshotLocationsDir, "*.json") - volumeSnapshotLocationsYaml, err := findFiles(volumeSnapshotLocationsGlob, excludeFiles) + volumeSnapshotLocationsJson, err := findFiles(volumeSnapshotLocationsGlob, excludeFiles) if err != nil { return nil, errors.Wrapf(err, "failed to find velero volume snapshot locations files under %s", volumeSnapshotLocationsDir) } volumeSnapshotLocations := []*velerov1.VolumeSnapshotLocation{} - for key, volumeSnapshotLocationYaml := range volumeSnapshotLocationsYaml { + for key, volumeSnapshotLocationJson := range volumeSnapshotLocationsJson { var volumeSnapshotLocationArray []*velerov1.VolumeSnapshotLocation - err := json.Unmarshal(volumeSnapshotLocationYaml, &volumeSnapshotLocationArray) + err := json.Unmarshal(volumeSnapshotLocationJson, &volumeSnapshotLocationArray) if err != nil { return nil, errors.Wrapf(err, "failed to unmarshal volume snapshot location json from %s", key) } From 729754b84579daaacf30de533c9da01ddcc75f1c Mon Sep 17 00:00:00 2001 From: Archit Sharma Date: Tue, 31 Oct 2023 13:18:02 +0530 Subject: [PATCH 4/9] init velero analyzer tests Signed-off-by: Archit Sharma --- pkg/analyze/velero.go | 106 +---- pkg/analyze/velero_test.go | 937 +++++++++++++++++++++++++++++++++++++ 2 files changed, 957 insertions(+), 86 deletions(-) create mode 100644 pkg/analyze/velero_test.go diff --git a/pkg/analyze/velero.go b/pkg/analyze/velero.go index e380c4655..c38973e9e 100644 --- a/pkg/analyze/velero.go +++ b/pkg/analyze/velero.go @@ -9,7 +9,6 @@ import ( "github.com/pkg/errors" troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - // velerov1beta1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1beta1" ) const ( @@ -89,24 +88,6 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze backups = append(backups, veleroBackups...) } - // velerov1.BackupRepositoryTypeRestic - // get resticrepositories.velero.io - // resticRepositoriesDir := GetVeleroResticRepositoriesDirectory(ns) - // resticRepositoriesGlob := filepath.Join(resticRepositoriesDir, "*.json") - // resticRepositoriesJson, err := findFiles(resticRepositoriesGlob, excludeFiles) - // if err != nil { - // return nil, errors.Wrapf(err, "failed to find velero restic repositories files under %s", resticRepositoriesDir) - // } - // resticRepositories := []*velerov1beta1.ResticRepository{} - // for key, resticRepositoryJson := range resticRepositoriesJson { - // resticRepository := &velerov1beta1.ResticRepository{} - // err := json.Unmarshal(resticRepositoryJson, resticRepository) - // if err != nil { - // return nil, errors.Wrapf(err, "failed to unmarshal restic repository json from %s", key) - // } - // resticRepositories = append(resticRepositories, resticRepository) - // } - // get backupstoragelocations.velero.io backupStorageLocationsDir := GetVeleroBackupStorageLocationsDirectory(ns) backupStorageLocationsGlob := filepath.Join(backupStorageLocationsDir, "*.json") @@ -141,23 +122,6 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze deleteBackupRequests = append(deleteBackupRequests, deleteBackupRequestArray...) } - // get downloadrequests.velero.io - // downloadRequestsDir := GetVeleroDownloadRequestsDirectory(ns) - // downloadRequestsGlob := filepath.Join(downloadRequestsDir, "*.json") - // downloadRequestsJson, err := findFiles(downloadRequestsGlob, excludeFiles) - // if err != nil { - // return nil, errors.Wrapf(err, "failed to find velero download requests files under %s", downloadRequestsDir) - // } - // downloadRequests := []*velerov1.DownloadRequest{} - // for key, downloadRequestJson := range downloadRequestsJson { - // downloadRequest := &velerov1.DownloadRequest{} - // err := json.Unmarshal(downloadRequestJson, downloadRequest) - // if err != nil { - // return nil, errors.Wrapf(err, "failed to unmarshal download request json from %s", key) - // } - // downloadRequests = append(downloadRequests, downloadRequest) - // } - // get podvolumebackups.velero.io podVolumeBackupsDir := GetVeleroPodVolumeBackupsDirectory(ns) podVolumeBackupsGlob := filepath.Join(podVolumeBackupsDir, "*.json") @@ -272,15 +236,12 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze results = append(results, analyzeLogs(logs)...) results = append(results, analyzeBackupRepositories(backupRepositories)...) results = append(results, analyzeBackups(backups)...) - // results = append(results, analyzeResticRepositories(resticRepositories)...) results = append(results, analyzeBackupStorageLocations(backupStorageLocations)...) results = append(results, analyzeDeleteBackupRequests(deleteBackupRequests)...) - // results = append(results, analyzeDownloadRequests(downloadRequests)...) results = append(results, analyzePodVolumeBackups(podVolumeBackups)...) results = append(results, analyzePodVolumeRestores(podVolumeRestores)...) results = append(results, analyzeRestores(restores)...) results = append(results, analyzeSchedules(schedules)...) - results = append(results, analyzeServerStatusRequests(serverStatusRequests)...) results = append(results, analyzeVolumeSnapshotLocations(volumeSnapshotLocations)...) return aggregateResults(results), nil @@ -374,7 +335,6 @@ func analyzeBackupStorageLocations(backupStorageLocations []*velerov1.BackupStor result.Message = fmt.Sprintf("Backup Storage Location [%s] is in phase %s", backupStorageLocation.Name, backupStorageLocation.Status.Phase) result.IsWarn = true results = append(results, result) - // result.Strict = false } else { availableCount++ } @@ -414,32 +374,6 @@ func analyzeDeleteBackupRequests(deleteBackupRequests []*velerov1.DeleteBackupRe return results } -// func analyzeDownloadRequests(downloadRequests []*velerov1.DownloadRequest) []*AnalyzeResult { -// results := []*AnalyzeResult{} -// // all -// processedCount := 0 -// newCount := 0 -// if len(downloadRequests) > 0 { -// for _, downloadRequest := range downloadRequests { -// if downloadRequest.Status.Phase == velerov1.DownloadRequestPhaseProcessed { -// processedCount++ -// } else if downloadRequest.Status.Phase == velerov1.DownloadRequestPhaseNew { -// newCount++ -// } -// } -// if processedCount > 0 || newCount > 0 { -// downloadRequestsResult := &AnalyzeResult{ -// Title: "Download Requests summary", -// } -// downloadRequestsResult.IsPass = true -// downloadRequestsResult.Message = fmt.Sprintf("Found %d processed and %d new download requests", processedCount, newCount) -// results = append(results, downloadRequestsResult) -// } -// } - -// return results -// } - func analyzePodVolumeBackups(podVolumeBackups []*velerov1.PodVolumeBackup) []*AnalyzeResult { results := []*AnalyzeResult{} failures := 0 @@ -479,7 +413,6 @@ func analyzePodVolumeRestores(podVolumeRestores []*velerov1.PodVolumeRestore) [] Title: fmt.Sprintf("Pod Volume Restore %s", podVolumeRestore.Name), } result.IsFail = true - // result.Strict = true result.Message = fmt.Sprintf("Pod Volume Restore %s phase is %s", podVolumeRestore.Name, podVolumeRestore.Status.Phase) results = append(results, result) failures++ @@ -558,12 +491,6 @@ func analyzeSchedules(schedules []*velerov1.Schedule) []*AnalyzeResult { return results } -func analyzeServerStatusRequests(serverStatusRequests []*velerov1.ServerStatusRequest) []*AnalyzeResult { - results := []*AnalyzeResult{} - // TODO - return results -} - func analyzeVolumeSnapshotLocations(volumeSnapshotLocations []*velerov1.VolumeSnapshotLocation) []*AnalyzeResult { results := []*AnalyzeResult{} failures := 0 @@ -574,7 +501,6 @@ func analyzeVolumeSnapshotLocations(volumeSnapshotLocations []*velerov1.VolumeSn Title: fmt.Sprintf("Volume Snapshot Location %s", volumeSnapshotLocation.Name), } result.IsFail = true - // result.Strict = true result.Message = fmt.Sprintf("Volume Snapshot Location %s phase is %s", volumeSnapshotLocation.Name, volumeSnapshotLocation.Status.Phase) results = append(results, result) failures++ @@ -598,24 +524,24 @@ func analyzeLogs(logs map[string][]byte) []*AnalyzeResult { for _, logBytes := range logs { logContent := string(logBytes) result := &AnalyzeResult{ - Title: fmt.Sprintf("Velero logs for pod [node-agent] "), + Title: fmt.Sprintf("Velero logs for pod [node-agent]"), } if strings.Contains(logContent, "permission denied") { - result.IsFail = true - result.Message = fmt.Sprintf("Found 'permission denied' in node-agent log file(s)") + result.IsWarn = true + result.Message = fmt.Sprintf("Found 'permission denied' in node-agent* pod log file(s)") results = append(results, result) continue } if strings.Contains(logContent, "error") || strings.Contains(logContent, "panic") || strings.Contains(logContent, "fatal") { result.IsWarn = true - result.Message = fmt.Sprintf("Found error|panic|fatal in node-agent log file(s)") + result.Message = fmt.Sprintf("Found error|panic|fatal in node-agent* pod log file(s)") results = append(results, result) } } results = append(results, &AnalyzeResult{ - Title: "Velero Logs", + Title: "Velero Logs analysis", IsPass: true, Message: fmt.Sprintf("Found %d log files", len(logs)), }) @@ -632,13 +558,21 @@ func aggregateResults(results []*AnalyzeResult) []*AnalyzeResult { } out = append(out, result) } - - if resultFailed == false { - out = append(out, &AnalyzeResult{ - Title: "Velero Status", - IsPass: true, - Message: "Backups and CRDs are healthy", - }) + if len(results) > 0 { + if resultFailed == false { + out = append(out, &AnalyzeResult{ + Title: "Velero Status", + IsPass: true, + Message: "Velero setup is healthy", + }) + } + if resultFailed == true { + out = append(out, &AnalyzeResult{ + Title: "Velero Status", + IsWarn: true, + Message: "Velero setup is not entirely healthy", + }) + } } return out diff --git a/pkg/analyze/velero_test.go b/pkg/analyze/velero_test.go new file mode 100644 index 000000000..92d6487cf --- /dev/null +++ b/pkg/analyze/velero_test.go @@ -0,0 +1,937 @@ +package analyzer + +import ( + "encoding/json" + "reflect" + "testing" + "time" + + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestAnalyzeVelero_BackupRepositories(t *testing.T) { + type args struct { + backupRepositories []*velerov1.BackupRepository + } + tests := []struct { + name string + args args + want []*AnalyzeResult + }{ + { + name: "no backup repositories", + args: args{ + backupRepositories: []*velerov1.BackupRepository{}, + }, + want: []*AnalyzeResult{ + { + Title: "At least 1 Velero Backup Repository configured", + Message: "No backup repositories configured", + IsFail: true, + }, + }, + }, + { + name: "1 backup repository and 1 Ready", + args: args{ + backupRepositories: []*velerov1.BackupRepository{ + { + Spec: velerov1.BackupRepositorySpec{ + BackupStorageLocation: "default", + VolumeNamespace: "velero", + }, + Status: velerov1.BackupRepositoryStatus{ + Phase: velerov1.BackupRepositoryPhaseReady, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "At least 1 Velero Backup Repository configured", + Message: "Found 1 backup repositories configured and 1 Ready", + IsPass: true, + }, + }, + }, + { + name: "2 backup repositories and 1 Ready", + args: args{ + backupRepositories: []*velerov1.BackupRepository{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "default-default-restic-245sd", + Namespace: "velero", + }, + Spec: velerov1.BackupRepositorySpec{ + BackupStorageLocation: "default", + VolumeNamespace: "velero", + }, + Status: velerov1.BackupRepositoryStatus{ + Phase: velerov1.BackupRepositoryPhaseReady, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "builders-default-restic-jdtd8", + Namespace: "velero", + }, + Spec: velerov1.BackupRepositorySpec{ + BackupStorageLocation: "builders-default-restic-jdtd8", + VolumeNamespace: "velero", + }, + Status: velerov1.BackupRepositoryStatus{ + Phase: velerov1.BackupRepositoryPhaseNotReady, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Backup Repository builders-default-restic-jdtd8", + Message: "Backup Repository [builders-default-restic-jdtd8] is in phase NotReady", + IsWarn: true, + }, + { + Title: "At least 1 Velero Backup Repository configured", + Message: "Found 2 backup repositories configured and 1 Ready", + IsPass: true, + }, + }, + }, + { + name: "1 backup repository and none Ready", + args: args{ + backupRepositories: []*velerov1.BackupRepository{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "velero", + }, + Spec: velerov1.BackupRepositorySpec{ + BackupStorageLocation: "default", + VolumeNamespace: "velero", + }, + Status: velerov1.BackupRepositoryStatus{ + Phase: velerov1.BackupRepositoryPhaseNotReady, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Backup Repository default", + Message: "Backup Repository [default] is in phase NotReady", + IsWarn: true, + }, + { + Title: "At least 1 Velero Backup Repository configured", + Message: "Found 1 configured backup repositories, but none are ready", + IsWarn: true, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := analyzeBackupRepositories(tt.args.backupRepositories); !reflect.DeepEqual(got, tt.want) { + t.Errorf("analyzeBackupRepositories() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAnalyzeVelero_Backups(t *testing.T) { + type args struct { + backups []*velerov1.Backup + } + tests := []struct { + name string + args args + want []*AnalyzeResult + }{ + { + name: "no backups", + args: args{ + backups: []*velerov1.Backup{}, + }, + want: []*AnalyzeResult{}, + }, + { + name: "1 backup and 1 Completed", + args: args{ + backups: []*velerov1.Backup{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "observability-backup", + Namespace: "velero", + }, + Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"monitoring"}, + }, + Status: velerov1.BackupStatus{ + Phase: velerov1.BackupPhaseCompleted, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Velero Backups", + Message: "Found 1 backups", + IsPass: true, + }, + }, + }, + { + name: "1 backup and 1 Failed", + args: args{ + backups: []*velerov1.Backup{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "application-backup", + Namespace: "velero", + }, + Spec: velerov1.BackupSpec{ + IncludedNamespaces: []string{"shazam"}, + }, + Status: velerov1.BackupStatus{ + Phase: velerov1.BackupPhaseFailed, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Backup application-backup", + Message: "Backup application-backup phase is Failed", + IsFail: true, + }, + { + Title: "Velero Backups", + Message: "Found 1 backups", + IsPass: true, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := analyzeBackups(tt.args.backups); !reflect.DeepEqual(got, tt.want) { + t.Errorf("analyzeBackups() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAnalyzeVelero_BackupStorageLocations(t *testing.T) { + type args struct { + backupStorageLocations []*velerov1.BackupStorageLocation + } + tests := []struct { + name string + args args + want []*AnalyzeResult + }{ + { + name: "no backup storage locations", + args: args{ + backupStorageLocations: []*velerov1.BackupStorageLocation{}, + }, + want: []*AnalyzeResult{ + { + Title: "At least 1 Velero Backup Storage Location configured", + Message: "No backup storage locations configured", + IsFail: true, + }, + }, + }, + { + name: "1 backup storage location and 1 Available", + args: args{ + backupStorageLocations: []*velerov1.BackupStorageLocation{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "velero", + }, + Spec: velerov1.BackupStorageLocationSpec{ + Provider: "aws", + }, + Status: velerov1.BackupStorageLocationStatus{ + Phase: velerov1.BackupStorageLocationPhaseAvailable, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "At least 1 Velero Backup Storage Location configured", + Message: "Found 1 backup storage locations configured and 1 Available", + IsPass: true, + }, + }, + }, + { + name: "1 backup storage location and 1 Unavailable", + args: args{ + backupStorageLocations: []*velerov1.BackupStorageLocation{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "velero", + }, + Spec: velerov1.BackupStorageLocationSpec{ + Provider: "aws", + }, + Status: velerov1.BackupStorageLocationStatus{ + Phase: velerov1.BackupStorageLocationPhaseUnavailable, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Backup Storage Location default", + Message: "Backup Storage Location [default] is in phase Unavailable", + IsWarn: true, + }, + { + Title: "At least 1 Velero Backup Storage Location configured", + Message: "Found 1 configured backup storage locations, but none are available", + IsWarn: true, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := analyzeBackupStorageLocations(tt.args.backupStorageLocations); !reflect.DeepEqual(got, tt.want) { + t.Errorf("analyzeBackupStorageLocations() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAnalyzeVelero_DeleteBackupRequests(t *testing.T) { + type args struct { + deleteBackupRequests []*velerov1.DeleteBackupRequest + } + tests := []struct { + name string + args args + want []*AnalyzeResult + }{ + { + name: "no backup delete requests", + args: args{ + deleteBackupRequests: []*velerov1.DeleteBackupRequest{}, + }, + want: []*AnalyzeResult{}, + }, + { + name: "backup delete requests completed", + args: args{ + deleteBackupRequests: []*velerov1.DeleteBackupRequest{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "observability-backup-20210308150016", + Namespace: "velero", + }, + Spec: velerov1.DeleteBackupRequestSpec{ + BackupName: "observability-backup", + }, + Status: velerov1.DeleteBackupRequestStatus{ + Phase: velerov1.DeleteBackupRequestPhaseProcessed, + }, + }, + }, + }, + want: []*AnalyzeResult{}, + }, + { + name: "backup delete requests summarize in progress", + args: args{ + deleteBackupRequests: []*velerov1.DeleteBackupRequest{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "observability-backup-20210308150016", + Namespace: "velero", + }, + Spec: velerov1.DeleteBackupRequestSpec{ + BackupName: "observability-backup", + }, + Status: velerov1.DeleteBackupRequestStatus{ + Phase: velerov1.DeleteBackupRequestPhaseInProgress, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Delete Backup Requests summary", + Message: "Found 1 delete backup requests in progress", + IsWarn: true, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := analyzeDeleteBackupRequests(tt.args.deleteBackupRequests); !reflect.DeepEqual(got, tt.want) { + t.Errorf("analyzeDeleteBackupRequests() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAnalyzeVelero_PodVolumeBackups(t *testing.T) { + type args struct { + podVolumeBackups []*velerov1.PodVolumeBackup + } + tests := []struct { + name string + args args + want []*AnalyzeResult + }{ + { + name: "no pod volume backups", + args: args{ + podVolumeBackups: []*velerov1.PodVolumeBackup{}, + }, + want: []*AnalyzeResult{}, + }, + { + name: "pod volume backups", + args: args{ + podVolumeBackups: []*velerov1.PodVolumeBackup{ + { + Spec: velerov1.PodVolumeBackupSpec{ + Node: "test-node-1", + Pod: corev1.ObjectReference{ + Kind: "Pod", + Name: "kotsadm-76ddbc96c4-fsr88", + Namespace: "default", + }, + Volume: "backup", + BackupStorageLocation: "default", + }, + Status: velerov1.PodVolumeBackupStatus{ + Phase: velerov1.PodVolumeBackupPhaseCompleted, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Pod Volume Backups", + Message: "Found 1 pod volume backups", + IsPass: true, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := analyzePodVolumeBackups(tt.args.podVolumeBackups); !reflect.DeepEqual(got, tt.want) { + t.Errorf("analyzePodVolumeBackups() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAnalyzeVelero_PodVolumeRestores(t *testing.T) { + type args struct { + podVolumeRestores []*velerov1.PodVolumeRestore + } + tests := []struct { + name string + args args + want []*AnalyzeResult + }{ + { + name: "no pod volume restores", + args: args{ + podVolumeRestores: []*velerov1.PodVolumeRestore{}, + }, + want: []*AnalyzeResult{}, + }, + { + name: "pod volume restores - no failures", + args: args{ + podVolumeRestores: []*velerov1.PodVolumeRestore{ + { + Spec: velerov1.PodVolumeRestoreSpec{ + Pod: corev1.ObjectReference{ + Kind: "Pod", + Name: "kotsadm-76ddbc96c4-fsr88", + Namespace: "default", + }, + Volume: "backup", + BackupStorageLocation: "default", + }, + Status: velerov1.PodVolumeRestoreStatus{ + Phase: velerov1.PodVolumeRestorePhaseCompleted, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Pod Volume Restores", + Message: "Found 1 pod volume restores", + IsPass: true, + }, + }, + }, + { + name: "pod volume restores - failures", + args: args{ + podVolumeRestores: []*velerov1.PodVolumeRestore{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "observability-backup-20210308150016", + Namespace: "velero", + }, + Spec: velerov1.PodVolumeRestoreSpec{ + Pod: corev1.ObjectReference{ + Kind: "Pod", + Name: "kotsadm-76ddbc96c4-fsr88", + Namespace: "default", + }, + Volume: "backup", + BackupStorageLocation: "default", + }, + Status: velerov1.PodVolumeRestoreStatus{ + Phase: velerov1.PodVolumeRestorePhaseFailed, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Pod Volume Restore observability-backup-20210308150016", + Message: "Pod Volume Restore observability-backup-20210308150016 phase is Failed", + IsFail: true, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := analyzePodVolumeRestores(tt.args.podVolumeRestores); !reflect.DeepEqual(got, tt.want) { + t.Errorf("analyzePodVolumeRestores() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAnalyzeVelero_Restores(t *testing.T) { + type args struct { + restores []*velerov1.Restore + } + tests := []struct { + name string + args args + want []*AnalyzeResult + }{ + { + name: "no restores", + args: args{ + restores: []*velerov1.Restore{}, + }, + want: []*AnalyzeResult{}, + }, + { + name: "restores completed", + args: args{ + restores: []*velerov1.Restore{ + { + Spec: velerov1.RestoreSpec{ + BackupName: "observability-backup", + }, + Status: velerov1.RestoreStatus{ + Phase: velerov1.RestorePhaseCompleted, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Velero Restores", + Message: "Found 1 restores", + IsPass: true, + }, + }, + }, + { + name: "restores - failures", + args: args{ + restores: []*velerov1.Restore{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "observability-backup-20210308150016", + Namespace: "velero", + }, + Spec: velerov1.RestoreSpec{ + BackupName: "observability-backup", + }, + Status: velerov1.RestoreStatus{ + Phase: velerov1.RestorePhaseWaitingForPluginOperationsPartiallyFailed, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Restore observability-backup-20210308150016", + Message: "Restore observability-backup-20210308150016 phase is WaitingForPluginOperationsPartiallyFailed", + IsFail: true, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := analyzeRestores(tt.args.restores); !reflect.DeepEqual(got, tt.want) { + t.Errorf("analyzeRestores() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAnalyzeVelero_Schedules(t *testing.T) { + type args struct { + schedules []*velerov1.Schedule + } + tests := []struct { + name string + args args + want []*AnalyzeResult + }{ + { + name: "no schedules", + args: args{ + schedules: []*velerov1.Schedule{}, + }, + want: []*AnalyzeResult{}, + }, + { + name: "schedules configured", + args: args{ + schedules: []*velerov1.Schedule{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "daily-backup", + Namespace: "velero", + }, + Spec: velerov1.ScheduleSpec{ + Schedule: "0 0 * * *", + Template: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{ + "default", + }, + IncludedResources: []string{ + "*", + }, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "velero", + }, + }, + }, + }, + Status: velerov1.ScheduleStatus{ + Phase: velerov1.SchedulePhaseEnabled, + LastBackup: &metav1.Time{ + Time: time.Date(2023, 3, 8, 15, 0, 16, 0, time.UTC), + }, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Velero Schedules", + Message: "Found 1 schedules", + IsPass: true, + }, + }, + }, + { + name: "schedules - failures", + args: args{ + schedules: []*velerov1.Schedule{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "daily-backup", + Namespace: "velero", + }, + Spec: velerov1.ScheduleSpec{ + Schedule: "0 0 * * *", + Template: velerov1.BackupSpec{ + StorageLocation: "default", + IncludedNamespaces: []string{ + "default", + }, + IncludedResources: []string{ + "*", + }, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "velero", + }, + }, + }, + }, + Status: velerov1.ScheduleStatus{ + Phase: velerov1.SchedulePhaseFailedValidation, + LastBackup: &metav1.Time{ + Time: time.Date(2023, 3, 8, 15, 0, 16, 0, time.UTC), + }, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Schedule daily-backup", + Message: "Schedule daily-backup phase is FailedValidation", + IsFail: true, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := analyzeSchedules(tt.args.schedules); !reflect.DeepEqual(got, tt.want) { + t.Errorf("analyzeSchedules() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAnalyzeVelero_VolumeSnapshotLocations(t *testing.T) { + type args struct { + volumeSnapshotLocations []*velerov1.VolumeSnapshotLocation + } + tests := []struct { + name string + args args + want []*AnalyzeResult + }{ + { + name: "no volume snapshot locations", + args: args{ + volumeSnapshotLocations: []*velerov1.VolumeSnapshotLocation{}, + }, + want: []*AnalyzeResult{}, + }, + { + name: "volume snapshot locations configured", + args: args{ + volumeSnapshotLocations: []*velerov1.VolumeSnapshotLocation{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "velero", + }, + Spec: velerov1.VolumeSnapshotLocationSpec{ + Provider: "aws", + Config: map[string]string{ + "region": "us-east-1", + }, + }, + Status: velerov1.VolumeSnapshotLocationStatus{ + Phase: velerov1.VolumeSnapshotLocationPhaseAvailable, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Velero Volume Snapshot Locations", + Message: "Found 1 volume snapshot locations", + IsPass: true, + }, + }, + }, + { + name: "volume snapshot locations - failures", + args: args{ + volumeSnapshotLocations: []*velerov1.VolumeSnapshotLocation{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Namespace: "velero", + }, + Spec: velerov1.VolumeSnapshotLocationSpec{ + Provider: "aws", + Config: map[string]string{ + "region": "us-east-1", + }, + }, + Status: velerov1.VolumeSnapshotLocationStatus{ + Phase: velerov1.VolumeSnapshotLocationPhaseUnavailable, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "Volume Snapshot Location default", + Message: "Volume Snapshot Location default phase is Unavailable", + IsFail: true, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := analyzeVolumeSnapshotLocations(tt.args.volumeSnapshotLocations); !reflect.DeepEqual(got, tt.want) { + t.Errorf("analyzeVolumeSnapshotLocations() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAnalyzeVelero_Logs(t *testing.T) { + type args struct { + logs map[string][]byte + } + tests := []struct { + name string + args args + want []*AnalyzeResult + }{ + { + name: "no logs", + args: args{ + logs: map[string][]byte{}, + }, + want: []*AnalyzeResult{}, + }, + { + name: "logs - no errors", + args: args{ + logs: map[string][]byte{ + "node-agent-m6n9j": []byte("level=info msg=... backup=velero/sample-app controller=podvolumebacku"), + }, + }, + want: []*AnalyzeResult{ + { + Title: "Velero Logs analysis", + Message: "Found 1 log files", + IsPass: true, + }, + }, + }, + { + name: "logs - errors", + args: args{ + logs: map[string][]byte{ + "node-agent-m6n9j": []byte("level=error msg=... backup=velero/sample-app controller=podvolumebacku"), + }, + }, + want: []*AnalyzeResult{ + { + Title: "Velero logs for pod [node-agent]", + Message: "Found error|panic|fatal in node-agent* pod log file(s)", + IsWarn: true, + }, + { + Title: "Velero Logs analysis", + Message: "Found 1 log files", + IsPass: true, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := analyzeLogs(tt.args.logs); !reflect.DeepEqual(got, tt.want) { + t.Errorf("analyzeLogs() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAnalyzeVelero_Results(t *testing.T) { + type args struct { + results []*AnalyzeResult + } + tests := []struct { + name string + args args + want []*AnalyzeResult + }{ + { + name: "no results", + args: args{ + results: []*AnalyzeResult{}, + }, + want: []*AnalyzeResult{}, + }, + { + name: "results - pass", + args: args{ + results: []*AnalyzeResult{ + { + Title: "random Velero CRD check", + IsPass: true, + Message: "CRD status is healthy", + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "random Velero CRD check", + IsPass: true, + Message: "CRD status is healthy", + }, + { + Title: "Velero Status", + IsPass: true, + Message: "Velero setup is healthy", + }, + }, + }, + { + name: "results - fail", + args: args{ + results: []*AnalyzeResult{ + { + Title: "random Velero CRD check failure", + IsFail: true, + Message: "CRD status - Failed", + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "random Velero CRD check failure", + IsFail: true, + Message: "CRD status - Failed", + }, + { + Title: "Velero Status", + IsWarn: true, + Message: "Velero setup is not entirely healthy", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := aggregateResults(tt.args.results); !reflect.DeepEqual(got, tt.want) { + t.Errorf("aggregateResults() = %v, want %v", got, tt.want) + gotJSON, _ := json.MarshalIndent(got, "", " ") + wantJSON, _ := json.MarshalIndent(tt.want, "", " ") + t.Logf("\nGot: %s\nWant: %s", gotJSON, wantJSON) + } + }) + } +} From ca6c46d3b7c91327a2042e8c424ae4a2c60991bd Mon Sep 17 00:00:00 2001 From: Archit Sharma Date: Fri, 3 Nov 2023 08:28:59 +0530 Subject: [PATCH 5/9] refactor for namespace and restric repo Signed-off-by: Archit Sharma --- pkg/analyze/velero.go | 87 ++++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 35 deletions(-) diff --git a/pkg/analyze/velero.go b/pkg/analyze/velero.go index c38973e9e..3653eae64 100644 --- a/pkg/analyze/velero.go +++ b/pkg/analyze/velero.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) const ( @@ -52,7 +53,7 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze excludeFiles := []string{} // get backuprepositories.velero.io - backupRepositoriesDir := GetVeleroBackupRepositoriesDirectory(ns) + backupRepositoriesDir := GetVeleroBackupRepositoriesDirectory() backupRepositoriesGlob := filepath.Join(backupRepositoriesDir, "*.json") backupRepositoriesJson, err := findFiles(backupRepositoriesGlob, excludeFiles) if err != nil { @@ -68,8 +69,24 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze backupRepositories = append(backupRepositories, backupRepositoryArray...) } + // old velero (v1.9.x) has a BackupRepositoryTypeRestic + // get resticrepositories.velero.io + resticRepositoriesDir := GetVeleroResticRepositoriesDirectory() + resticRepositoriesGlob := filepath.Join(resticRepositoriesDir, "*.json") + resticRepositoriesJson, err := findFiles(resticRepositoriesGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero restic repositories files under %s", resticRepositoriesDir) + } + resticRepositories := []unstructured.Unstructured{} + for key, resticRepositoryJson := range resticRepositoriesJson { + err := json.Unmarshal(resticRepositoryJson, &resticRepositories) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal restic repository json from %s", key) + } + } + // get backups.velero.io - backupsDir := GetVeleroBackupsDirectory(ns) + backupsDir := GetVeleroBackupsDirectory() backupsGlob := filepath.Join(backupsDir, "*.json") veleroJSONs, err := findFiles(backupsGlob, excludeFiles) if err != nil { @@ -89,7 +106,7 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze } // get backupstoragelocations.velero.io - backupStorageLocationsDir := GetVeleroBackupStorageLocationsDirectory(ns) + backupStorageLocationsDir := GetVeleroBackupStorageLocationsDirectory() backupStorageLocationsGlob := filepath.Join(backupStorageLocationsDir, "*.json") backupStorageLocationsJson, err := findFiles(backupStorageLocationsGlob, excludeFiles) if err != nil { @@ -106,7 +123,7 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze } // get deletebackuprequests.velero.io - deleteBackupRequestsDir := GetVeleroDeleteBackupRequestsDirectory(ns) + deleteBackupRequestsDir := GetVeleroDeleteBackupRequestsDirectory() deleteBackupRequestsGlob := filepath.Join(deleteBackupRequestsDir, "*.json") deleteBackupRequestsJson, err := findFiles(deleteBackupRequestsGlob, excludeFiles) if err != nil { @@ -123,7 +140,7 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze } // get podvolumebackups.velero.io - podVolumeBackupsDir := GetVeleroPodVolumeBackupsDirectory(ns) + podVolumeBackupsDir := GetVeleroPodVolumeBackupsDirectory() podVolumeBackupsGlob := filepath.Join(podVolumeBackupsDir, "*.json") podVolumeBackupsJson, err := findFiles(podVolumeBackupsGlob, excludeFiles) if err != nil { @@ -140,7 +157,7 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze } // get podvolumerestores.velero.io - podVolumeRestoresDir := GetVeleroPodVolumeRestoresDirectory(ns) + podVolumeRestoresDir := GetVeleroPodVolumeRestoresDirectory() podVolumeRestoresGlob := filepath.Join(podVolumeRestoresDir, "*.json") podVolumeRestoresJson, err := findFiles(podVolumeRestoresGlob, excludeFiles) if err != nil { @@ -157,7 +174,7 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze } // get restores.velero.io - restoresDir := GetVeleroRestoresDirectory(ns) + restoresDir := GetVeleroRestoresDirectory() restoresGlob := filepath.Join(restoresDir, "*.json") restoresJson, err := findFiles(restoresGlob, excludeFiles) if err != nil { @@ -174,7 +191,7 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze } // get schedules.velero.io - schedulesDir := GetVeleroSchedulesDirectory(ns) + schedulesDir := GetVeleroSchedulesDirectory() schedulesGlob := filepath.Join(schedulesDir, "*.json") schedulesJson, err := findFiles(schedulesGlob, excludeFiles) if err != nil { @@ -191,7 +208,7 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze } // get serverstatusrequests.velero.io - serverStatusRequestsDir := GetVeleroServerStatusRequestsDirectory(ns) + serverStatusRequestsDir := GetVeleroServerStatusRequestsDirectory() serverStatusRequestsGlob := filepath.Join(serverStatusRequestsDir, "*.json") serverStatusRequestsJson, err := findFiles(serverStatusRequestsGlob, excludeFiles) if err != nil { @@ -208,7 +225,7 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze } // get volumesnapshotlocations.velero.io - volumeSnapshotLocationsDir := GetVeleroVolumeSnapshotLocationsDirectory(ns) + volumeSnapshotLocationsDir := GetVeleroVolumeSnapshotLocationsDirectory() volumeSnapshotLocationsGlob := filepath.Join(volumeSnapshotLocationsDir, "*.json") volumeSnapshotLocationsJson, err := findFiles(volumeSnapshotLocationsGlob, excludeFiles) if err != nil { @@ -578,54 +595,54 @@ func aggregateResults(results []*AnalyzeResult) []*AnalyzeResult { return out } -func GetVeleroBackupsDirectory(namespace string) string { - return fmt.Sprintf("cluster-resources/custom-resources/backups.velero.io") +func GetVeleroBackupsDirectory() string { + return "cluster-resources/custom-resources/backups.velero.io" } -func GetVeleroBackupRepositoriesDirectory(namespace string) string { - return fmt.Sprintf("cluster-resources/custom-resources/backuprepositories.velero.io") +func GetVeleroBackupRepositoriesDirectory() string { + return "cluster-resources/custom-resources/backuprepositories.velero.io" } -func GetVeleroBackupStorageLocationsDirectory(namespace string) string { - return fmt.Sprintf("cluster-resources/custom-resources/backupstoragelocations.velero.io") +func GetVeleroBackupStorageLocationsDirectory() string { + return "cluster-resources/custom-resources/backupstoragelocations.velero.io" } -func GetVeleroDeleteBackupRequestsDirectory(namespace string) string { - return fmt.Sprintf("cluster-resources/custom-resources/deletebackuprequests.velero.io") +func GetVeleroDeleteBackupRequestsDirectory() string { + return "cluster-resources/custom-resources/deletebackuprequests.velero.io" } -func GetVeleroDownloadRequestsDirectory(namespace string) string { - return fmt.Sprintf("cluster-resources/custom-resources/downloadrequests.velero.io") +func GetVeleroDownloadRequestsDirectory() string { + return "cluster-resources/custom-resources/downloadrequests.velero.io" } func GetVeleroLogsDirectory(namespace string) string { - return fmt.Sprintf("velero/logs") + return fmt.Sprint("%s/logs", namespace) } -func GetVeleroPodVolumeBackupsDirectory(namespace string) string { - return fmt.Sprintf("cluster-resources/custom-resources/podvolumebackups.velero.io") +func GetVeleroPodVolumeBackupsDirectory() string { + return "cluster-resources/custom-resources/podvolumebackups.velero.io" } -func GetVeleroPodVolumeRestoresDirectory(namespace string) string { - return fmt.Sprintf("cluster-resources/custom-resources/podvolumerestores.velero.io") +func GetVeleroPodVolumeRestoresDirectory() string { + return "cluster-resources/custom-resources/podvolumerestores.velero.io" } -func GetVeleroRestoresDirectory(namespace string) string { - return fmt.Sprintf("cluster-resources/custom-resources/restores.velero.io") +func GetVeleroRestoresDirectory() string { + return "cluster-resources/custom-resources/restores.velero.io" } -func GetVeleroSchedulesDirectory(namespace string) string { - return fmt.Sprintf("cluster-resources/custom-resources/schedules.velero.io") +func GetVeleroSchedulesDirectory() string { + return "cluster-resources/custom-resources/schedules.velero.io" } -func GetVeleroServerStatusRequestsDirectory(namespace string) string { - return fmt.Sprintf("cluster-resources/custom-resources/serverstatusrequests.velero.io") +func GetVeleroServerStatusRequestsDirectory() string { + return "cluster-resources/custom-resources/serverstatusrequests.velero.io" } -func GetVeleroVolumeSnapshotLocationsDirectory(namespace string) string { - return fmt.Sprintf("cluster-resources/custom-resources/volumesnapshotlocations.velero.io") +func GetVeleroVolumeSnapshotLocationsDirectory() string { + return "cluster-resources/custom-resources/volumesnapshotlocations.velero.io" } -func GetVeleroResticRepositoriesDirectory(namespace string) string { - return fmt.Sprintf("cluster-resources/custom-resources/resticrepositories.velero.io") +func GetVeleroResticRepositoriesDirectory() string { + return "cluster-resources/custom-resources/resticrepositories.velero.io" } From 20f8290c4234d0e90686891dbd43cdb51df1818a Mon Sep 17 00:00:00 2001 From: Archit Sharma Date: Fri, 3 Nov 2023 14:52:44 +0530 Subject: [PATCH 6/9] we can afford an extra (now) deprecated Restic repository type ..better than using unstructured and be the deer in headlights Signed-off-by: Archit Sharma --- pkg/analyze/types/restic_types.go | 62 +++++++++++++++++++++++++ pkg/analyze/velero.go | 67 ++++++++++++++++++++------- pkg/analyze/velero_test.go | 74 +++++++++++++++++++++++++++--- pkg/supportbundle/test/velero.yaml | 3 +- pkg/types/types.go | 4 +- 5 files changed, 183 insertions(+), 27 deletions(-) create mode 100644 pkg/analyze/types/restic_types.go diff --git a/pkg/analyze/types/restic_types.go b/pkg/analyze/types/restic_types.go new file mode 100644 index 000000000..2161e04e4 --- /dev/null +++ b/pkg/analyze/types/restic_types.go @@ -0,0 +1,62 @@ +package analyzer + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ResticRepositorySpec is the specification for a ResticRepository. +type ResticRepositorySpec struct { + // VolumeNamespace is the namespace this restic repository contains + // pod volume backups for. + VolumeNamespace string `json:"volumeNamespace"` + + // BackupStorageLocation is the name of the BackupStorageLocation + // that should contain this repository. + BackupStorageLocation string `json:"backupStorageLocation"` + + // ResticIdentifier is the full restic-compatible string for identifying + // this repository. + ResticIdentifier string `json:"resticIdentifier"` + + // MaintenanceFrequency is how often maintenance should be run. + MaintenanceFrequency metav1.Duration `json:"maintenanceFrequency"` +} + +// ResticRepositoryPhase represents the lifecycle phase of a ResticRepository. +// +kubebuilder:validation:Enum=New;Ready;NotReady +type ResticRepositoryPhase string + +const ( + ResticRepositoryPhaseNew ResticRepositoryPhase = "New" + ResticRepositoryPhaseReady ResticRepositoryPhase = "Ready" + ResticRepositoryPhaseNotReady ResticRepositoryPhase = "NotReady" +) + +// ResticRepositoryStatus is the current status of a ResticRepository. +type ResticRepositoryStatus struct { + // Phase is the current state of the ResticRepository. + // +optional + Phase ResticRepositoryPhase `json:"phase,omitempty"` + + // Message is a message about the current status of the ResticRepository. + // +optional + Message string `json:"message,omitempty"` + + // LastMaintenanceTime is the last time maintenance was run. + // +optional + // +nullable + LastMaintenanceTime *metav1.Time `json:"lastMaintenanceTime,omitempty"` +} + +type ResticRepository struct { + metav1.TypeMeta `json:",inline"` + + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // +optional + Spec ResticRepositorySpec `json:"spec,omitempty"` + + // +optional + Status ResticRepositoryStatus `json:"status,omitempty"` +} diff --git a/pkg/analyze/velero.go b/pkg/analyze/velero.go index 3653eae64..df96f920d 100644 --- a/pkg/analyze/velero.go +++ b/pkg/analyze/velero.go @@ -6,14 +6,11 @@ import ( "path/filepath" "strings" + restic_types "github.com/replicatedhq/troubleshoot/pkg/analyze/types" + "github.com/pkg/errors" troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" -) - -const ( - DefaultVeleroNamespace = "velero" ) type AnalyzeVelero struct { @@ -45,11 +42,6 @@ func (a *AnalyzeVelero) Analyze(getFile getCollectedFileContents, findFiles getC } func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze, getFileContents getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) { - ns := DefaultVeleroNamespace - if analyzer.Namespace != "" { - ns = analyzer.Namespace - } - excludeFiles := []string{} // get backuprepositories.velero.io @@ -77,12 +69,14 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze if err != nil { return nil, errors.Wrapf(err, "failed to find velero restic repositories files under %s", resticRepositoriesDir) } - resticRepositories := []unstructured.Unstructured{} + resticRepositories := []*restic_types.ResticRepository{} for key, resticRepositoryJson := range resticRepositoriesJson { + var resticRepositoryArray []*restic_types.ResticRepository err := json.Unmarshal(resticRepositoryJson, &resticRepositories) if err != nil { return nil, errors.Wrapf(err, "failed to unmarshal restic repository json from %s", key) } + resticRepositories = append(resticRepositories, resticRepositoryArray...) } // get backups.velero.io @@ -241,7 +235,7 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze volumeSnapshotLocations = append(volumeSnapshotLocations, volumeSnapshotLocationArray...) } - logsDir := GetVeleroLogsDirectory(ns) + logsDir := GetVeleroLogsDirectory() logsGlob := filepath.Join(logsDir, "node-agent*", "*.log") logs, err := findFiles(logsGlob, excludeFiles) @@ -252,6 +246,7 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze results := []*AnalyzeResult{} results = append(results, analyzeLogs(logs)...) results = append(results, analyzeBackupRepositories(backupRepositories)...) + results = append(results, analyzeResticRepositories(resticRepositories)...) results = append(results, analyzeBackups(backups)...) results = append(results, analyzeBackupStorageLocations(backupStorageLocations)...) results = append(results, analyzeDeleteBackupRequests(deleteBackupRequests)...) @@ -268,7 +263,7 @@ func analyzeBackupRepositories(backupRepositories []*velerov1.BackupRepository) results := []*AnalyzeResult{} readyCount := 0 backupRepositoriesResult := &AnalyzeResult{ - Title: "At least 1 Velero Backup Repository configured", + Title: "At least 1 Backup Repository configured", } if len(backupRepositories) == 0 { backupRepositoriesResult.IsFail = true @@ -295,11 +290,49 @@ func analyzeBackupRepositories(backupRepositories []*velerov1.BackupRepository) } } results = append(results, backupRepositoriesResult) - return results } +func analyzeResticRepositories(resticRepositories []*restic_types.ResticRepository) []*AnalyzeResult { + results := []*AnalyzeResult{} + readyCount := 0 + resticRepositoriesResult := &AnalyzeResult{ + Title: "At least 1 Restic Repository configured", + } + if len(resticRepositories) == 0 { + resticRepositoriesResult.IsFail = true + resticRepositoriesResult.Message = "No restic repositories configured" + } else { + for _, resticRepository := range resticRepositories { + // phase, _, err := unstructured.NestedString(resticRepository.Object, "status", "phase") + // if err != nil { + // klog.V(2).Infof("Failed to get phase for restic repository %s: %v", resticRepository.GetName(), err) + // } + // if phase != "Ready" { + if resticRepository.Status.Phase != restic_types.ResticRepositoryPhaseReady { + result := &AnalyzeResult{ + Title: fmt.Sprintf("Restic Repository %s", resticRepository.GetName()), + } + result.Message = fmt.Sprintf("Restic Repository [%s] is in phase %s", resticRepository.Name, resticRepository.Status.Phase) + result.IsWarn = true + results = append(results, result) + } else { + readyCount++ + } + } + if readyCount > 0 { + resticRepositoriesResult.IsPass = true + resticRepositoriesResult.Message = fmt.Sprintf("Found %d restic repositories configured and %d Ready", len(resticRepositories), readyCount) + } else { + resticRepositoriesResult.IsWarn = true + resticRepositoriesResult.Message = fmt.Sprintf("Found %d configured restic repositories, but none are ready", len(resticRepositories)) + } + } + results = append(results, resticRepositoriesResult) + return results +} + func analyzeBackups(backups []*velerov1.Backup) []*AnalyzeResult { results := []*AnalyzeResult{} @@ -337,7 +370,7 @@ func analyzeBackupStorageLocations(backupStorageLocations []*velerov1.BackupStor results := []*AnalyzeResult{} availableCount := 0 bslResult := &AnalyzeResult{ - Title: "At least 1 Velero Backup Storage Location configured", + Title: "At least 1 Backup Storage Location configured", } if len(backupStorageLocations) == 0 { @@ -615,8 +648,8 @@ func GetVeleroDownloadRequestsDirectory() string { return "cluster-resources/custom-resources/downloadrequests.velero.io" } -func GetVeleroLogsDirectory(namespace string) string { - return fmt.Sprint("%s/logs", namespace) +func GetVeleroLogsDirectory() string { + return "velero/logs" } func GetVeleroPodVolumeBackupsDirectory() string { diff --git a/pkg/analyze/velero_test.go b/pkg/analyze/velero_test.go index 92d6487cf..337ec649b 100644 --- a/pkg/analyze/velero_test.go +++ b/pkg/analyze/velero_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + restic_types "github.com/replicatedhq/troubleshoot/pkg/analyze/types" velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,7 +28,7 @@ func TestAnalyzeVelero_BackupRepositories(t *testing.T) { }, want: []*AnalyzeResult{ { - Title: "At least 1 Velero Backup Repository configured", + Title: "At least 1 Backup Repository configured", Message: "No backup repositories configured", IsFail: true, }, @@ -50,7 +51,7 @@ func TestAnalyzeVelero_BackupRepositories(t *testing.T) { }, want: []*AnalyzeResult{ { - Title: "At least 1 Velero Backup Repository configured", + Title: "At least 1 Backup Repository configured", Message: "Found 1 backup repositories configured and 1 Ready", IsPass: true, }, @@ -95,7 +96,7 @@ func TestAnalyzeVelero_BackupRepositories(t *testing.T) { IsWarn: true, }, { - Title: "At least 1 Velero Backup Repository configured", + Title: "At least 1 Backup Repository configured", Message: "Found 2 backup repositories configured and 1 Ready", IsPass: true, }, @@ -127,7 +128,7 @@ func TestAnalyzeVelero_BackupRepositories(t *testing.T) { IsWarn: true, }, { - Title: "At least 1 Velero Backup Repository configured", + Title: "At least 1 Backup Repository configured", Message: "Found 1 configured backup repositories, but none are ready", IsWarn: true, }, @@ -143,6 +144,65 @@ func TestAnalyzeVelero_BackupRepositories(t *testing.T) { } } +func TestAnalyzeVelero_ResticRepositories(t *testing.T) { + type args struct { + resticRepositories []*restic_types.ResticRepository + } + tests := []struct { + name string + args args + want []*AnalyzeResult + }{ + { + name: "no restic repositories", + args: args{ + resticRepositories: []*restic_types.ResticRepository{}, + }, + want: []*AnalyzeResult{ + { + Title: "At least 1 Restic Repository configured", + Message: "No restic repositories configured", + IsFail: true, + }, + }, + }, + { + name: "1 restic repository and 1 Ready", + args: args{ + resticRepositories: []*restic_types.ResticRepository{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "default-default-restic-245sd", + Namespace: "velero", + }, + Spec: restic_types.ResticRepositorySpec{ + BackupStorageLocation: "default", + VolumeNamespace: "velero", + }, + Status: restic_types.ResticRepositoryStatus{ + Phase: restic_types.ResticRepositoryPhaseReady, + }, + }, + }, + }, + want: []*AnalyzeResult{ + { + Title: "At least 1 Restic Repository configured", + Message: "Found 1 restic repositories configured and 1 Ready", + IsPass: true, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := analyzeResticRepositories(tt.args.resticRepositories); !reflect.DeepEqual(got, tt.want) { + t.Errorf("analyzeResticRepositories() = %v, want %v", got, tt.want) + } + }) + } +} + func TestAnalyzeVelero_Backups(t *testing.T) { type args struct { backups []*velerov1.Backup @@ -242,7 +302,7 @@ func TestAnalyzeVelero_BackupStorageLocations(t *testing.T) { }, want: []*AnalyzeResult{ { - Title: "At least 1 Velero Backup Storage Location configured", + Title: "At least 1 Backup Storage Location configured", Message: "No backup storage locations configured", IsFail: true, }, @@ -268,7 +328,7 @@ func TestAnalyzeVelero_BackupStorageLocations(t *testing.T) { }, want: []*AnalyzeResult{ { - Title: "At least 1 Velero Backup Storage Location configured", + Title: "At least 1 Backup Storage Location configured", Message: "Found 1 backup storage locations configured and 1 Available", IsPass: true, }, @@ -299,7 +359,7 @@ func TestAnalyzeVelero_BackupStorageLocations(t *testing.T) { IsWarn: true, }, { - Title: "At least 1 Velero Backup Storage Location configured", + Title: "At least 1 Backup Storage Location configured", Message: "Found 1 configured backup storage locations, but none are available", IsWarn: true, }, diff --git a/pkg/supportbundle/test/velero.yaml b/pkg/supportbundle/test/velero.yaml index c684db427..18abb23a3 100644 --- a/pkg/supportbundle/test/velero.yaml +++ b/pkg/supportbundle/test/velero.yaml @@ -8,5 +8,4 @@ spec: namespace: velero name: velero/logs analyzers: - - velero: - strict: true + - velero: {} \ No newline at end of file diff --git a/pkg/types/types.go b/pkg/types/types.go index 3809912c4..d710362a7 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -1,6 +1,8 @@ package types -import "fmt" +import ( + "fmt" +) type NotFoundError struct { Name string From d892bd876f9a7e7033e4e276ad6f2e59b5ecf52e Mon Sep 17 00:00:00 2001 From: Archit Sharma Date: Fri, 3 Nov 2023 16:02:05 +0530 Subject: [PATCH 7/9] use versions from deployment to check deprecated type Signed-off-by: Archit Sharma --- pkg/analyze/velero.go | 115 +++++++++++++++++++++-------- pkg/supportbundle/test/velero.yaml | 2 +- 2 files changed, 87 insertions(+), 30 deletions(-) diff --git a/pkg/analyze/velero.go b/pkg/analyze/velero.go index df96f920d..a9b7af147 100644 --- a/pkg/analyze/velero.go +++ b/pkg/analyze/velero.go @@ -6,7 +6,10 @@ import ( "path/filepath" "strings" + appsV1 "k8s.io/api/apps/v1" + restic_types "github.com/replicatedhq/troubleshoot/pkg/analyze/types" + "golang.org/x/mod/semver" "github.com/pkg/errors" troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2" @@ -43,40 +46,67 @@ func (a *AnalyzeVelero) Analyze(getFile getCollectedFileContents, findFiles getC func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze, getFileContents getCollectedFileContents, findFiles getChildCollectedFileContents) ([]*AnalyzeResult, error) { excludeFiles := []string{} + results := []*AnalyzeResult{} - // get backuprepositories.velero.io - backupRepositoriesDir := GetVeleroBackupRepositoriesDirectory() - backupRepositoriesGlob := filepath.Join(backupRepositoriesDir, "*.json") - backupRepositoriesJson, err := findFiles(backupRepositoriesGlob, excludeFiles) + oldVeleroRepoType := false + veleroVersion, err := getVeleroVersion(excludeFiles, findFiles) if err != nil { - return nil, errors.Wrapf(err, "failed to find velero backup repositories files under %s", backupRepositoriesDir) + return nil, errors.Wrap(err, "Unable to find velero deployment") } - backupRepositories := []*velerov1.BackupRepository{} - for key, backupRepositoryJson := range backupRepositoriesJson { - var backupRepositoryArray []*velerov1.BackupRepository - err := json.Unmarshal(backupRepositoryJson, &backupRepositoryArray) - if err != nil { - return nil, errors.Wrapf(err, "failed to unmarshal backup repository json from %s", key) - } - backupRepositories = append(backupRepositories, backupRepositoryArray...) + + // Check if the version string is valid erer + if !semver.IsValid(veleroVersion) { + return nil, errors.Errorf("Invalid velero semver: %s", veleroVersion) } - // old velero (v1.9.x) has a BackupRepositoryTypeRestic - // get resticrepositories.velero.io - resticRepositoriesDir := GetVeleroResticRepositoriesDirectory() - resticRepositoriesGlob := filepath.Join(resticRepositoriesDir, "*.json") - resticRepositoriesJson, err := findFiles(resticRepositoriesGlob, excludeFiles) - if err != nil { - return nil, errors.Wrapf(err, "failed to find velero restic repositories files under %s", resticRepositoriesDir) + // check if veleroVersion is less than 1.10.x + compareResult := semver.Compare(veleroVersion, "1.10.0") + if compareResult < 0 { + fmt.Printf("Version %s is less than %s\n", veleroVersion, "1.10.0") + oldVeleroRepoType = true } - resticRepositories := []*restic_types.ResticRepository{} - for key, resticRepositoryJson := range resticRepositoriesJson { - var resticRepositoryArray []*restic_types.ResticRepository - err := json.Unmarshal(resticRepositoryJson, &resticRepositories) + + if oldVeleroRepoType == true { + // old velero (v1.9.x) has a BackupRepositoryTypeRestic + // get resticrepositories.velero.io + resticRepositoriesDir := GetVeleroResticRepositoriesDirectory() + resticRepositoriesGlob := filepath.Join(resticRepositoriesDir, "*.json") + resticRepositoriesJson, err := findFiles(resticRepositoriesGlob, excludeFiles) if err != nil { - return nil, errors.Wrapf(err, "failed to unmarshal restic repository json from %s", key) + return nil, errors.Wrapf(err, "failed to find velero restic repositories files under %s", resticRepositoriesDir) + } + resticRepositories := []*restic_types.ResticRepository{} + for key, resticRepositoryJson := range resticRepositoriesJson { + var resticRepositoryArray []*restic_types.ResticRepository + err := json.Unmarshal(resticRepositoryJson, &resticRepositories) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal restic repository json from %s", key) + } + resticRepositories = append(resticRepositories, resticRepositoryArray...) + } + results = append(results, analyzeResticRepositories(resticRepositories)...) + + } else { + + // velerov1.Version + // get backuprepositories.velero.io + backupRepositoriesDir := GetVeleroBackupRepositoriesDirectory() + backupRepositoriesGlob := filepath.Join(backupRepositoriesDir, "*.json") + backupRepositoriesJson, err := findFiles(backupRepositoriesGlob, excludeFiles) + if err != nil { + return nil, errors.Wrapf(err, "failed to find velero backup repositories files under %s", backupRepositoriesDir) + } + backupRepositories := []*velerov1.BackupRepository{} + for key, backupRepositoryJson := range backupRepositoriesJson { + var backupRepositoryArray []*velerov1.BackupRepository + err := json.Unmarshal(backupRepositoryJson, &backupRepositoryArray) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal backup repository json from %s", key) + } + backupRepositories = append(backupRepositories, backupRepositoryArray...) } - resticRepositories = append(resticRepositories, resticRepositoryArray...) + results = append(results, analyzeBackupRepositories(backupRepositories)...) + } // get backups.velero.io @@ -243,10 +273,7 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze return nil, errors.Wrapf(err, "failed to find velero logs files under %s", logsDir) } - results := []*AnalyzeResult{} results = append(results, analyzeLogs(logs)...) - results = append(results, analyzeBackupRepositories(backupRepositories)...) - results = append(results, analyzeResticRepositories(resticRepositories)...) results = append(results, analyzeBackups(backups)...) results = append(results, analyzeBackupStorageLocations(backupStorageLocations)...) results = append(results, analyzeDeleteBackupRequests(deleteBackupRequests)...) @@ -628,6 +655,36 @@ func aggregateResults(results []*AnalyzeResult) []*AnalyzeResult { return out } +func getVeleroVersion(excludedFiles []string, findFiles getChildCollectedFileContents) (string, error) { + veleroDeploymentDir := "cluster-resources/deployments" + veleroVersion := "" + veleroDeploymentGlob := filepath.Join(veleroDeploymentDir, "velero.json") + veleroDeploymentJson, err := findFiles(veleroDeploymentGlob, excludedFiles) + if err != nil { + return "", errors.Wrapf(err, "failed to find velero deployment file under %s", veleroDeploymentDir) + } + var deploymentList *appsV1.DeploymentList + // should run only once + for key, veleroDeploymentJsonBytes := range veleroDeploymentJson { + err := json.Unmarshal(veleroDeploymentJsonBytes, &deploymentList) + if err != nil { + return "", errors.Wrapf(err, "failed to unmarshal velero deployment json from %s", key) + } + break + } + for _, deployment := range deploymentList.Items { + for _, container := range deployment.Spec.Template.Spec.Containers { + if container.Name == "velero" { + container_image := container.Image + veleroVersion = strings.Split(container_image, ":")[1] + return veleroVersion, nil + } + } + } + + return "", errors.Errorf("Unable to get velero version. Could not find velero container in deployment!") +} + func GetVeleroBackupsDirectory() string { return "cluster-resources/custom-resources/backups.velero.io" } diff --git a/pkg/supportbundle/test/velero.yaml b/pkg/supportbundle/test/velero.yaml index 18abb23a3..41d89c9e5 100644 --- a/pkg/supportbundle/test/velero.yaml +++ b/pkg/supportbundle/test/velero.yaml @@ -8,4 +8,4 @@ spec: namespace: velero name: velero/logs analyzers: - - velero: {} \ No newline at end of file + - velero: {} From a0e1c8c22e41f4d97d472d9294df054fd57c2698 Mon Sep 17 00:00:00 2001 From: Archit Sharma Date: Fri, 3 Nov 2023 17:29:23 +0530 Subject: [PATCH 8/9] read for both velero pod kinds Signed-off-by: Archit Sharma --- pkg/analyze/velero.go | 21 ++++++++++++--------- pkg/analyze/velero_test.go | 35 +++++++++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/pkg/analyze/velero.go b/pkg/analyze/velero.go index a9b7af147..ab2493dd5 100644 --- a/pkg/analyze/velero.go +++ b/pkg/analyze/velero.go @@ -267,13 +267,16 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze logsDir := GetVeleroLogsDirectory() logsGlob := filepath.Join(logsDir, "node-agent*", "*.log") - logs, err := findFiles(logsGlob, excludeFiles) - + nodeAgentlogs, err := findFiles(logsGlob, excludeFiles) if err != nil { return nil, errors.Wrapf(err, "failed to find velero logs files under %s", logsDir) } - results = append(results, analyzeLogs(logs)...) + veleroLogsGlob := filepath.Join(logsDir, "velero*", "*.log") + veleroLogs, err := findFiles(veleroLogsGlob, excludeFiles) + + results = append(results, analyzeLogs(nodeAgentlogs, "node-agent*")...) + results = append(results, analyzeLogs(veleroLogs, "velero*")...) results = append(results, analyzeBackups(backups)...) results = append(results, analyzeBackupStorageLocations(backupStorageLocations)...) results = append(results, analyzeDeleteBackupRequests(deleteBackupRequests)...) @@ -595,30 +598,30 @@ func analyzeVolumeSnapshotLocations(volumeSnapshotLocations []*velerov1.VolumeSn return results } -func analyzeLogs(logs map[string][]byte) []*AnalyzeResult { +func analyzeLogs(logs map[string][]byte, kind string) []*AnalyzeResult { results := []*AnalyzeResult{} if len(logs) > 0 { - for _, logBytes := range logs { + for key, logBytes := range logs { logContent := string(logBytes) result := &AnalyzeResult{ - Title: fmt.Sprintf("Velero logs for pod [node-agent]"), + Title: fmt.Sprintf("Velero logs for pod [%s]", key), } if strings.Contains(logContent, "permission denied") { result.IsWarn = true - result.Message = fmt.Sprintf("Found 'permission denied' in node-agent* pod log file(s)") + result.Message = fmt.Sprintf("Found 'permission denied' in %s pod log file(s)", kind) results = append(results, result) continue } if strings.Contains(logContent, "error") || strings.Contains(logContent, "panic") || strings.Contains(logContent, "fatal") { result.IsWarn = true - result.Message = fmt.Sprintf("Found error|panic|fatal in node-agent* pod log file(s)") + result.Message = fmt.Sprintf("Found error|panic|fatal in %s pod log file(s)", kind) results = append(results, result) } } results = append(results, &AnalyzeResult{ - Title: "Velero Logs analysis", + Title: fmt.Sprintf("Velero Logs analysis for kind [%s]", kind), IsPass: true, Message: fmt.Sprintf("Found %d log files", len(logs)), }) diff --git a/pkg/analyze/velero_test.go b/pkg/analyze/velero_test.go index 337ec649b..c0d39c2b7 100644 --- a/pkg/analyze/velero_test.go +++ b/pkg/analyze/velero_test.go @@ -861,6 +861,7 @@ func TestAnalyzeVelero_VolumeSnapshotLocations(t *testing.T) { func TestAnalyzeVelero_Logs(t *testing.T) { type args struct { logs map[string][]byte + kind string } tests := []struct { name string @@ -871,39 +872,58 @@ func TestAnalyzeVelero_Logs(t *testing.T) { name: "no logs", args: args{ logs: map[string][]byte{}, + kind: "node-agent*", }, want: []*AnalyzeResult{}, }, { - name: "logs - no errors", + name: "logs - no errors in node-agent* pods", args: args{ logs: map[string][]byte{ "node-agent-m6n9j": []byte("level=info msg=... backup=velero/sample-app controller=podvolumebacku"), }, + kind: "node-agent*", }, want: []*AnalyzeResult{ { - Title: "Velero Logs analysis", + Title: "Velero Logs analysis for kind [node-agent*]", Message: "Found 1 log files", IsPass: true, }, }, }, { - name: "logs - errors", + name: "logs - no errors in velero* pods", + args: args{ + logs: map[string][]byte{ + "velero-788ff7c9dd-mslfl": []byte("level=info msg=BackupStorageLocations... controller=backup-storage-location"), + }, + kind: "velero*", + }, + want: []*AnalyzeResult{ + { + Title: "Velero Logs analysis for kind [velero*]", + Message: "Found 1 log files", + IsPass: true, + }, + }, + }, + { + name: "logs - errors in node-agent* pods", args: args{ logs: map[string][]byte{ "node-agent-m6n9j": []byte("level=error msg=... backup=velero/sample-app controller=podvolumebacku"), }, + kind: "node-agent*", }, want: []*AnalyzeResult{ { - Title: "Velero logs for pod [node-agent]", + Title: "Velero logs for pod [node-agent-m6n9j]", Message: "Found error|panic|fatal in node-agent* pod log file(s)", IsWarn: true, }, { - Title: "Velero Logs analysis", + Title: "Velero Logs analysis for kind [node-agent*]", Message: "Found 1 log files", IsPass: true, }, @@ -912,8 +932,11 @@ func TestAnalyzeVelero_Logs(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := analyzeLogs(tt.args.logs); !reflect.DeepEqual(got, tt.want) { + if got := analyzeLogs(tt.args.logs, tt.args.kind); !reflect.DeepEqual(got, tt.want) { t.Errorf("analyzeLogs() = %v, want %v", got, tt.want) + gotJSON, _ := json.MarshalIndent(got, "", " ") + wantJSON, _ := json.MarshalIndent(tt.want, "", " ") + t.Logf("\nGot: %s\nWant: %s", gotJSON, wantJSON) } }) } From 12110f20d2d85c984839780102dae46444570dbc Mon Sep 17 00:00:00 2001 From: Archit Sharma Date: Fri, 3 Nov 2023 18:31:42 +0530 Subject: [PATCH 9/9] cleanup; generate schemas Signed-off-by: Archit Sharma --- config/crds/troubleshoot.sh_analyzers.yaml | 39 ------------ config/crds/troubleshoot.sh_preflights.yaml | 39 ------------ .../crds/troubleshoot.sh_supportbundles.yaml | 39 ------------ go.mod | 2 +- pkg/analyze/velero.go | 6 -- .../troubleshoot/v1beta2/analyzer_shared.go | 5 +- .../v1beta2/zz_generated.deepcopy.go | 11 ---- schemas/analyzer-troubleshoot-v1beta2.json | 60 ------------------- schemas/preflight-troubleshoot-v1beta2.json | 60 ------------------- .../supportbundle-troubleshoot-v1beta2.json | 60 ------------------- 10 files changed, 2 insertions(+), 319 deletions(-) diff --git a/config/crds/troubleshoot.sh_analyzers.yaml b/config/crds/troubleshoot.sh_analyzers.yaml index 4d506d1d9..982aaf093 100644 --- a/config/crds/troubleshoot.sh_analyzers.yaml +++ b/config/crds/troubleshoot.sh_analyzers.yaml @@ -1497,49 +1497,10 @@ spec: type: object checkName: type: string - collectorName: - type: string exclude: type: BoolString - namespace: - type: string - outcomes: - items: - properties: - fail: - properties: - message: - type: string - uri: - type: string - when: - type: string - type: object - pass: - properties: - message: - type: string - uri: - type: string - when: - type: string - type: object - warn: - properties: - message: - type: string - uri: - type: string - when: - type: string - type: object - type: object - type: array strict: type: BoolString - required: - - namespace - - outcomes type: object weaveReport: properties: diff --git a/config/crds/troubleshoot.sh_preflights.yaml b/config/crds/troubleshoot.sh_preflights.yaml index 7b020c4c0..949711ae7 100644 --- a/config/crds/troubleshoot.sh_preflights.yaml +++ b/config/crds/troubleshoot.sh_preflights.yaml @@ -1497,49 +1497,10 @@ spec: type: object checkName: type: string - collectorName: - type: string exclude: type: BoolString - namespace: - type: string - outcomes: - items: - properties: - fail: - properties: - message: - type: string - uri: - type: string - when: - type: string - type: object - pass: - properties: - message: - type: string - uri: - type: string - when: - type: string - type: object - warn: - properties: - message: - type: string - uri: - type: string - when: - type: string - type: object - type: object - type: array strict: type: BoolString - required: - - namespace - - outcomes type: object weaveReport: properties: diff --git a/config/crds/troubleshoot.sh_supportbundles.yaml b/config/crds/troubleshoot.sh_supportbundles.yaml index efc6a70e3..405d62cc0 100644 --- a/config/crds/troubleshoot.sh_supportbundles.yaml +++ b/config/crds/troubleshoot.sh_supportbundles.yaml @@ -1528,49 +1528,10 @@ spec: type: object checkName: type: string - collectorName: - type: string exclude: type: BoolString - namespace: - type: string - outcomes: - items: - properties: - fail: - properties: - message: - type: string - uri: - type: string - when: - type: string - type: object - pass: - properties: - message: - type: string - uri: - type: string - when: - type: string - type: object - warn: - properties: - message: - type: string - uri: - type: string - when: - type: string - type: object - type: object - type: array strict: type: BoolString - required: - - namespace - - outcomes type: object weaveReport: properties: diff --git a/go.mod b/go.mod index 19f4c9d22..15610291b 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( go.opentelemetry.io/otel v1.19.0 go.opentelemetry.io/otel/sdk v1.19.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 + golang.org/x/mod v0.12.0 golang.org/x/sync v0.4.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.28.2 @@ -105,7 +106,6 @@ require ( go.opentelemetry.io/otel/metric v1.19.0 // indirect go.opentelemetry.io/otel/trace v1.19.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/mod v0.12.0 // indirect golang.org/x/tools v0.13.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect diff --git a/pkg/analyze/velero.go b/pkg/analyze/velero.go index ab2493dd5..2512f5dfa 100644 --- a/pkg/analyze/velero.go +++ b/pkg/analyze/velero.go @@ -62,7 +62,6 @@ func (a *AnalyzeVelero) veleroStatus(analyzer *troubleshootv1beta2.VeleroAnalyze // check if veleroVersion is less than 1.10.x compareResult := semver.Compare(veleroVersion, "1.10.0") if compareResult < 0 { - fmt.Printf("Version %s is less than %s\n", veleroVersion, "1.10.0") oldVeleroRepoType = true } @@ -335,11 +334,6 @@ func analyzeResticRepositories(resticRepositories []*restic_types.ResticReposito resticRepositoriesResult.Message = "No restic repositories configured" } else { for _, resticRepository := range resticRepositories { - // phase, _, err := unstructured.NestedString(resticRepository.Object, "status", "phase") - // if err != nil { - // klog.V(2).Infof("Failed to get phase for restic repository %s: %v", resticRepository.GetName(), err) - // } - // if phase != "Ready" { if resticRepository.Status.Phase != restic_types.ResticRepositoryPhaseReady { result := &AnalyzeResult{ Title: fmt.Sprintf("Restic Repository %s", resticRepository.GetName()), diff --git a/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go b/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go index a206ad7bd..62a2a2527 100644 --- a/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go +++ b/pkg/apis/troubleshoot/v1beta2/analyzer_shared.go @@ -187,10 +187,7 @@ type CephStatusAnalyze struct { } type VeleroAnalyze struct { - AnalyzeMeta `json:",inline" yaml:",inline"` - Outcomes []*Outcome `json:"outcomes" yaml:"outcomes"` - CollectorName string `json:"collectorName,omitempty" yaml:"collectorName,omitempty"` - Namespace string `json:"namespace" yaml:"namespace"` + AnalyzeMeta `json:",inline" yaml:",inline"` } type LonghornAnalyze struct { diff --git a/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go b/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go index 9b8219a30..9d388f7d6 100644 --- a/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go +++ b/pkg/apis/troubleshoot/v1beta2/zz_generated.deepcopy.go @@ -4663,17 +4663,6 @@ func (in *UDPPortStatusAnalyze) DeepCopy() *UDPPortStatusAnalyze { func (in *VeleroAnalyze) DeepCopyInto(out *VeleroAnalyze) { *out = *in in.AnalyzeMeta.DeepCopyInto(&out.AnalyzeMeta) - if in.Outcomes != nil { - in, out := &in.Outcomes, &out.Outcomes - *out = make([]*Outcome, len(*in)) - for i := range *in { - if (*in)[i] != nil { - in, out := &(*in)[i], &(*out)[i] - *out = new(Outcome) - (*in).DeepCopyInto(*out) - } - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VeleroAnalyze. diff --git a/schemas/analyzer-troubleshoot-v1beta2.json b/schemas/analyzer-troubleshoot-v1beta2.json index c07e25235..592a4c94e 100644 --- a/schemas/analyzer-troubleshoot-v1beta2.json +++ b/schemas/analyzer-troubleshoot-v1beta2.json @@ -2257,10 +2257,6 @@ }, "velero": { "type": "object", - "required": [ - "namespace", - "outcomes" - ], "properties": { "annotations": { "type": "object", @@ -2271,65 +2267,9 @@ "checkName": { "type": "string" }, - "collectorName": { - "type": "string" - }, "exclude": { "oneOf": [{"type": "string"},{"type": "boolean"}] }, - "namespace": { - "type": "string" - }, - "outcomes": { - "type": "array", - "items": { - "type": "object", - "properties": { - "fail": { - "type": "object", - "properties": { - "message": { - "type": "string" - }, - "uri": { - "type": "string" - }, - "when": { - "type": "string" - } - } - }, - "pass": { - "type": "object", - "properties": { - "message": { - "type": "string" - }, - "uri": { - "type": "string" - }, - "when": { - "type": "string" - } - } - }, - "warn": { - "type": "object", - "properties": { - "message": { - "type": "string" - }, - "uri": { - "type": "string" - }, - "when": { - "type": "string" - } - } - } - } - } - }, "strict": { "oneOf": [{"type": "string"},{"type": "boolean"}] } diff --git a/schemas/preflight-troubleshoot-v1beta2.json b/schemas/preflight-troubleshoot-v1beta2.json index fae3371c7..db19376bd 100644 --- a/schemas/preflight-troubleshoot-v1beta2.json +++ b/schemas/preflight-troubleshoot-v1beta2.json @@ -2257,10 +2257,6 @@ }, "velero": { "type": "object", - "required": [ - "namespace", - "outcomes" - ], "properties": { "annotations": { "type": "object", @@ -2271,65 +2267,9 @@ "checkName": { "type": "string" }, - "collectorName": { - "type": "string" - }, "exclude": { "oneOf": [{"type": "string"},{"type": "boolean"}] }, - "namespace": { - "type": "string" - }, - "outcomes": { - "type": "array", - "items": { - "type": "object", - "properties": { - "fail": { - "type": "object", - "properties": { - "message": { - "type": "string" - }, - "uri": { - "type": "string" - }, - "when": { - "type": "string" - } - } - }, - "pass": { - "type": "object", - "properties": { - "message": { - "type": "string" - }, - "uri": { - "type": "string" - }, - "when": { - "type": "string" - } - } - }, - "warn": { - "type": "object", - "properties": { - "message": { - "type": "string" - }, - "uri": { - "type": "string" - }, - "when": { - "type": "string" - } - } - } - } - } - }, "strict": { "oneOf": [{"type": "string"},{"type": "boolean"}] } diff --git a/schemas/supportbundle-troubleshoot-v1beta2.json b/schemas/supportbundle-troubleshoot-v1beta2.json index 14767b014..0fc868784 100644 --- a/schemas/supportbundle-troubleshoot-v1beta2.json +++ b/schemas/supportbundle-troubleshoot-v1beta2.json @@ -2303,10 +2303,6 @@ }, "velero": { "type": "object", - "required": [ - "namespace", - "outcomes" - ], "properties": { "annotations": { "type": "object", @@ -2317,65 +2313,9 @@ "checkName": { "type": "string" }, - "collectorName": { - "type": "string" - }, "exclude": { "oneOf": [{"type": "string"},{"type": "boolean"}] }, - "namespace": { - "type": "string" - }, - "outcomes": { - "type": "array", - "items": { - "type": "object", - "properties": { - "fail": { - "type": "object", - "properties": { - "message": { - "type": "string" - }, - "uri": { - "type": "string" - }, - "when": { - "type": "string" - } - } - }, - "pass": { - "type": "object", - "properties": { - "message": { - "type": "string" - }, - "uri": { - "type": "string" - }, - "when": { - "type": "string" - } - } - }, - "warn": { - "type": "object", - "properties": { - "message": { - "type": "string" - }, - "uri": { - "type": "string" - }, - "when": { - "type": "string" - } - } - } - } - } - }, "strict": { "oneOf": [{"type": "string"},{"type": "boolean"}] }