From 277fd338f886ca6b274c9b8b91938eae953dab83 Mon Sep 17 00:00:00 2001 From: Tarun Khandelwal Date: Mon, 30 May 2022 22:08:32 +0530 Subject: [PATCH 1/5] feat: add support for github in file scrapping --- .gitignore | 5 +- api/v1/file.go | 7 ++- api/v1/types.go | 2 +- db/update.go | 17 ++++-- filesystem/file.go | 6 +- filesystem/interface.go | 2 +- go.mod | 11 ++-- go.sum | 14 ++++- samples/file-git.yaml | 6 ++ samples/file-local.yaml | 5 ++ samples/test.yaml | 6 ++ scrapers/file/file.go | 115 ++++++++++++++++++++++++++--------- scrapers/runscrapers_test.go | 2 +- 13 files changed, 147 insertions(+), 51 deletions(-) create mode 100644 samples/file-git.yaml create mode 100644 samples/file-local.yaml create mode 100644 samples/test.yaml diff --git a/.gitignore b/.gitignore index 7c9c01d9..b20b5496 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ -/*.json +*.json .vscode _DS_Store .bin/ vendor -confighub \ No newline at end of file +confighub +scraped/ \ No newline at end of file diff --git a/api/v1/file.go b/api/v1/file.go index 1c30d79b..5fececb9 100644 --- a/api/v1/file.go +++ b/api/v1/file.go @@ -2,7 +2,8 @@ package v1 // File ... type File struct { - ID string `json:"id"` - Type string `json:"type"` - Glob []string `json:"path"` + ID string `json:"id,omitempty"` + Type string `json:"type,omitempty"` + URL string `json:"url,omitempty"` + Paths []string `json:"paths,omitempty"` } diff --git a/api/v1/types.go b/api/v1/types.go index ed6575c5..35512e1e 100644 --- a/api/v1/types.go +++ b/api/v1/types.go @@ -9,5 +9,5 @@ type ConfigScraper struct { // IsEmpty ... func (c ConfigScraper) IsEmpty() bool { - return len(c.AWS) == 0 + return len(c.AWS) == 0 && len(c.File) == 0 } diff --git a/db/update.go b/db/update.go index ecbc36ba..19f1638c 100644 --- a/db/update.go +++ b/db/update.go @@ -30,13 +30,20 @@ func NewConfigItemFromResult(result v1.ScrapeResult) models.ConfigItem { func Update(ctx v1.ScrapeContext, results []v1.ScrapeResult) error { // boil.DebugMode = true for _, result := range results { - data, err := json.Marshal(result.Config) - if err != nil { - return errors.Wrapf(err, "Unable to marshal: %v", result.Config) + var dataStr string + switch data := result.Config.(type) { + case string: + dataStr = data + case []byte: + dataStr = string(data) + default: + bytes, err := json.Marshal(data) + if err != nil { + return errors.Wrapf(err, "Unable to marshal: %v", result.Config) + } + dataStr = string(bytes) } - ci := NewConfigItemFromResult(result) - dataStr := string(data) ci.Config = &dataStr existing, err := GetConfigItem(result.ID) diff --git a/filesystem/file.go b/filesystem/file.go index 9b9c2697..657519fd 100644 --- a/filesystem/file.go +++ b/filesystem/file.go @@ -19,6 +19,8 @@ func (fm *fileFinder) Find(path string) ([]string, error) { } // Read ... -func (fm *fileFinder) Read(match string) ([]byte, error) { - return os.ReadFile(match) +func (fm *fileFinder) Read(match string) ([]byte, string, error) { + content, err := os.ReadFile(match) + filename := filepath.Base(match) + return content, filename, err } diff --git a/filesystem/interface.go b/filesystem/interface.go index aecfce0b..9be52bcb 100644 --- a/filesystem/interface.go +++ b/filesystem/interface.go @@ -3,5 +3,5 @@ package filesystem // Finder defines the contract for any file system finder to impletent type Finder interface { Find(string) ([]string, error) - Read(string) ([]byte, error) + Read(string) ([]byte, string, error) } diff --git a/go.mod b/go.mod index 2e19455c..d4043656 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/flanksource/commons v1.5.11 github.com/flanksource/kommons v0.26.0 github.com/google/uuid v1.3.0 + github.com/hashicorp/go-getter v1.6.1 github.com/henvic/httpretty v0.0.6 github.com/jackc/pgx/v4 v4.14.1 github.com/labstack/echo/v4 v4.6.3 @@ -32,6 +33,7 @@ require ( gorm.io/gorm v1.23.2 k8s.io/apimachinery v0.20.4 k8s.io/client-go v11.0.0+incompatible + sigs.k8s.io/yaml v1.2.0 ) require ( @@ -56,6 +58,7 @@ require ( github.com/aws/smithy-go v1.11.2 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/boltdb/bolt v1.3.1 // indirect + github.com/cheggaaa/pb v1.0.27 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect @@ -90,8 +93,7 @@ require ( github.com/hairyhenderson/toml v0.3.1-0.20191004034452-2a4f3b6160f2 // indirect github.com/hashicorp/consul/api v1.10.1 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/go-cleanhttp v0.5.1 // indirect - github.com/hashicorp/go-getter v1.4.1 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-hclog v0.12.0 // indirect github.com/hashicorp/go-immutable-radix v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.0 // indirect @@ -122,12 +124,14 @@ require ( github.com/joho/godotenv v1.3.0 // indirect github.com/json-iterator/go v1.1.11 // indirect github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd // indirect + github.com/klauspost/compress v1.11.2 // indirect github.com/kr/pretty v0.2.1 // indirect github.com/kr/text v0.1.0 // indirect github.com/labstack/gommon v0.3.1 // indirect github.com/mailru/easyjson v0.7.0 // indirect github.com/mattn/go-colorable v0.1.11 // indirect github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-runewidth v0.0.8 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/mapstructure v1.4.2 // indirect @@ -158,7 +162,7 @@ require ( golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect - golang.org/x/sys v0.0.0-20211103235746-7861aae1554b // indirect + golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e // indirect golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect @@ -184,7 +188,6 @@ require ( k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect sigs.k8s.io/kustomize v2.0.3+incompatible // indirect sigs.k8s.io/structured-merge-diff/v4 v4.0.2 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect ) replace ( diff --git a/go.sum b/go.sum index 8981c46b..4cab31ec 100644 --- a/go.sum +++ b/go.sum @@ -188,6 +188,7 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= +github.com/cheggaaa/pb v1.0.27 h1:wIkZHkNfC7R6GI5w7l/PdAdzXzlrbcI3p8OAlnkTsnc= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -466,10 +467,12 @@ github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOj github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-getter v1.4.1 h1:3A2Mh8smGFcf5M+gmcv898mZdrxpseik45IpcyISLsA= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-getter v1.4.1/go.mod h1:7qxyCd8rBfcShwsvxgIguu4KbS3l8bUCwg2Umn7RjeY= +github.com/hashicorp/go-getter v1.6.1 h1:NASsgP4q6tL94WH6nJxKWj8As2H/2kop/bB1d8JMyRY= +github.com/hashicorp/go-getter v1.6.1/go.mod h1:IZCrswsZPeWv9IkVnLElzRU/gz/QPi6pZHn4tv6vbwA= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= @@ -626,6 +629,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.11.2 h1:MiK62aErc3gIiVEtyzKfeOHgW7atJb5g/KNX5m3c2nQ= +github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -685,6 +690,7 @@ github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0= github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= @@ -909,6 +915,7 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ulikunitz/xz v0.5.5/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -1203,8 +1210,9 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211103235746-7861aae1554b h1:1VkfZQv42XQlA/jchYumAnv1UPo6RgF9rJFkTgZIxO4= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e h1:w36l2Uw3dRan1K3TyXriXvY+6T56GNmlKGcqiQUJDfM= +golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/samples/file-git.yaml b/samples/file-git.yaml new file mode 100644 index 00000000..64e4a56d --- /dev/null +++ b/samples/file-git.yaml @@ -0,0 +1,6 @@ +file: + - type: "github" + id: filename + url: github.com/flanksource/canary-checker + paths: + - fixtures/aws/**.yaml \ No newline at end of file diff --git a/samples/file-local.yaml b/samples/file-local.yaml new file mode 100644 index 00000000..72071d8f --- /dev/null +++ b/samples/file-local.yaml @@ -0,0 +1,5 @@ +file: + - type: "temo" + id: filename + paths: + - test.yaml \ No newline at end of file diff --git a/samples/test.yaml b/samples/test.yaml new file mode 100644 index 00000000..c3c95951 --- /dev/null +++ b/samples/test.yaml @@ -0,0 +1,6 @@ + aws: + - region: eu-west-1 + compliance: true + patch_states: true + patch_details: true + inventory: true \ No newline at end of file diff --git a/scrapers/file/file.go b/scrapers/file/file.go index 75e057cf..1a9681ef 100644 --- a/scrapers/file/file.go +++ b/scrapers/file/file.go @@ -1,63 +1,120 @@ package file import ( + "math/rand" + "os" + "path/filepath" + "time" + "github.com/flanksource/commons/logger" v1 "github.com/flanksource/confighub/api/v1" + "github.com/flanksource/confighub/filesystem" + "github.com/hashicorp/go-getter" "github.com/tidwall/gjson" + "sigs.k8s.io/yaml" ) // JSONScrapper ... type JSONScrapper struct { } +const charset = "abcdefghijklmnopqrstuvwxyz" + + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + // Scrape ... func (file JSONScrapper) Scrape(ctx v1.ScrapeContext, config v1.ConfigScraper, manager v1.Manager) []v1.ScrapeResult { - results := []v1.ScrapeResult{} - finder := manager.Finder - for _, fileConfig := range config.File { - - logger.Infof("Scraping JSON file id=%s type=%s", fileConfig.ID, fileConfig.Type) - - globMatches := []string{} - - for _, path := range fileConfig.Glob { - matches, err := finder.Find(path) - if err != nil { - logger.Tracef("could not match glob pattern(%s): %v", path, err) - continue - } - - globMatches = append(globMatches, matches...) // using a seperate slice to avoid nested loops and complexity + var globMatches []string + if fileConfig.URL != "" { + var tempDir string + globMatches, tempDir = findFilesFromURL(fileConfig.URL, fileConfig.Paths, finder) + defer os.RemoveAll(tempDir) + } else { + globMatches = findFiles("", fileConfig.Paths, finder) } - for _, match := range globMatches { - contentByte, err := finder.Read(match) + if !isYaml(match) && !isJson(match) { + logger.Debugf("skipping file %s, not a yaml or json file", match) + continue + } + contentByte, filename, err := finder.Read(match) if err != nil { logger.Errorf("failed to reading matched file: %v", err) continue } - - jsonContent := string(contentByte) - - resultID := gjson.Get(jsonContent, fileConfig.ID) - resultType := gjson.Get(jsonContent, fileConfig.Type) - - if !(resultID.Exists() && resultType.Exists()) { - continue + var jsonContent string + if isYaml(match) { + contentByte, err := yaml.YAMLToJSON(contentByte) + if err != nil { + logger.Errorf("failed to convert yaml to json: %v", err) + continue + } + jsonContent = string(contentByte) + } else { + jsonContent = string(contentByte) + } + resultID := gjson.Get(jsonContent, fileConfig.ID).String() + resultType := gjson.Get(jsonContent, fileConfig.Type).String() + if resultID == "" { + resultID = fileConfig.ID + "-" + filename + } + if resultType == "" { + resultType = fileConfig.Type } results = append(results, v1.ScrapeResult{ Config: jsonContent, - Type: resultType.String(), - ID: resultID.String(), + Type: resultType, + ID: resultID, }) } } - return results +} + +func findFilesFromURL(url string, paths []string, finder filesystem.Finder) (matches []string, dirname string) { + tempDir := GetTempDirName(10, charset) + if err := getter.GetAny(tempDir, url); err != nil { + logger.Errorf("Error downloading file: %s", err) + } + return findFiles(tempDir, paths, finder), tempDir +} + +func findFiles(dir string, paths []string, finder filesystem.Finder) []string { + matches := []string{} + if paths == nil { + logger.Debugf("no paths specified, scrapping all json and yaml/yml files") + paths = append(paths, "**.json", "**.yaml", "**.yml") + } + for _, path := range paths { + match, err := finder.Find(filepath.Join(dir, path)) + if err != nil { + logger.Tracef("could not match glob pattern(%s): %v", dir+"/"+path, err) + continue + } + matches = append(matches, match...) // using a seperate slice to avoid nested loops and complexity + } + return matches +} + +func isYaml(filename string) bool { + return filepath.Ext(filename) == ".yaml" || filepath.Ext(filename) == ".yml" +} +func isJson(filename string) bool { + return filepath.Ext(filename) == ".json" +} + +var seededRand *rand.Rand = rand.New( + rand.NewSource(time.Now().UnixNano())) + +func GetTempDirName(length int, charset string) string { + b := make([]byte, length) + for i := range b { + b[i] = charset[seededRand.Intn(len(charset))] + } + return string(b) } diff --git a/scrapers/runscrapers_test.go b/scrapers/runscrapers_test.go index dcad754e..9d5d6524 100644 --- a/scrapers/runscrapers_test.go +++ b/scrapers/runscrapers_test.go @@ -26,7 +26,7 @@ func TestRun(t *testing.T) { { ID: "Config.InstanceId", Type: "Config.InstanceType", - Glob: []string{ + Paths: []string{ "../fixtures/config*.json", "../fixtures/test*.json", }, From d72dc1a7706911b83a482c9952d4f8275f777842 Mon Sep 17 00:00:00 2001 From: Tarun Khandelwal Date: Tue, 31 May 2022 09:28:16 +0530 Subject: [PATCH 2/5] chore: rename JSONScrapper to FileScrapper --- scrapers/common.go | 2 +- scrapers/file/file.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapers/common.go b/scrapers/common.go index 773db00e..887ed681 100644 --- a/scrapers/common.go +++ b/scrapers/common.go @@ -9,5 +9,5 @@ import ( // All is the scrappers registry var All = []v1.Scraper{ aws.Scraper{}, - file.JSONScrapper{}, + file.FileScrapper{}, } diff --git a/scrapers/file/file.go b/scrapers/file/file.go index 1a9681ef..afd86580 100644 --- a/scrapers/file/file.go +++ b/scrapers/file/file.go @@ -14,15 +14,15 @@ import ( "sigs.k8s.io/yaml" ) -// JSONScrapper ... -type JSONScrapper struct { +// FileScrapper ... +type FileScrapper struct { } const charset = "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" // Scrape ... -func (file JSONScrapper) Scrape(ctx v1.ScrapeContext, config v1.ConfigScraper, manager v1.Manager) []v1.ScrapeResult { +func (file FileScrapper) Scrape(ctx v1.ScrapeContext, config v1.ConfigScraper, manager v1.Manager) []v1.ScrapeResult { results := []v1.ScrapeResult{} finder := manager.Finder for _, fileConfig := range config.File { From dc81d6a8536acbe52049183023cb2af9d8faa308 Mon Sep 17 00:00:00 2001 From: Tarun Khandelwal Date: Tue, 31 May 2022 19:41:45 +0530 Subject: [PATCH 3/5] chore: move samples to fixtures --- {samples => fixtures}/file-git.yaml | 0 {samples => fixtures}/file-local.yaml | 0 {samples => fixtures}/scrapper.yaml | 0 {samples => fixtures}/test.yaml | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {samples => fixtures}/file-git.yaml (100%) rename {samples => fixtures}/file-local.yaml (100%) rename {samples => fixtures}/scrapper.yaml (100%) rename {samples => fixtures}/test.yaml (100%) diff --git a/samples/file-git.yaml b/fixtures/file-git.yaml similarity index 100% rename from samples/file-git.yaml rename to fixtures/file-git.yaml diff --git a/samples/file-local.yaml b/fixtures/file-local.yaml similarity index 100% rename from samples/file-local.yaml rename to fixtures/file-local.yaml diff --git a/samples/scrapper.yaml b/fixtures/scrapper.yaml similarity index 100% rename from samples/scrapper.yaml rename to fixtures/scrapper.yaml diff --git a/samples/test.yaml b/fixtures/test.yaml similarity index 100% rename from samples/test.yaml rename to fixtures/test.yaml From ad7d7e148e9e683c42c8048766b31e1eecd62ce4 Mon Sep 17 00:00:00 2001 From: Tarun Khandelwal Date: Tue, 31 May 2022 20:42:24 +0530 Subject: [PATCH 4/5] chore: add test for url based yaml file --- scrapers/file/file.go | 6 ++-- scrapers/runscrapers_test.go | 56 ++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/scrapers/file/file.go b/scrapers/file/file.go index afd86580..daab4cc8 100644 --- a/scrapers/file/file.go +++ b/scrapers/file/file.go @@ -58,10 +58,8 @@ func (file FileScrapper) Scrape(ctx v1.ScrapeContext, config v1.ConfigScraper, m resultID := gjson.Get(jsonContent, fileConfig.ID).String() resultType := gjson.Get(jsonContent, fileConfig.Type).String() if resultID == "" { - resultID = fileConfig.ID + "-" + filename - } - if resultType == "" { - resultType = fileConfig.Type + logger.Errorf("failed to the specified gjson path: %s in file %s", fileConfig.ID, filename) + continue } results = append(results, v1.ScrapeResult{ Config: jsonContent, diff --git a/scrapers/runscrapers_test.go b/scrapers/runscrapers_test.go index 9d5d6524..fcfb6ef3 100644 --- a/scrapers/runscrapers_test.go +++ b/scrapers/runscrapers_test.go @@ -31,6 +31,14 @@ func TestRun(t *testing.T) { "../fixtures/test*.json", }, }, + { + ID: "metadata.name", + Type: "kind", + Paths: []string{ + "fixtures/minimal/http_pass.yaml", + }, + URL: "github.com/flanksource/canary-checker", + }, }, }, expectedResult: []v1.ScrapeResult{ @@ -44,6 +52,54 @@ func TestRun(t *testing.T) { Type: "instance_type_2", ID: "instance_id_2", }, + { + Config: `{ + "apiVersion": "canaries.flanksource.com/v1", + "kind": "Canary", + "metadata": { + "name": "http-pass" + }, + "spec": { + "interval": 30, + "http": [ + { + "endpoint": "http://status.savanttools.com/?code=200", + "thresholdMillis": 3000, + "responseCodes": [ + 201, + 200, + 301 + ], + "responseContent": "", + "maxSSLExpiry": 7, + "test": { + "expr": "code == 200" + } + }, + { + "endpoint": "http://status.savanttools.com/?code=404", + "thresholdMillis": 3000, + "responseCodes": [ + 404 + ], + "responseContent": "", + "maxSSLExpiry": 7 + }, + { + "endpoint": "http://status.savanttools.com/?code=500", + "thresholdMillis": 3000, + "responseCodes": [ + 500 + ], + "responseContent": "", + "maxSSLExpiry": 7 + } + ] + } + }`, + Type: "Canary", + ID: "http-pass", + }, }, }, } From e76f5d69ee1652d38c5d9d730cb7b0ee82c466de Mon Sep 17 00:00:00 2001 From: Tarun Khandelwal Date: Tue, 31 May 2022 21:05:08 +0530 Subject: [PATCH 5/5] chore: add source column in config_item to show the filename --- api/v1/interface.go | 1 + db/migrations/6_source_alter.sql | 6 ++++++ db/models/config_item.go | 1 + db/update.go | 1 + fixtures/file-git.yaml | 6 +++--- fixtures/file-local.yaml | 6 +++--- scrapers/file/file.go | 1 + 7 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 db/migrations/6_source_alter.sql diff --git a/api/v1/interface.go b/api/v1/interface.go index 322c03bd..3fae0b56 100644 --- a/api/v1/interface.go +++ b/api/v1/interface.go @@ -41,6 +41,7 @@ type ScrapeResult struct { Name string `json:"name,omitempty"` Namespace string `json:"namespace,omitempty"` ID string `json:"id,omitempty"` + Source string `json:"source,omitempty"` Config interface{} `json:"config,omitempty"` } diff --git a/db/migrations/6_source_alter.sql b/db/migrations/6_source_alter.sql new file mode 100644 index 00000000..fadf2cf4 --- /dev/null +++ b/db/migrations/6_source_alter.sql @@ -0,0 +1,6 @@ +-- +goose Up +-- +goose StatementBegin +--- +ALTER TABLE config_item ADD source text; + +-- +goose StatementEnd \ No newline at end of file diff --git a/db/models/config_item.go b/db/models/config_item.go index 11e6808e..560c9e3f 100644 --- a/db/models/config_item.go +++ b/db/models/config_item.go @@ -20,6 +20,7 @@ type ConfigItem struct { Network *string `gorm:"column:network;default:null" json:"network,omitempty" toml:"network" yaml:"network,omitempty"` Subnet *string `gorm:"column:subnet;default:null" json:"subnet,omitempty" toml:"subnet" yaml:"subnet,omitempty"` Config *string `gorm:"column:config;default:null" json:"config,omitempty" toml:"config" yaml:"config,omitempty"` + Source *string `gorm:"column:source;default:null" json:"source,omitempty" toml:"source" yaml:"source,omitempty"` CreatedAt time.Time `gorm:"column:created_at" json:"created_at" toml:"created_at" yaml:"created_at"` UpdatedAt time.Time `gorm:"column:updated_at" json:"updated_at" toml:"updated_at" yaml:"updated_at"` } diff --git a/db/update.go b/db/update.go index 19f1638c..69b96b94 100644 --- a/db/update.go +++ b/db/update.go @@ -23,6 +23,7 @@ func NewConfigItemFromResult(result v1.ScrapeResult) models.ConfigItem { Network: &result.Network, Subnet: &result.Subnet, Name: &result.Name, + Source: &result.Source, } } diff --git a/fixtures/file-git.yaml b/fixtures/file-git.yaml index 64e4a56d..1a169d69 100644 --- a/fixtures/file-git.yaml +++ b/fixtures/file-git.yaml @@ -1,6 +1,6 @@ file: - - type: "github" - id: filename + - type: kind + id: metadata.name url: github.com/flanksource/canary-checker paths: - - fixtures/aws/**.yaml \ No newline at end of file + - fixtures/minimal/**.yaml \ No newline at end of file diff --git a/fixtures/file-local.yaml b/fixtures/file-local.yaml index 72071d8f..38b4d75a 100644 --- a/fixtures/file-local.yaml +++ b/fixtures/file-local.yaml @@ -1,5 +1,5 @@ file: - - type: "temo" - id: filename + - type: aws[0].region + id: aws[0].region paths: - - test.yaml \ No newline at end of file + - fixtures/test.yaml \ No newline at end of file diff --git a/scrapers/file/file.go b/scrapers/file/file.go index daab4cc8..a9dbc272 100644 --- a/scrapers/file/file.go +++ b/scrapers/file/file.go @@ -65,6 +65,7 @@ func (file FileScrapper) Scrape(ctx v1.ScrapeContext, config v1.ConfigScraper, m Config: jsonContent, Type: resultType, ID: resultID, + Source: filename, }) }