Skip to content

Commit

Permalink
chore: save results from targetted scrape
Browse files Browse the repository at this point in the history 
- go mod tidy on hack
- bug fix. KetOne can return empty object without any error.
  • Loading branch information
@adityathebe
adityathebe committed Sep 11, 2023
1 parent c0be9d9 commit 99e0387
Show file tree
Hide file tree
Showing 6 changed files with 95 additions and 70 deletions.
18 changes: 9 additions & 9 deletions cmd/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,18 +102,18 @@ func startScraperCron(configFiles []string) {
}
scrapers.AddToCron(_scraper)

fn := func() {
ctx := api.NewScrapeContext(context.Background(), _scraper)
if _, err := scrapers.RunScraper(ctx); err != nil {
logger.Errorf("Error running scraper(id=%s): %v", scraper.ID, err)
}
}
go scrapers.AtomicRunner(scraper.ID.String(), fn)()

for _, k := range _scraper.Spec.Kubernetes {
ctx := api.NewScrapeContext(context.Background(), _scraper)
go exitOnError(kubernetes.WatchEvents(ctx, k, kubernetesChangeEventConsumer), "error watching events")
}

// fn := func() {
// ctx := api.NewScrapeContext(context.Background(), _scraper)
// if _, err := scrapers.RunScraper(ctx); err != nil {
// logger.Errorf("Error running scraper(id=%s): %v", scraper.ID, err)
// }
// }
// go scrapers.AtomicRunner(scraper.ID.String(), fn)()
}
}

Expand Down Expand Up @@ -152,7 +152,7 @@ func kubernetesChangeEventConsumer(ctx *v1.ScrapeContext, resourcesPerKind map[s
}
}

if _, err := scrapers.RunSome(ctx, kubernetes.KubernetesScraper{}, 0, resourceIDs); err != nil {
if _, err := scrapers.RunTargettedScraper(ctx, kubernetes.KubernetesScraper{}, 0, resourceIDs); err != nil {
logger.Errorf("Error running scraper(id=%s): %v", ctx.ScrapeConfig.GetUID(), err)
}
}
36 changes: 18 additions & 18 deletions hack/generate-schemas/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b
github.com/flanksource/commons v1.10.2
github.com/flanksource/config-db v0.0.65
github.com/spf13/cobra v1.6.1
github.com/spf13/cobra v1.7.0
)

require (
Expand All @@ -26,27 +26,27 @@ require (
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/emicklei/go-restful/v3 v3.10.2 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/flanksource/duty v1.0.129 // indirect
github.com/flanksource/gomplate/v3 v3.20.9 // indirect
github.com/flanksource/is-healthy v0.0.0-20230713150444-ad2a5ef4bb37 // indirect
github.com/flanksource/mapstructure v1.6.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-openapi/inflect v0.19.0 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonpointer v0.20.0 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/go-openapi/swag v0.22.4 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/cel-go v0.17.1 // indirect
github.com/google/gnostic v0.6.9 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/google/uuid v1.3.1 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/gosimple/slug v1.13.1 // indirect
Expand Down Expand Up @@ -102,18 +102,18 @@ require (
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/crypto v0.13.0 // indirect
golang.org/x/exp v0.0.0-20230728194245-b0cb94b80691 // indirect
golang.org/x/net v0.12.0 // indirect
golang.org/x/oauth2 v0.10.0 // indirect
golang.org/x/net v0.15.0 // indirect
golang.org/x/oauth2 v0.12.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/term v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/term v0.12.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.134.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230726155614-23370e0ffb3e // indirect
Expand All @@ -126,14 +126,14 @@ require (
gopkg.in/yaml.v3 v3.0.1 // indirect
gorm.io/driver/postgres v1.5.2 // indirect
gorm.io/gorm v1.25.0 // indirect
k8s.io/api v0.26.4 // indirect
k8s.io/apimachinery v0.26.4 // indirect
k8s.io/client-go v0.26.4 // indirect
k8s.io/api v0.28.0 // indirect
k8s.io/apimachinery v0.28.0 // indirect
k8s.io/client-go v0.28.0 // indirect
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/kube-openapi v0.0.0-20230525220651-2546d827e515 // indirect
k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
layeh.com/gopher-json v0.0.0-20201124131017-552bb3c4c3bf // indirect
sigs.k8s.io/controller-runtime v0.14.6 // indirect
sigs.k8s.io/controller-runtime v0.15.0 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
Expand Down
Loading

0 comments on commit 99e0387

Please sign in to comment.