From 95239f27033a70d6886a742e04d4d01c65049ce3 Mon Sep 17 00:00:00 2001 From: Augustin Husson Date: Wed, 30 Oct 2024 10:17:25 +0100 Subject: [PATCH] add an http client to send metrics_usage to a remote server Signed-off-by: Augustin Husson --- config/collector.go | 31 ++++++++++++++------- dev/config.yaml | 4 +-- docs/configuration.md | 19 ++++++++++--- pkg/client/client.go | 57 +++++++++++++++++++++++++++++++++++++++ source/grafana/grafana.go | 37 ++++++++++++++++++------- source/perses/perses.go | 35 +++++++++++++++++------- source/rules/rules.go | 33 +++++++++++++++++------ 7 files changed, 175 insertions(+), 41 deletions(-) create mode 100644 pkg/client/client.go diff --git a/config/collector.go b/config/collector.go index 38ed236..8e0befa 100644 --- a/config/collector.go +++ b/config/collector.go @@ -70,9 +70,11 @@ func (c *MetricCollector) Verify() error { } type RulesCollector struct { - Enable bool `yaml:"enable"` - Period model.Duration `yaml:"period,omitempty"` - HTTPClient HTTPClient `yaml:"http_client"` + Enable bool `yaml:"enable"` + Period model.Duration `yaml:"period,omitempty"` + // MetricUsageClient is a client to send the metrics usage to a remote metrics_usage server. + MetricUsageClient *HTTPClient `yaml:"metric_usage_client,omitempty"` + HTTPClient HTTPClient `yaml:"prometheus_client"` } func (c *RulesCollector) Verify() error { @@ -85,13 +87,17 @@ func (c *RulesCollector) Verify() error { if c.HTTPClient.URL == nil { return fmt.Errorf("missing Prometheus URL for the rules collector") } + if c.MetricUsageClient != nil && c.MetricUsageClient.URL == nil { + return fmt.Errorf("missing Metrics Usage URL for the rules collector") + } return nil } type PersesCollector struct { - Enable bool `yaml:"enable"` - Period model.Duration `yaml:"period,omitempty"` - HTTPClient config.RestConfigClient `yaml:"http_client"` + Enable bool `yaml:"enable"` + Period model.Duration `yaml:"period,omitempty"` + MetricUsageClient *HTTPClient `yaml:"metric_usage_client,omitempty"` + HTTPClient config.RestConfigClient `yaml:"perses_client"` } func (c *PersesCollector) Verify() error { @@ -104,13 +110,17 @@ func (c *PersesCollector) Verify() error { if c.HTTPClient.URL == nil { return fmt.Errorf("missing Rest URL for the perses collector") } + if c.MetricUsageClient != nil && c.MetricUsageClient.URL == nil { + return fmt.Errorf("missing Metrics Usage URL for the rules collector") + } return nil } type GrafanaCollector struct { - Enable bool `yaml:"enable"` - Period model.Duration `yaml:"period,omitempty"` - HTTPClient HTTPClient `yaml:"http_client"` + Enable bool `yaml:"enable"` + Period model.Duration `yaml:"period,omitempty"` + MetricUsageClient *HTTPClient `yaml:"metric_usage_client,omitempty"` + HTTPClient HTTPClient `yaml:"grafana_client"` } func (c *GrafanaCollector) Verify() error { @@ -123,5 +133,8 @@ func (c *GrafanaCollector) Verify() error { if c.HTTPClient.URL == nil { return fmt.Errorf("missing Rest URL for the perses collector") } + if c.MetricUsageClient != nil && c.MetricUsageClient.URL == nil { + return fmt.Errorf("missing Metrics Usage URL for the rules collector") + } return nil } diff --git a/dev/config.yaml b/dev/config.yaml index 0e30a71..cac00ae 100644 --- a/dev/config.yaml +++ b/dev/config.yaml @@ -5,10 +5,10 @@ metric_collector: rules_collectors: - enable: true - http_client: + prometheus_client: url: "https://prometheus.demo.do.prometheus.io" perses_collector: enable: true - http_client: + perses_client: url: "https://demo.perses.dev" diff --git a/docs/configuration.md b/docs/configuration.md index bb58b5e..99a022f 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -65,7 +65,12 @@ http_client: ```yaml [ enable: | default=false ] [ period: | default="12h" ] -http_client: + +# It is a client to send the metrics usage to a remote metrics_usage server. +[ metric_usage_client: ] + +# The prometheus client used to retrieve the rules +prometheus_client: ``` ### Perses_Collector Config @@ -73,7 +78,11 @@ http_client: ```yaml [ enable: | default=false ] [ period: | default="12h" ] -http_client: +# It is a client to send the metrics usage to a remote metrics_usage server. +[ metric_usage_client: ] + +# the Perses client used to retrieve the dashboards +perses_client: url: [ tls_config: ] auth: @@ -89,7 +98,11 @@ http_client: ```yaml [ enable: | default=false ] [ period: | default="12h" ] -http_client: < HTTPClient config> +# It is a client to send the metrics usage to a remote metrics_usage server. +[ metric_usage_client: ] + +# the Grafana client used to retrieve the dashboards +grafana_client: < HTTPClient config> ``` ### TLS Config diff --git a/pkg/client/client.go b/pkg/client/client.go new file mode 100644 index 0000000..9d9b9be --- /dev/null +++ b/pkg/client/client.go @@ -0,0 +1,57 @@ +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/url" + "path" + + "github.com/perses/metrics-usage/config" + modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1" +) + +type Client interface { + Usage(map[string]*modelAPIV1.MetricUsage) error +} + +func New(cfg config.HTTPClient) (Client, error) { + httpClient, err := config.NewHTTPClient(cfg) + if err != nil { + return nil, err + } + return &client{ + endpoint: cfg.URL.URL, + httpClient: httpClient, + }, nil +} + +type client struct { + endpoint *url.URL + httpClient *http.Client +} + +func (c *client) Usage(metrics map[string]*modelAPIV1.MetricUsage) error { + data, err := json.Marshal(metrics) + if err != nil { + return err + } + body := bytes.NewBuffer(data) + resp, err := c.httpClient.Post(c.url("/api/v1/metrics").String(), "application/json", body) + if err != nil { + return err + } + if resp.StatusCode < http.StatusOK || resp.StatusCode > http.StatusPartialContent { + return fmt.Errorf("when sending metrics usage, unexpected status code: %d", resp.StatusCode) + } + return nil +} + +func (c *client) url(ep string) *url.URL { + p := path.Join(c.endpoint.Path, ep) + u := *c.endpoint + u.Path = p + + return &u +} diff --git a/source/grafana/grafana.go b/source/grafana/grafana.go index 372020b..4d851e0 100644 --- a/source/grafana/grafana.go +++ b/source/grafana/grafana.go @@ -14,6 +14,7 @@ import ( "github.com/perses/metrics-usage/config" "github.com/perses/metrics-usage/database" modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1" + "github.com/perses/metrics-usage/pkg/client" "github.com/perses/metrics-usage/utils" "github.com/perses/metrics-usage/utils/prometheus" "github.com/sirupsen/logrus" @@ -36,25 +37,34 @@ func NewCollector(db database.Database, cfg config.GrafanaCollector) (async.Simp if err != nil { return nil, err } + var metricUsageClient client.Client + if cfg.MetricUsageClient != nil { + metricUsageClient, err = client.New(*cfg.MetricUsageClient) + if err != nil { + return nil, err + } + } transportConfig := &grafanaapi.TransportConfig{ Host: url.Host, BasePath: grafanaapi.DefaultBasePath, Schemes: []string{url.Scheme}, Client: httpClient, } - client := grafanaapi.NewHTTPClientWithConfig(strfmt.Default, transportConfig) + grafanaClient := grafanaapi.NewHTTPClientWithConfig(strfmt.Default, transportConfig) return &grafanaCollector{ - db: db, - grafanaURL: url.String(), - client: client, + db: db, + grafanaURL: url.String(), + grafanaClient: grafanaClient, + metricUsageClient: metricUsageClient, }, nil } type grafanaCollector struct { async.SimpleTask - db database.Database - grafanaURL string - client *grafanaapi.GrafanaHTTPAPI + db database.Database + metricUsageClient client.Client + grafanaURL string + grafanaClient *grafanaapi.GrafanaHTTPAPI } func (c *grafanaCollector) Execute(ctx context.Context, _ context.CancelFunc) error { @@ -74,7 +84,14 @@ func (c *grafanaCollector) Execute(ctx context.Context, _ context.CancelFunc) er c.extractMetricUsage(metricUsage, dashboard) } if len(metricUsage) > 0 { - c.db.EnqueueUsage(metricUsage) + if c.metricUsageClient != nil { + // In this case, that means we have to send the data to a remote server. + if sendErr := c.metricUsageClient.Usage(metricUsage); sendErr != nil { + logrus.WithError(sendErr).Error("Failed to send usage metric") + } + } else { + c.db.EnqueueUsage(metricUsage) + } } return nil } @@ -103,7 +120,7 @@ func (c *grafanaCollector) extractMetricUsageFromPanels(metricUsage map[string]* } func (c *grafanaCollector) getDashboard(uid string) (*simplifiedDashboard, error) { - response, err := c.client.Dashboards.GetDashboardByUID(uid) + response, err := c.grafanaClient.Dashboards.GetDashboardByUID(uid) if err != nil { return nil, err } @@ -123,7 +140,7 @@ func (c *grafanaCollector) collectAllDashboardUID(ctx context.Context) ([]*grafa searchType := "dash-db" for searchOk { - nextPageResult, err := c.client.Search.Search(&search.SearchParams{ + nextPageResult, err := c.grafanaClient.Search.Search(&search.SearchParams{ Context: ctx, Type: &searchType, Page: ¤tPage, diff --git a/source/perses/perses.go b/source/perses/perses.go index da3e494..adcc59f 100644 --- a/source/perses/perses.go +++ b/source/perses/perses.go @@ -10,6 +10,7 @@ import ( "github.com/perses/metrics-usage/config" "github.com/perses/metrics-usage/database" modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1" + "github.com/perses/metrics-usage/pkg/client" "github.com/perses/metrics-usage/utils" "github.com/perses/metrics-usage/utils/prometheus" "github.com/perses/perses/go-sdk/prometheus/query" @@ -39,23 +40,32 @@ func NewCollector(db database.Database, cfg config.PersesCollector) (async.Simpl if err != nil { return nil, err } + var metricUsageClient client.Client + if cfg.MetricUsageClient != nil { + metricUsageClient, err = client.New(*cfg.MetricUsageClient) + if err != nil { + return nil, err + } + } return &persesCollector{ - SimpleTask: nil, - client: persesClientV1.NewWithClient(restClient).Dashboard(""), - db: db, - persesURL: cfg.HTTPClient.URL.String(), + SimpleTask: nil, + persesClient: persesClientV1.NewWithClient(restClient).Dashboard(""), + db: db, + metricUsageClient: metricUsageClient, + persesURL: cfg.HTTPClient.URL.String(), }, nil } type persesCollector struct { async.SimpleTask - client persesClientV1.DashboardInterface - db database.Database - persesURL string + persesClient persesClientV1.DashboardInterface + db database.Database + metricUsageClient client.Client + persesURL string } func (c *persesCollector) Execute(_ context.Context, _ context.CancelFunc) error { - dashboards, err := c.client.List("") + dashboards, err := c.persesClient.List("") if err != nil { logrus.WithError(err).Error("Failed to get dashboards") return nil @@ -68,7 +78,14 @@ func (c *persesCollector) Execute(_ context.Context, _ context.CancelFunc) error c.extractMetricUsageFromPanels(metricUsage, dash.Spec.Panels, dash) } if len(metricUsage) > 0 { - c.db.EnqueueUsage(metricUsage) + if c.metricUsageClient != nil { + // In this case, that means we have to send the data to a remote server. + if sendErr := c.metricUsageClient.Usage(metricUsage); sendErr != nil { + logrus.WithError(sendErr).Error("Failed to send usage metric") + } + } else { + c.db.EnqueueUsage(metricUsage) + } } return nil } diff --git a/source/rules/rules.go b/source/rules/rules.go index 628b794..18f600e 100644 --- a/source/rules/rules.go +++ b/source/rules/rules.go @@ -7,6 +7,7 @@ import ( "github.com/perses/metrics-usage/config" "github.com/perses/metrics-usage/database" modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1" + "github.com/perses/metrics-usage/pkg/client" "github.com/perses/metrics-usage/utils" "github.com/perses/metrics-usage/utils/prometheus" v1 "github.com/prometheus/client_golang/api/prometheus/v1" @@ -18,29 +19,45 @@ func NewCollector(db database.Database, cfg *config.RulesCollector) (async.Simpl if err != nil { return nil, err } + var metricUsageClient client.Client + if cfg.MetricUsageClient != nil { + metricUsageClient, err = client.New(*cfg.MetricUsageClient) + if err != nil { + return nil, err + } + } return &rulesCollector{ - client: promClient, - db: db, - promURL: cfg.HTTPClient.URL.String(), + promClient: promClient, + db: db, + metricUsageClient: metricUsageClient, + promURL: cfg.HTTPClient.URL.String(), }, nil } type rulesCollector struct { async.SimpleTask - client v1.API - db database.Database - promURL string + promClient v1.API + db database.Database + metricUsageClient client.Client + promURL string } func (c *rulesCollector) Execute(ctx context.Context, _ context.CancelFunc) error { - result, err := c.client.Rules(ctx) + result, err := c.promClient.Rules(ctx) if err != nil { logrus.WithError(err).Error("Failed to get rules") return nil } metricUsage := extractMetricUsageFromRules(result.Groups, c.promURL) if len(metricUsage) > 0 { - c.db.EnqueueUsage(metricUsage) + if c.metricUsageClient != nil { + // In this case, that means we have to send the data to a remote server. + if sendErr := c.metricUsageClient.Usage(metricUsage); sendErr != nil { + logrus.WithError(sendErr).Error("Failed to send usage metric") + } + } else { + c.db.EnqueueUsage(metricUsage) + } } return nil }