add an http client to send metrics_usage to a remote server

Signed-off-by: Augustin Husson <[email protected]>
perses · Oct 30, 2024 · 95239f2 · 95239f2
1 parent 8761bc1
commit 95239f2
Show file tree

Hide file tree

Showing 7 changed files with 175 additions and 41 deletions.
diff --git a/config/collector.go b/config/collector.go
@@ -70,9 +70,11 @@ func (c *MetricCollector) Verify() error {
 }
 
 type RulesCollector struct {
-	Enable     bool           `yaml:"enable"`
-	Period     model.Duration `yaml:"period,omitempty"`
-	HTTPClient HTTPClient     `yaml:"http_client"`
+	Enable bool           `yaml:"enable"`
+	Period model.Duration `yaml:"period,omitempty"`
+	// MetricUsageClient is a client to send the metrics usage to a remote metrics_usage server.
+	MetricUsageClient *HTTPClient `yaml:"metric_usage_client,omitempty"`
+	HTTPClient        HTTPClient  `yaml:"prometheus_client"`
 }
 
 func (c *RulesCollector) Verify() error {
@@ -85,13 +87,17 @@ func (c *RulesCollector) Verify() error {
 	if c.HTTPClient.URL == nil {
 		return fmt.Errorf("missing Prometheus URL for the rules collector")
 	}
+	if c.MetricUsageClient != nil && c.MetricUsageClient.URL == nil {
+		return fmt.Errorf("missing Metrics Usage URL for the rules collector")
+	}
 	return nil
 }
 
 type PersesCollector struct {
-	Enable     bool                    `yaml:"enable"`
-	Period     model.Duration          `yaml:"period,omitempty"`
-	HTTPClient config.RestConfigClient `yaml:"http_client"`
+	Enable            bool                    `yaml:"enable"`
+	Period            model.Duration          `yaml:"period,omitempty"`
+	MetricUsageClient *HTTPClient             `yaml:"metric_usage_client,omitempty"`
+	HTTPClient        config.RestConfigClient `yaml:"perses_client"`
 }
 
 func (c *PersesCollector) Verify() error {
@@ -104,13 +110,17 @@ func (c *PersesCollector) Verify() error {
 	if c.HTTPClient.URL == nil {
 		return fmt.Errorf("missing Rest URL for the perses collector")
 	}
+	if c.MetricUsageClient != nil && c.MetricUsageClient.URL == nil {
+		return fmt.Errorf("missing Metrics Usage URL for the rules collector")
+	}
 	return nil
 }
 
 type GrafanaCollector struct {
-	Enable     bool           `yaml:"enable"`
-	Period     model.Duration `yaml:"period,omitempty"`
-	HTTPClient HTTPClient     `yaml:"http_client"`
+	Enable            bool           `yaml:"enable"`
+	Period            model.Duration `yaml:"period,omitempty"`
+	MetricUsageClient *HTTPClient    `yaml:"metric_usage_client,omitempty"`
+	HTTPClient        HTTPClient     `yaml:"grafana_client"`
 }
 
 func (c *GrafanaCollector) Verify() error {
@@ -123,5 +133,8 @@ func (c *GrafanaCollector) Verify() error {
 	if c.HTTPClient.URL == nil {
 		return fmt.Errorf("missing Rest URL for the perses collector")
 	}
+	if c.MetricUsageClient != nil && c.MetricUsageClient.URL == nil {
+		return fmt.Errorf("missing Metrics Usage URL for the rules collector")
+	}
 	return nil
 }
diff --git a/dev/config.yaml b/dev/config.yaml
@@ -5,10 +5,10 @@ metric_collector:
 
 rules_collectors:
   - enable: true
-    http_client:
+    prometheus_client:
       url: "https://prometheus.demo.do.prometheus.io"
 
 perses_collector:
   enable: true
-  http_client:
+  perses_client:
     url: "https://demo.perses.dev"
diff --git a/docs/configuration.md b/docs/configuration.md
@@ -65,15 +65,24 @@ http_client: <HTTPClient config>
 ```yaml
 [ enable: <boolean> | default=false ]
 [ period: <duration> | default="12h" ]
-http_client: <HTTPClient config>
+
+# It is a client to send the metrics usage to a remote metrics_usage server.
+[ metric_usage_client: <HTTPClient config> ]
+
+# The prometheus client used to retrieve the rules
+prometheus_client: <HTTPClient config>
 ```
 
 ### Perses_Collector Config
 
 ```yaml
 [ enable: <boolean> | default=false ]
 [ period: <duration> | default="12h" ]
-http_client:
+# It is a client to send the metrics usage to a remote metrics_usage server.
+[ metric_usage_client: <HTTPClient config> ]
+
+# the Perses client used to retrieve the dashboards
+perses_client:
   url: <string>
   [ tls_config: <TLS Config> ]
   auth:
@@ -89,7 +98,11 @@ http_client:
 ```yaml
 [ enable: <boolean> | default=false ]
 [ period: <duration> | default="12h" ]
-http_client: < HTTPClient config>
+# It is a client to send the metrics usage to a remote metrics_usage server.
+[ metric_usage_client: <HTTPClient config> ]
+
+# the Grafana client used to retrieve the dashboards
+grafana_client: < HTTPClient config>
 ```
 
 ### TLS Config

diff --git a/pkg/client/client.go b/pkg/client/client.go
@@ -0,0 +1,57 @@
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+
+	"github.com/perses/metrics-usage/config"
+	modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1"
+)
+
+type Client interface {
+	Usage(map[string]*modelAPIV1.MetricUsage) error
+}
+
+func New(cfg config.HTTPClient) (Client, error) {
+	httpClient, err := config.NewHTTPClient(cfg)
+	if err != nil {
+		return nil, err
+	}
+	return &client{
+		endpoint:   cfg.URL.URL,
+		httpClient: httpClient,
+	}, nil
+}
+
+type client struct {
+	endpoint   *url.URL
+	httpClient *http.Client
+}
+
+func (c *client) Usage(metrics map[string]*modelAPIV1.MetricUsage) error {
+	data, err := json.Marshal(metrics)
+	if err != nil {
+		return err
+	}
+	body := bytes.NewBuffer(data)
+	resp, err := c.httpClient.Post(c.url("/api/v1/metrics").String(), "application/json", body)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode < http.StatusOK || resp.StatusCode > http.StatusPartialContent {
+		return fmt.Errorf("when sending metrics usage, unexpected status code: %d", resp.StatusCode)
+	}
+	return nil
+}
+
+func (c *client) url(ep string) *url.URL {
+	p := path.Join(c.endpoint.Path, ep)
+	u := *c.endpoint
+	u.Path = p
+
+	return &u
+}
diff --git a/source/grafana/grafana.go b/source/grafana/grafana.go
@@ -14,6 +14,7 @@ import (
 	"github.com/perses/metrics-usage/config"
 	"github.com/perses/metrics-usage/database"
 	modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1"
+	"github.com/perses/metrics-usage/pkg/client"
 	"github.com/perses/metrics-usage/utils"
 	"github.com/perses/metrics-usage/utils/prometheus"
 	"github.com/sirupsen/logrus"
@@ -36,25 +37,34 @@ func NewCollector(db database.Database, cfg config.GrafanaCollector) (async.Simp
 	if err != nil {
 		return nil, err
 	}
+	var metricUsageClient client.Client
+	if cfg.MetricUsageClient != nil {
+		metricUsageClient, err = client.New(*cfg.MetricUsageClient)
+		if err != nil {
+			return nil, err
+		}
+	}
 	transportConfig := &grafanaapi.TransportConfig{
 		Host:     url.Host,
 		BasePath: grafanaapi.DefaultBasePath,
 		Schemes:  []string{url.Scheme},
 		Client:   httpClient,
 	}
-	client := grafanaapi.NewHTTPClientWithConfig(strfmt.Default, transportConfig)
+	grafanaClient := grafanaapi.NewHTTPClientWithConfig(strfmt.Default, transportConfig)
 	return &grafanaCollector{
-		db:         db,
-		grafanaURL: url.String(),
-		client:     client,
+		db:                db,
+		grafanaURL:        url.String(),
+		grafanaClient:     grafanaClient,
+		metricUsageClient: metricUsageClient,
 	}, nil
 }
 
 type grafanaCollector struct {
 	async.SimpleTask
-	db         database.Database
-	grafanaURL string
-	client     *grafanaapi.GrafanaHTTPAPI
+	db                database.Database
+	metricUsageClient client.Client
+	grafanaURL        string
+	grafanaClient     *grafanaapi.GrafanaHTTPAPI
 }
 
 func (c *grafanaCollector) Execute(ctx context.Context, _ context.CancelFunc) error {
@@ -74,7 +84,14 @@ func (c *grafanaCollector) Execute(ctx context.Context, _ context.CancelFunc) er
 		c.extractMetricUsage(metricUsage, dashboard)
 	}
 	if len(metricUsage) > 0 {
-		c.db.EnqueueUsage(metricUsage)
+		if c.metricUsageClient != nil {
+			// In this case, that means we have to send the data to a remote server.
+			if sendErr := c.metricUsageClient.Usage(metricUsage); sendErr != nil {
+				logrus.WithError(sendErr).Error("Failed to send usage metric")
+			}
+		} else {
+			c.db.EnqueueUsage(metricUsage)
+		}
 	}
 	return nil
 }
@@ -103,7 +120,7 @@ func (c *grafanaCollector) extractMetricUsageFromPanels(metricUsage map[string]*
 }
 
 func (c *grafanaCollector) getDashboard(uid string) (*simplifiedDashboard, error) {
-	response, err := c.client.Dashboards.GetDashboardByUID(uid)
+	response, err := c.grafanaClient.Dashboards.GetDashboardByUID(uid)
 	if err != nil {
 		return nil, err
 	}
@@ -123,7 +140,7 @@ func (c *grafanaCollector) collectAllDashboardUID(ctx context.Context) ([]*grafa
 	searchType := "dash-db"
 
 	for searchOk {
-		nextPageResult, err := c.client.Search.Search(&search.SearchParams{
+		nextPageResult, err := c.grafanaClient.Search.Search(&search.SearchParams{
 			Context: ctx,
 			Type:    &searchType,
 			Page:    &currentPage,

diff --git a/source/perses/perses.go b/source/perses/perses.go
@@ -10,6 +10,7 @@ import (
 	"github.com/perses/metrics-usage/config"
 	"github.com/perses/metrics-usage/database"
 	modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1"
+	"github.com/perses/metrics-usage/pkg/client"
 	"github.com/perses/metrics-usage/utils"
 	"github.com/perses/metrics-usage/utils/prometheus"
 	"github.com/perses/perses/go-sdk/prometheus/query"
@@ -39,23 +40,32 @@ func NewCollector(db database.Database, cfg config.PersesCollector) (async.Simpl
 	if err != nil {
 		return nil, err
 	}
+	var metricUsageClient client.Client
+	if cfg.MetricUsageClient != nil {
+		metricUsageClient, err = client.New(*cfg.MetricUsageClient)
+		if err != nil {
+			return nil, err
+		}
+	}
 	return &persesCollector{
-		SimpleTask: nil,
-		client:     persesClientV1.NewWithClient(restClient).Dashboard(""),
-		db:         db,
-		persesURL:  cfg.HTTPClient.URL.String(),
+		SimpleTask:        nil,
+		persesClient:      persesClientV1.NewWithClient(restClient).Dashboard(""),
+		db:                db,
+		metricUsageClient: metricUsageClient,
+		persesURL:         cfg.HTTPClient.URL.String(),
 	}, nil
 }
 
 type persesCollector struct {
 	async.SimpleTask
-	client    persesClientV1.DashboardInterface
-	db        database.Database
-	persesURL string
+	persesClient      persesClientV1.DashboardInterface
+	db                database.Database
+	metricUsageClient client.Client
+	persesURL         string
 }
 
 func (c *persesCollector) Execute(_ context.Context, _ context.CancelFunc) error {
-	dashboards, err := c.client.List("")
+	dashboards, err := c.persesClient.List("")
 	if err != nil {
 		logrus.WithError(err).Error("Failed to get dashboards")
 		return nil
@@ -68,7 +78,14 @@ func (c *persesCollector) Execute(_ context.Context, _ context.CancelFunc) error
 		c.extractMetricUsageFromPanels(metricUsage, dash.Spec.Panels, dash)
 	}
 	if len(metricUsage) > 0 {
-		c.db.EnqueueUsage(metricUsage)
+		if c.metricUsageClient != nil {
+			// In this case, that means we have to send the data to a remote server.
+			if sendErr := c.metricUsageClient.Usage(metricUsage); sendErr != nil {
+				logrus.WithError(sendErr).Error("Failed to send usage metric")
+			}
+		} else {
+			c.db.EnqueueUsage(metricUsage)
+		}
 	}
 	return nil
 }

diff --git a/source/rules/rules.go b/source/rules/rules.go
@@ -7,6 +7,7 @@ import (
 	"github.com/perses/metrics-usage/config"
 	"github.com/perses/metrics-usage/database"
 	modelAPIV1 "github.com/perses/metrics-usage/pkg/api/v1"
+	"github.com/perses/metrics-usage/pkg/client"
 	"github.com/perses/metrics-usage/utils"
 	"github.com/perses/metrics-usage/utils/prometheus"
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
@@ -18,29 +19,45 @@ func NewCollector(db database.Database, cfg *config.RulesCollector) (async.Simpl
 	if err != nil {
 		return nil, err
 	}
+	var metricUsageClient client.Client
+	if cfg.MetricUsageClient != nil {
+		metricUsageClient, err = client.New(*cfg.MetricUsageClient)
+		if err != nil {
+			return nil, err
+		}
+	}
 	return &rulesCollector{
-		client:  promClient,
-		db:      db,
-		promURL: cfg.HTTPClient.URL.String(),
+		promClient:        promClient,
+		db:                db,
+		metricUsageClient: metricUsageClient,
+		promURL:           cfg.HTTPClient.URL.String(),
 	}, nil
 }
 
 type rulesCollector struct {
 	async.SimpleTask
-	client  v1.API
-	db      database.Database
-	promURL string
+	promClient        v1.API
+	db                database.Database
+	metricUsageClient client.Client
+	promURL           string
 }
 
 func (c *rulesCollector) Execute(ctx context.Context, _ context.CancelFunc) error {
-	result, err := c.client.Rules(ctx)
+	result, err := c.promClient.Rules(ctx)
 	if err != nil {
 		logrus.WithError(err).Error("Failed to get rules")
 		return nil
 	}
 	metricUsage := extractMetricUsageFromRules(result.Groups, c.promURL)
 	if len(metricUsage) > 0 {
-		c.db.EnqueueUsage(metricUsage)
+		if c.metricUsageClient != nil {
+			// In this case, that means we have to send the data to a remote server.
+			if sendErr := c.metricUsageClient.Usage(metricUsage); sendErr != nil {
+				logrus.WithError(sendErr).Error("Failed to send usage metric")
+			}
+		} else {
+			c.db.EnqueueUsage(metricUsage)
+		}
 	}
 	return nil
 }