move duplicated fields from CloudwatchData to a new JobContext (#1106)

prometheus-community · Sep 11, 2023 · 36e4d8d · 36e4d8d
1 parent 039ba23
commit 36e4d8d
Show file tree

Hide file tree

Showing 10 changed files with 181 additions and 171 deletions.
diff --git a/go.mod b/go.mod
@@ -19,6 +19,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/storagegateway v1.19.6
 	github.com/aws/aws-sdk-go-v2/service/sts v1.21.5
 	github.com/aws/smithy-go v1.14.2
+	github.com/brianvoe/gofakeit/v6 v6.23.1
 	github.com/go-kit/log v0.2.1
 	github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db
 	github.com/prometheus/client_golang v1.16.0

diff --git a/go.sum b/go.sum
@@ -46,6 +46,8 @@ github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ=
 github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/brianvoe/gofakeit/v6 v6.23.1 h1:k2gX0hQpJStvixDbbw8oJOvPBg0XmHJWbSOF5JkiUHw=
+github.com/brianvoe/gofakeit/v6 v6.23.1/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=

diff --git a/pkg/job/custom.go b/pkg/job/custom.go
@@ -17,8 +17,6 @@ func runCustomNamespaceJob(
 	ctx context.Context,
 	logger logging.Logger,
 	job *config.CustomNamespace,
-	region string,
-	accountID string,
 	clientCloudwatch cloudwatch.Client,
 	metricsPerQuery int,
 ) []*model.CloudwatchData {
@@ -27,7 +25,7 @@ func runCustomNamespaceJob(
 	mux := &sync.Mutex{}
 	var wg sync.WaitGroup
 
-	getMetricDatas := getMetricDataForQueriesForCustomNamespace(ctx, job, region, accountID, clientCloudwatch, logger)
+	getMetricDatas := getMetricDataForQueriesForCustomNamespace(ctx, job, clientCloudwatch, logger)
 	metricDataLength := len(getMetricDatas)
 	if metricDataLength == 0 {
 		logger.Debug("No metrics data found")
@@ -87,8 +85,6 @@ func findGetMetricDataByIDForCustomNamespace(getMetricDatas []*model.CloudwatchD
 func getMetricDataForQueriesForCustomNamespace(
 	ctx context.Context,
 	customNamespaceJob *config.CustomNamespace,
-	region string,
-	accountID string,
 	clientCloudwatch cloudwatch.Client,
 	logger logging.Logger,
 ) []*model.CloudwatchData {
@@ -128,10 +124,7 @@ func getMetricDataForQueriesForCustomNamespace(
 						Statistics:             []string{stats},
 						NilToZero:              metric.NilToZero,
 						AddCloudwatchTimestamp: metric.AddCloudwatchTimestamp,
-						CustomTags:             customNamespaceJob.CustomTags,
 						Dimensions:             cwMetric.Dimensions,
-						Region:                 &region,
-						AccountID:              &accountID,
 						Period:                 metric.Period,
 					})
 				}

diff --git a/pkg/job/discovery.go b/pkg/job/discovery.go
@@ -29,7 +29,6 @@ func runDiscoveryJob(
 	logger logging.Logger,
 	job *config.Job,
 	region string,
-	accountID string,
 	tagsOnMetrics model.ExportedTagsOnMetrics,
 	clientTag tagging.Client,
 	clientCloudwatch cloudwatch.Client,
@@ -55,7 +54,7 @@ func runDiscoveryJob(
 	}
 
 	svc := config.SupportedServices.GetService(job.Type)
-	getMetricDatas := getMetricDataForQueries(ctx, logger, job, svc, region, accountID, tagsOnMetrics, clientCloudwatch, resources)
+	getMetricDatas := getMetricDataForQueries(ctx, logger, job, svc, tagsOnMetrics, clientCloudwatch, resources)
 	metricDataLength := len(getMetricDatas)
 	if metricDataLength == 0 {
 		logger.Info("No metrics data found")
@@ -175,8 +174,6 @@ func getMetricDataForQueries(
 	logger logging.Logger,
 	discoveryJob *config.Job,
 	svc *config.ServiceConfig,
-	region string,
-	accountID string,
 	tagsOnMetrics model.ExportedTagsOnMetrics,
 	clientCloudwatch cloudwatch.Client,
 	resources []*model.TaggedResource,
@@ -204,7 +201,7 @@ func getMetricDataForQueries(
 				}
 
 				_, err := clientCloudwatch.ListMetrics(ctx, svc.Namespace, metric, discoveryJob.RecentlyActiveOnly, func(page []*model.Metric) {
-					data := getFilteredMetricDatas(logger, region, accountID, discoveryJob.Type, discoveryJob.CustomTags, tagsOnMetrics, page, discoveryJob.DimensionNameRequirements, metric, assoc)
+					data := getFilteredMetricDatas(logger, discoveryJob.Type, tagsOnMetrics, page, discoveryJob.DimensionNameRequirements, metric, assoc)
 
 					mux.Lock()
 					getMetricDatas = append(getMetricDatas, data...)
@@ -234,7 +231,7 @@ func getMetricDataForQueries(
 					return
 				}
 
-				data := getFilteredMetricDatas(logger, region, accountID, discoveryJob.Type, discoveryJob.CustomTags, tagsOnMetrics, metricsList, discoveryJob.DimensionNameRequirements, metric, assoc)
+				data := getFilteredMetricDatas(logger, discoveryJob.Type, tagsOnMetrics, metricsList, discoveryJob.DimensionNameRequirements, metric, assoc)
 
 				mux.Lock()
 				getMetricDatas = append(getMetricDatas, data...)
@@ -249,10 +246,7 @@ func getMetricDataForQueries(
 
 func getFilteredMetricDatas(
 	logger logging.Logger,
-	region string,
-	accountID string,
 	namespace string,
-	customTags []model.Tag,
 	tagsOnMetrics model.ExportedTagsOnMetrics,
 	metricsList []*model.Metric,
 	dimensionNameList []string,
@@ -284,8 +278,8 @@ func getFilteredMetricDatas(
 				Namespace: namespace,
 			}
 		}
-		metricTags := resource.MetricTags(tagsOnMetrics)
 
+		metricTags := resource.MetricTags(tagsOnMetrics)
 		for _, stats := range m.Statistics {
 			id := fmt.Sprintf("id_%d", rand.Int())
 
@@ -298,10 +292,7 @@ func getFilteredMetricDatas(
 				NilToZero:              m.NilToZero,
 				AddCloudwatchTimestamp: m.AddCloudwatchTimestamp,
 				Tags:                   metricTags,
-				CustomTags:             customTags,
 				Dimensions:             cwMetric.Dimensions,
-				Region:                 &region,
-				AccountID:              &accountID,
 				Period:                 m.Period,
 			})
 		}

diff --git a/pkg/job/discovery_test.go b/pkg/job/discovery_test.go
@@ -91,7 +91,6 @@ func Test_getFilteredMetricDatas(t *testing.T) {
 			},
 			[]model.CloudwatchData{
 				{
-					AccountID:              aws.String("123123123123"),
 					AddCloudwatchTimestamp: aws.Bool(false),
 					Dimensions: []*model.Dimension{
 						{
@@ -108,7 +107,6 @@ func Test_getFilteredMetricDatas(t *testing.T) {
 					Namespace: aws.String("efs"),
 					NilToZero: aws.Bool(false),
 					Period:    60,
-					Region:    aws.String("us-east-1"),
 					Statistics: []string{
 						"Average",
 					},
@@ -178,7 +176,6 @@ func Test_getFilteredMetricDatas(t *testing.T) {
 			},
 			[]model.CloudwatchData{
 				{
-					AccountID:              aws.String("123123123123"),
 					AddCloudwatchTimestamp: aws.Bool(false),
 					Dimensions: []*model.Dimension{
 						{
@@ -191,7 +188,6 @@ func Test_getFilteredMetricDatas(t *testing.T) {
 					Namespace: aws.String("ec2"),
 					NilToZero: aws.Bool(false),
 					Period:    60,
-					Region:    aws.String("us-east-1"),
 					Statistics: []string{
 						"Average",
 					},
@@ -261,7 +257,6 @@ func Test_getFilteredMetricDatas(t *testing.T) {
 			},
 			[]model.CloudwatchData{
 				{
-					AccountID:              aws.String("123123123123"),
 					AddCloudwatchTimestamp: aws.Bool(false),
 					Dimensions: []*model.Dimension{
 						{
@@ -274,7 +269,6 @@ func Test_getFilteredMetricDatas(t *testing.T) {
 					Namespace: aws.String("kafka"),
 					NilToZero: aws.Bool(false),
 					Period:    60,
-					Region:    aws.String("us-east-1"),
 					Statistics: []string{
 						"Average",
 					},
@@ -386,7 +380,6 @@ func Test_getFilteredMetricDatas(t *testing.T) {
 			},
 			[]model.CloudwatchData{
 				{
-					AccountID:              aws.String("123123123123"),
 					AddCloudwatchTimestamp: aws.Bool(false),
 					Dimensions: []*model.Dimension{
 						{
@@ -403,7 +396,6 @@ func Test_getFilteredMetricDatas(t *testing.T) {
 					Namespace: aws.String("alb"),
 					NilToZero: aws.Bool(false),
 					Period:    60,
-					Region:    aws.String("us-east-1"),
 					Statistics: []string{
 						"Sum",
 					},
@@ -415,14 +407,11 @@ func Test_getFilteredMetricDatas(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			assoc := associator.NewAssociator(tt.args.dimensionRegexps, tt.args.resources)
-			metricDatas := getFilteredMetricDatas(logging.NewNopLogger(), tt.args.region, tt.args.accountID, tt.args.namespace, tt.args.customTags, tt.args.tagsOnMetrics, tt.args.metricsList, tt.args.dimensionNameRequirements, tt.args.m, assoc)
+			metricDatas := getFilteredMetricDatas(logging.NewNopLogger(), tt.args.namespace, tt.args.tagsOnMetrics, tt.args.metricsList, tt.args.dimensionNameRequirements, tt.args.m, assoc)
 			if len(metricDatas) != len(tt.wantGetMetricsData) {
 				t.Errorf("len(getFilteredMetricDatas()) = %v, want %v", len(metricDatas), len(tt.wantGetMetricsData))
 			}
 			for i, got := range metricDatas {
-				if *got.AccountID != *tt.wantGetMetricsData[i].AccountID {
-					t.Errorf("getFilteredMetricDatas().AccountId = %v, want %v", *got.AccountID, *tt.wantGetMetricsData[i].AccountID)
-				}
 				if *got.ID != *tt.wantGetMetricsData[i].ID {
 					t.Errorf("getFilteredMetricDatas().ID = %v, want %v", *got.ID, *tt.wantGetMetricsData[i].ID)
 				}
@@ -447,9 +436,6 @@ func Test_getFilteredMetricDatas(t *testing.T) {
 				if !reflect.DeepEqual(got.Statistics, tt.wantGetMetricsData[i].Statistics) {
 					t.Errorf("getFilteredMetricDatas().Statistics = %+v, want %+v", got.Statistics, tt.wantGetMetricsData[i].Statistics)
 				}
-				if *got.Region != *tt.wantGetMetricsData[i].Region {
-					t.Errorf("getFilteredMetricDatas().Region = %v, want %v", *got.Region, *tt.wantGetMetricsData[i].Region)
-				}
 				if !reflect.DeepEqual(got.Tags, tt.wantGetMetricsData[i].Tags) {
 					t.Errorf("getFilteredMetricDatas().Tags = %+v, want %+v", got.Tags, tt.wantGetMetricsData[i].Tags)
 				}
@@ -460,7 +446,6 @@ func Test_getFilteredMetricDatas(t *testing.T) {
 
 func getSampleMetricDatas(id string) *model.CloudwatchData {
 	return &model.CloudwatchData{
-		AccountID:              aws.String("123123123123"),
 		AddCloudwatchTimestamp: aws.Bool(false),
 		Dimensions: []*model.Dimension{
 			{
@@ -478,7 +463,6 @@ func getSampleMetricDatas(id string) *model.CloudwatchData {
 		Namespace: aws.String("efs"),
 		NilToZero: aws.Bool(false),
 		Period:    60,
-		Region:    aws.String("us-east-1"),
 		Statistics: []string{
 			"Average",
 		},

diff --git a/pkg/job/scrape.go b/pkg/job/scrape.go
@@ -18,10 +18,10 @@ func ScrapeAwsData(
 	metricsPerQuery int,
 	cloudWatchAPIConcurrency int,
 	taggingAPIConcurrency int,
-) ([]*model.TaggedResource, []*model.CloudwatchData) {
+) ([][]*model.TaggedResource, []model.CloudwatchMetricResult) {
 	mux := &sync.Mutex{}
-	cwData := make([]*model.CloudwatchData, 0)
-	awsInfoData := make([]*model.TaggedResource, 0)
+	cwData := make([]model.CloudwatchMetricResult, 0)
+	awsInfoData := make([][]*model.TaggedResource, 0)
 	var wg sync.WaitGroup
 
 	for _, discoveryJob := range cfg.Discovery.Jobs {
@@ -38,16 +38,24 @@ func ScrapeAwsData(
 					}
 					jobLogger = jobLogger.With("account", accountID)
 
-					resources, metrics := runDiscoveryJob(ctx, jobLogger, discoveryJob, region, accountID, cfg.Discovery.ExportedTagsOnMetrics, factory.GetTaggingClient(region, role, taggingAPIConcurrency), factory.GetCloudwatchClient(region, role, cloudWatchAPIConcurrency), metricsPerQuery, cloudWatchAPIConcurrency)
+					resources, metrics := runDiscoveryJob(ctx, jobLogger, discoveryJob, region, cfg.Discovery.ExportedTagsOnMetrics, factory.GetTaggingClient(region, role, taggingAPIConcurrency), factory.GetCloudwatchClient(region, role, cloudWatchAPIConcurrency), metricsPerQuery, cloudWatchAPIConcurrency)
+					metricResult := model.CloudwatchMetricResult{
+						Context: &model.JobContext{
+							Region:     region,
+							AccountID:  accountID,
+							CustomTags: discoveryJob.CustomTags,
+						},
+						Data: metrics,
+					}
 
 					addDataToOutput := len(metrics) != 0
 					if config.FlagsFromCtx(ctx).IsFeatureEnabled(config.AlwaysReturnInfoMetrics) {
 						addDataToOutput = addDataToOutput || len(resources) != 0
 					}
 					if addDataToOutput {
 						mux.Lock()
-						awsInfoData = append(awsInfoData, resources...)
-						cwData = append(cwData, metrics...)
+						awsInfoData = append(awsInfoData, resources)
+						cwData = append(cwData, metricResult)
 						mux.Unlock()
 					}
 				}(discoveryJob, region, role)
@@ -69,10 +77,17 @@ func ScrapeAwsData(
 					}
 					jobLogger = jobLogger.With("account", accountID)
 
-					metrics := runStaticJob(ctx, jobLogger, staticJob, region, accountID, factory.GetCloudwatchClient(region, role, cloudWatchAPIConcurrency))
-
+					metrics := runStaticJob(ctx, jobLogger, staticJob, factory.GetCloudwatchClient(region, role, cloudWatchAPIConcurrency))
+					metricResult := model.CloudwatchMetricResult{
+						Context: &model.JobContext{
+							Region:     region,
+							AccountID:  accountID,
+							CustomTags: staticJob.CustomTags,
+						},
+						Data: metrics,
+					}
 					mux.Lock()
-					cwData = append(cwData, metrics...)
+					cwData = append(cwData, metricResult)
 					mux.Unlock()
 				}(staticJob, region, role)
 			}
@@ -95,10 +110,17 @@ func ScrapeAwsData(
 					}
 					jobLogger = jobLogger.With("account", accountID)
 
-					metrics := runCustomNamespaceJob(ctx, jobLogger, customNamespaceJob, region, accountID, factory.GetCloudwatchClient(region, role, cloudWatchAPIConcurrency), metricsPerQuery)
-
+					metrics := runCustomNamespaceJob(ctx, jobLogger, customNamespaceJob, factory.GetCloudwatchClient(region, role, cloudWatchAPIConcurrency), metricsPerQuery)
+					metricResult := model.CloudwatchMetricResult{
+						Context: &model.JobContext{
+							Region:     region,
+							AccountID:  accountID,
+							CustomTags: customNamespaceJob.CustomTags,
+						},
+						Data: metrics,
+					}
 					mux.Lock()
-					cwData = append(cwData, metrics...)
+					cwData = append(cwData, metricResult)
 					mux.Unlock()
 				}(customNamespaceJob, region, role)
 			}

diff --git a/pkg/job/static.go b/pkg/job/static.go
@@ -14,8 +14,6 @@ func runStaticJob(
 	ctx context.Context,
 	logger logging.Logger,
 	resource *config.Static,
-	region string,
-	accountID string,
 	clientCloudwatch cloudwatch.Client,
 ) []*model.CloudwatchData {
 	cw := []*model.CloudwatchData{}
@@ -36,10 +34,7 @@ func runStaticJob(
 				Statistics:             metric.Statistics,
 				NilToZero:              metric.NilToZero,
 				AddCloudwatchTimestamp: metric.AddCloudwatchTimestamp,
-				CustomTags:             resource.CustomTags,
 				Dimensions:             createStaticDimensions(resource.Dimensions),
-				Region:                 &region,
-				AccountID:              &accountID,
 			}
 
 			data.Points = clientCloudwatch.GetMetricStatistics(ctx, logger, data.Dimensions, resource.Namespace, metric)

diff --git a/pkg/model/model.go b/pkg/model/model.go
@@ -57,6 +57,17 @@ type Datapoint struct {
 	Timestamp *time.Time
 }
 
+type CloudwatchMetricResult struct {
+	Context *JobContext
+	Data    []*CloudwatchData
+}
+
+type JobContext struct {
+	Region     string
+	AccountID  string
+	CustomTags []Tag
+}
+
 // CloudwatchData is an internal representation of a CloudWatch
 // metric with attached data points, metric and resource information.
 type CloudwatchData struct {
@@ -70,11 +81,8 @@ type CloudwatchData struct {
 	GetMetricDataTimestamps time.Time
 	NilToZero               *bool
 	AddCloudwatchTimestamp  *bool
-	CustomTags              []Tag
 	Tags                    []Tag
 	Dimensions              []*Dimension
-	Region                  *string
-	AccountID               *string
 	Period                  int64
 }