Merge pull request #337 from overmindtech/334-review-tagging-to-ensur…

…e-it-fails-gracefully Handle tag errors in a uniform way
overmindtech · Nov 10, 2023 · a3fecb0 · a3fecb0
2 parents 08afbf6 + 8d3655d
commit a3fecb0
Show file tree

Hide file tree

Showing 20 changed files with 111 additions and 59 deletions.
diff --git a/sources/cloudfront/distribution.go b/sources/cloudfront/distribution.go
@@ -2,7 +2,6 @@ package cloudfront
 
 import (
 	"context"
-	"fmt"
 	"regexp"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -29,13 +28,17 @@ func distributionGetFunc(ctx context.Context, client CloudFrontClient, scope str
 		}
 	}
 
+	var tags map[string]string
+
 	// get tags
 	tagsOut, err := client.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{
 		Resource: d.ARN,
 	})
 
-	if err != nil {
-		return nil, fmt.Errorf("failed to get tags for distribution %v: %w", *d.ARN, err)
+	if err == nil {
+		tags = tagsToMap(tagsOut.Tags)
+	} else {
+		tags = sources.HandleTagsError(ctx, err)
 	}
 
 	attributes, err := sources.ToAttributesCase(d)
@@ -49,7 +52,7 @@ func distributionGetFunc(ctx context.Context, client CloudFrontClient, scope str
 		UniqueAttribute: "id",
 		Attributes:      attributes,
 		Scope:           scope,
-		Tags:            tagsToMap(tagsOut.Tags),
+		Tags:            tags,
 	}
 
 	if err != nil {

diff --git a/sources/cloudfront/streaming_distribution.go b/sources/cloudfront/streaming_distribution.go
@@ -26,11 +26,19 @@ func streamingDistributionGetFunc(ctx context.Context, client CloudFrontClient,
 		}
 	}
 
+	var tags map[string]string
+
 	// Get the tags
 	tagsOut, err := client.ListTagsForResource(ctx, &cloudfront.ListTagsForResourceInput{
 		Resource: d.ARN,
 	})
 
+	if err == nil {
+		tags = tagsToMap(tagsOut.Tags)
+	} else {
+		tags = sources.HandleTagsError(ctx, err)
+	}
+
 	if err != nil {
 		return nil, fmt.Errorf("failed to get tags for streaming distribution %v: %w", *d.Id, err)
 	}
@@ -46,7 +54,7 @@ func streamingDistributionGetFunc(ctx context.Context, client CloudFrontClient,
 		UniqueAttribute: "id",
 		Attributes:      attributes,
 		Scope:           scope,
-		Tags:            tagsToMap(tagsOut.Tags),
+		Tags:            tags,
 	}
 
 	if d.Status != nil {

diff --git a/sources/cloudwatch/alarm.go b/sources/cloudwatch/alarm.go
@@ -86,11 +86,19 @@ func alarmOutputMapper(ctx context.Context, client CloudwatchClient, scope strin
 			return nil, err
 		}
 
+		var tags map[string]string
+
 		// Get the tags
 		tagsOut, err := client.ListTagsForResource(ctx, &cloudwatch.ListTagsForResourceInput{
 			ResourceARN: arn,
 		})
 
+		if err == nil {
+			tags = tagsToMap(tagsOut.Tags)
+		} else {
+			tags = sources.HandleTagsError(ctx, err)
+		}
+
 		if err != nil {
 			return nil, err
 		}
@@ -100,7 +108,7 @@ func alarmOutputMapper(ctx context.Context, client CloudwatchClient, scope strin
 			UniqueAttribute: "alarmName",
 			Scope:           scope,
 			Attributes:      attrs,
-			Tags:            tagsToMap(tagsOut.Tags),
+			Tags:            tags,
 		}
 
 		// Combine all actions so that we can link the targeted item

diff --git a/sources/dynamodb/table.go b/sources/dynamodb/table.go
@@ -3,7 +3,6 @@ package dynamodb
 import (
 	"context"
 	"errors"
-	"fmt"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
@@ -35,7 +34,8 @@ func tableGetFunc(ctx context.Context, client Client, scope string, input *dynam
 		})
 
 		if err != nil {
-			return nil, fmt.Errorf("failed to get tags for table: %w", err)
+			tagsMap = sources.HandleTagsError(ctx, err)
+			break
 		}
 
 		// Add tags to map

diff --git a/sources/elb/elb.go b/sources/elb/elb.go
@@ -44,13 +44,12 @@ func loadBalancerOutputMapper(ctx context.Context, client elbClient, scope strin
 		tagsOut, err := client.DescribeTags(ctx, &elb.DescribeTagsInput{
 			LoadBalancerNames: loadBalancerNames,
 		})
-		if err != nil {
-			return nil, err
-		}
 
-		for _, tagDesc := range tagsOut.TagDescriptions {
-			if tagDesc.LoadBalancerName != nil {
-				tagsMap[*tagDesc.LoadBalancerName] = tagDesc.Tags
+		if err == nil {
+			for _, tagDesc := range tagsOut.TagDescriptions {
+				if tagDesc.LoadBalancerName != nil {
+					tagsMap[*tagDesc.LoadBalancerName] = tagDesc.Tags
+				}
 			}
 		}
 	}
@@ -65,7 +64,11 @@ func loadBalancerOutputMapper(ctx context.Context, client elbClient, scope strin
 		var tags map[string]string
 
 		if desc.LoadBalancerName != nil {
-			tags = tagsToMap(tagsMap[*desc.LoadBalancerName])
+			m, ok := tagsMap[*desc.LoadBalancerName]
+
+			if ok {
+				tags = tagsToMap(m)
+			}
 		}
 
 		item := sdp.Item{

diff --git a/sources/elbv2/elb.go b/sources/elbv2/elb.go
@@ -22,11 +22,7 @@ func loadBalancerOutputMapper(ctx context.Context, client elbClient, scope strin
 		}
 	}
 
-	tagsMap, err := getTagsMap(ctx, client, arns)
-
-	if err != nil {
-		return nil, err
-	}
+	tagsMap := getTagsMap(ctx, client, arns)
 
 	for _, lb := range output.LoadBalancers {
 		attrs, err := sources.ToAttributesCase(lb)

diff --git a/sources/elbv2/listener.go b/sources/elbv2/listener.go
@@ -25,11 +25,7 @@ func listenerOutputMapper(ctx context.Context, client elbClient, scope string, _
 		}
 	}
 
-	tagsMap, err := getTagsMap(ctx, client, arns)
-
-	if err != nil {
-		return nil, err
-	}
+	tagsMap := getTagsMap(ctx, client, arns)
 
 	for _, listener := range output.Listeners {
 		// Redact the client secret and replace with the first 12 characters of

diff --git a/sources/elbv2/rule.go b/sources/elbv2/rule.go
@@ -20,11 +20,7 @@ func ruleOutputMapper(ctx context.Context, client elbClient, scope string, _ *el
 		}
 	}
 
-	tagsMap, err := getTagsMap(ctx, client, ruleArns)
-
-	if err != nil {
-		return nil, err
-	}
+	tagsMap := getTagsMap(ctx, client, ruleArns)
 
 	for _, rule := range output.Rules {
 		attrs, err := sources.ToAttributesCase(rule)

diff --git a/sources/elbv2/shared.go b/sources/elbv2/shared.go
@@ -5,6 +5,7 @@ import (
 
 	elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2"
 	"github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types"
+	"github.com/overmindtech/aws-source/sources"
 )
 
 type elbClient interface {
@@ -28,15 +29,22 @@ func tagsToMap(tags []types.Tag) map[string]string {
 }
 
 // Gets a map of ARN to tags (in map[string]string format) for the given ARNs
-func getTagsMap(ctx context.Context, client elbClient, arns []string) (map[string]map[string]string, error) {
+func getTagsMap(ctx context.Context, client elbClient, arns []string) map[string]map[string]string {
 	tagsMap := make(map[string]map[string]string)
 
 	if len(arns) > 0 {
 		tagsOut, err := client.DescribeTags(ctx, &elbv2.DescribeTagsInput{
 			ResourceArns: arns,
 		})
 		if err != nil {
-			return nil, err
+			tags := sources.HandleTagsError(ctx, err)
+
+			// Set these tags for all ARNs
+			for _, arn := range arns {
+				tagsMap[arn] = tags
+			}
+
+			return tagsMap
 		}
 
 		for _, tagDescription := range tagsOut.TagDescriptions {
@@ -46,5 +54,5 @@ func getTagsMap(ctx context.Context, client elbClient, arns []string) (map[strin
 		}
 	}
 
-	return tagsMap, nil
+	return tagsMap
 }
diff --git a/sources/elbv2/target_group.go b/sources/elbv2/target_group.go
@@ -20,11 +20,7 @@ func targetGroupOutputMapper(ctx context.Context, client elbClient, scope string
 		}
 	}
 
-	tagsMap, err := getTagsMap(ctx, client, tgArns)
-
-	if err != nil {
-		return nil, err
-	}
+	tagsMap := getTagsMap(ctx, client, tgArns)
 
 	for _, tg := range output.TargetGroups {
 		attrs, err := sources.ToAttributesCase(tg)

diff --git a/sources/get_list_source.go b/sources/get_list_source.go
@@ -163,8 +163,7 @@ func (s *GetListSource[AWSItem, ClientStruct, Options]) Get(ctx context.Context,
 	if s.ListTagsFunc != nil {
 		item.Tags, err = s.ListTagsFunc(ctx, awsItem, s.Client)
 		if err != nil {
-			s.cache.StoreError(err, s.CacheDuration, ck)
-			return nil, WrapAWSError(err)
+			item.Tags = HandleTagsError(ctx, err)
 		}
 	}
 

diff --git a/sources/iam/instance_profile.go b/sources/iam/instance_profile.go
@@ -97,7 +97,7 @@ func instanceProfileItemMapper(scope string, awsItem *types.InstanceProfile) (*s
 	return &item, nil
 }
 
-func instanceProfileListTagsFunc(ctx context.Context, ip *types.InstanceProfile, client *iam.Client) (map[string]string, error) {
+func instanceProfileListTagsFunc(ctx context.Context, ip *types.InstanceProfile, client *iam.Client) map[string]string {
 	tags := make(map[string]string)
 
 	paginator := iam.NewListInstanceProfileTagsPaginator(client, &iam.ListInstanceProfileTagsInput{
@@ -108,7 +108,7 @@ func instanceProfileListTagsFunc(ctx context.Context, ip *types.InstanceProfile,
 		out, err := paginator.NextPage(ctx)
 
 		if err != nil {
-			return nil, err
+			return sources.HandleTagsError(ctx, err)
 		}
 
 		for _, tag := range out.Tags {
@@ -118,7 +118,7 @@ func instanceProfileListTagsFunc(ctx context.Context, ip *types.InstanceProfile,
 		}
 	}
 
-	return tags, nil
+	return tags
 }
 
 //go:generate docgen ../../docs-data
@@ -147,7 +147,7 @@ func NewInstanceProfileSource(config aws.Config, accountID string, region string
 		},
 		ListTagsFunc: func(ctx context.Context, ip *types.InstanceProfile, c *iam.Client) (map[string]string, error) {
 			limit.Wait(ctx) // Wait for rate limiting
-			return instanceProfileListTagsFunc(ctx, ip, c)
+			return instanceProfileListTagsFunc(ctx, ip, c), nil
 		},
 		ItemMapper: instanceProfileItemMapper,
 	}

diff --git a/sources/iam/policy.go b/sources/iam/policy.go
@@ -256,7 +256,7 @@ func policyListTagsFunc(ctx context.Context, p *PolicyDetails, client IAMClient)
 		out, err := paginator.NextPage(ctx)
 
 		if err != nil {
-			return nil, err
+			return sources.HandleTagsError(ctx, err), nil
 		}
 
 		for _, tag := range out.Tags {

diff --git a/sources/iam/role.go b/sources/iam/role.go
@@ -279,7 +279,7 @@ func roleListTagsFunc(ctx context.Context, r *RoleDetails, client IAMClient) (ma
 		out, err := paginator.NextPage(ctx)
 
 		if err != nil {
-			return nil, err
+			return sources.HandleTagsError(ctx, err), nil
 		}
 
 		for _, tag := range out.Tags {

diff --git a/sources/iam/user.go b/sources/iam/user.go
@@ -155,7 +155,7 @@ func userListTagsFunc(ctx context.Context, u *UserDetails, client IAMClient) (ma
 		out, err := paginator.NextPage(ctx)
 
 		if err != nil {
-			return nil, err
+			return sources.HandleTagsError(ctx, err), nil
 		}
 
 		for _, tag := range out.Tags {

diff --git a/sources/rds/db_cluster.go b/sources/rds/db_cluster.go
@@ -13,13 +13,17 @@ func dBClusterOutputMapper(ctx context.Context, client rdsClient, scope string,
 	items := make([]*sdp.Item, 0)
 
 	for _, cluster := range output.DBClusters {
+		var tags map[string]string
+
 		// Get tags for the cluster
 		tagsOut, err := client.ListTagsForResource(ctx, &rds.ListTagsForResourceInput{
 			ResourceName: cluster.DBClusterArn,
 		})
 
-		if err != nil {
-			return nil, err
+		if err == nil {
+			tags = tagsToMap(tagsOut.TagList)
+		} else {
+			tags = sources.HandleTagsError(ctx, err)
 		}
 
 		attributes, err := sources.ToAttributesCase(cluster)
@@ -33,7 +37,7 @@ func dBClusterOutputMapper(ctx context.Context, client rdsClient, scope string,
 			UniqueAttribute: "dBClusterIdentifier",
 			Attributes:      attributes,
 			Scope:           scope,
-			Tags:            tagsToMap(tagsOut.TagList),
+			Tags:            tags,
 		}
 
 		var a *sources.ARN

diff --git a/sources/rds/db_instance.go b/sources/rds/db_instance.go
@@ -76,13 +76,17 @@ func dBInstanceOutputMapper(ctx context.Context, client rdsClient, scope string,
 	items := make([]*sdp.Item, 0)
 
 	for _, instance := range output.DBInstances {
+		var tags map[string]string
+
 		// Get the tags for the instance
 		tagsOut, err := client.ListTagsForResource(ctx, &rds.ListTagsForResourceInput{
 			ResourceName: instance.DBInstanceArn,
 		})
 
-		if err != nil {
-			return nil, err
+		if err == nil {
+			tags = tagsToMap(tagsOut.TagList)
+		} else {
+			tags = sources.HandleTagsError(ctx, err)
 		}
 
 		var dbSubnetGroup *string
@@ -106,7 +110,7 @@ func dBInstanceOutputMapper(ctx context.Context, client rdsClient, scope string,
 			UniqueAttribute: "dBInstanceIdentifier",
 			Attributes:      attributes,
 			Scope:           scope,
-			Tags:            tagsToMap(tagsOut.TagList),
+			Tags:            tags,
 		}
 
 		if instance.DBInstanceStatus != nil {

diff --git a/sources/rds/db_subnet_group.go b/sources/rds/db_subnet_group.go
@@ -13,13 +13,17 @@ func dBSubnetGroupOutputMapper(ctx context.Context, client rdsClient, scope stri
 	items := make([]*sdp.Item, 0)
 
 	for _, sg := range output.DBSubnetGroups {
+		var tags map[string]string
+
 		// Get tags
 		tagsOut, err := client.ListTagsForResource(ctx, &rds.ListTagsForResourceInput{
 			ResourceName: sg.DBSubnetGroupArn,
 		})
 
-		if err != nil {
-			return nil, err
+		if err == nil {
+			tags = tagsToMap(tagsOut.TagList)
+		} else {
+			tags = sources.HandleTagsError(ctx, err)
 		}
 
 		attributes, err := sources.ToAttributesCase(sg)
@@ -33,7 +37,7 @@ func dBSubnetGroupOutputMapper(ctx context.Context, client rdsClient, scope stri
 			UniqueAttribute: "dBSubnetGroupName",
 			Attributes:      attributes,
 			Scope:           scope,
-			Tags:            tagsToMap(tagsOut.TagList),
+			Tags:            tags,
 		}
 
 		var a *sources.ARN