Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Increase map parallelism & improve wait tracking #297

Merged
merged 2 commits into from
Sep 22, 2023
+88 −88
Merged

Increase map parallelism & improve wait tracking #297

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
92a7acf
Increase map parallelism & improve wait tracking
dylanratcliffe Sep 22, 2023
525dba9
Standardise rate limiting
dylanratcliffe Sep 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion sources/autoscaling/auto_scaling_group.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,7 @@ func NewAutoScalingGroupSource(config aws.Config, accountID string, limit *sourc
return autoscaling.NewDescribeAutoScalingGroupsPaginator(client, params)
},
DescribeFunc: func(ctx context.Context, client *autoscaling.Client, input *autoscaling.DescribeAutoScalingGroupsInput) (*autoscaling.DescribeAutoScalingGroupsOutput, error) {
<-limit.C // Wait for rate limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for rate limiting
return client.DescribeAutoScalingGroups(ctx, input)
},
OutputMapper: autoScalingGroupOutputMapper,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/address.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ func NewAddressSource(config aws.Config, accountID string, limit *sources.LimitB
AccountID: accountID,
ItemType: "ec2-address",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeAddressesInput) (*ec2.DescribeAddressesOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeAddresses(ctx, input)
},
InputMapperGet: addressInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/az.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func NewAvailabilityZoneSource(config aws.Config, accountID string, limit *sourc
AccountID: accountID,
ItemType: "ec2-availability-zone",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeAvailabilityZonesInput) (*ec2.DescribeAvailabilityZonesOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeAvailabilityZones(ctx, input)
},
InputMapperGet: availabilityZoneInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/capacity_reservation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func NewCapacityReservationSource(config aws.Config, accountID string, limit *so
AccountID: accountID,
ItemType: "ec2-capacity-reservation",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeCapacityReservationsInput) (*ec2.DescribeCapacityReservationsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeCapacityReservations(ctx, input)
},
InputMapperGet: func(scope, query string) (*ec2.DescribeCapacityReservationsInput, error) {
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/capacity_reservation_fleet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ func NewCapacityReservationFleetSource(config aws.Config, accountID string, limi
AccountID: accountID,
ItemType: "ec2-capacity-reservation-fleet",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeCapacityReservationFleetsInput) (*ec2.DescribeCapacityReservationFleetsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeCapacityReservationFleets(ctx, input)
},
InputMapperGet: func(scope, query string) (*ec2.DescribeCapacityReservationFleetsInput, error) {
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/egress_internet_gateway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func NewEgressOnlyInternetGatewaySource(config aws.Config, accountID string, lim
AccountID: accountID,
ItemType: "ec2-egress-only-internet-gateway",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeEgressOnlyInternetGatewaysInput) (*ec2.DescribeEgressOnlyInternetGatewaysOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeEgressOnlyInternetGateways(ctx, input)
},
InputMapperGet: egressOnlyInternetGatewayInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/iam_instance_profile_association.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func NewIamInstanceProfileAssociationSource(config aws.Config, accountID string,
AccountID: accountID,
ItemType: "ec2-iam-instance-profile-association",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeIamInstanceProfileAssociationsInput) (*ec2.DescribeIamInstanceProfileAssociationsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeIamInstanceProfileAssociations(ctx, input)
},
InputMapperGet: func(scope, query string) (*ec2.DescribeIamInstanceProfileAssociationsInput, error) {
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/image.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func NewImageSource(config aws.Config, accountID string, limit *sources.LimitBuc
AccountID: accountID,
ItemType: "ec2-image",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeImages(ctx, input)
},
InputMapperGet: imageInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/instance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -489,7 +489,7 @@ func NewInstanceSource(config aws.Config, accountID string, limit *sources.Limit
AccountID: accountID,
ItemType: "ec2-instance",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeInstancesInput) (*ec2.DescribeInstancesOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeInstances(ctx, input)
},
InputMapperGet: instanceInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/instance_event_window.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func NewInstanceEventWindowSource(config aws.Config, accountID string, limit *so
AccountID: accountID,
ItemType: "ec2-instance-event-window",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeInstanceEventWindowsInput) (*ec2.DescribeInstanceEventWindowsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeInstanceEventWindows(ctx, input)
},
InputMapperGet: instanceEventWindowInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/instance_status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func NewInstanceStatusSource(config aws.Config, accountID string, limit *sources
AccountID: accountID,
ItemType: "ec2-instance-status",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeInstanceStatusInput) (*ec2.DescribeInstanceStatusOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeInstanceStatus(ctx, input)
},
InputMapperGet: instanceStatusInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/internet_gateway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func NewInternetGatewaySource(config aws.Config, accountID string, limit *source
AccountID: accountID,
ItemType: "ec2-internet-gateway",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeInternetGatewaysInput) (*ec2.DescribeInternetGatewaysOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeInternetGateways(ctx, input)
},
InputMapperGet: internetGatewayInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/key_pair.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func NewKeyPairSource(config aws.Config, accountID string, limit *sources.LimitB
AccountID: accountID,
ItemType: "ec2-key-pair",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeKeyPairsInput) (*ec2.DescribeKeyPairsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeKeyPairs(ctx, input)
},
InputMapperGet: keyPairInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/launch_template.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func NewLaunchTemplateSource(config aws.Config, accountID string, limit *sources
AccountID: accountID,
ItemType: "ec2-launch-template",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeLaunchTemplatesInput) (*ec2.DescribeLaunchTemplatesOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeLaunchTemplates(ctx, input)
},
InputMapperGet: launchTemplateInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/launch_template_version.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@ func NewLaunchTemplateVersionSource(config aws.Config, accountID string, limit *
AccountID: accountID,
ItemType: "ec2-launch-template-version",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeLaunchTemplateVersionsInput) (*ec2.DescribeLaunchTemplateVersionsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeLaunchTemplateVersions(ctx, input)
},
InputMapperGet: launchTemplateVersionInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/nat_gateway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ func NewNatGatewaySource(config aws.Config, accountID string, limit *sources.Lim
AccountID: accountID,
ItemType: "ec2-nat-gateway",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeNatGatewaysInput) (*ec2.DescribeNatGatewaysOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeNatGateways(ctx, input)
},
InputMapperGet: natGatewayInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/network_acl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func NewNetworkAclSource(config aws.Config, accountID string, limit *sources.Lim
AccountID: accountID,
ItemType: "ec2-network-acl",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeNetworkAclsInput) (*ec2.DescribeNetworkAclsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeNetworkAcls(ctx, input)
},
InputMapperGet: networkAclInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/network_interface.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ func NewNetworkInterfaceSource(config aws.Config, accountID string, limit *sourc
AccountID: accountID,
ItemType: "ec2-network-interface",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeNetworkInterfacesInput) (*ec2.DescribeNetworkInterfacesOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeNetworkInterfaces(ctx, input)
},
InputMapperGet: networkInterfaceInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/network_interface_permissions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func NewNetworkInterfacePermissionSource(config aws.Config, accountID string, li
AccountID: accountID,
ItemType: "ec2-network-interface-permission",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeNetworkInterfacePermissionsInput) (*ec2.DescribeNetworkInterfacePermissionsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeNetworkInterfacePermissions(ctx, input)
},
InputMapperGet: networkInterfacePermissionInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/placement_group.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func NewPlacementGroupSource(config aws.Config, accountID string, limit *sources
AccountID: accountID,
ItemType: "ec2-placement-group",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribePlacementGroupsInput) (*ec2.DescribePlacementGroupsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribePlacementGroups(ctx, input)
},
InputMapperGet: placementGroupInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/region.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func NewRegionSource(config aws.Config, accountID string, limit *sources.LimitBu
AccountID: accountID,
ItemType: "ec2-region",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeRegions(ctx, input)
},
InputMapperGet: regionInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/reserved_instance.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func NewReservedInstanceSource(config aws.Config, accountID string, limit *sourc
AccountID: accountID,
ItemType: "ec2-reserved-instance",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeReservedInstancesInput) (*ec2.DescribeReservedInstancesOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeReservedInstances(ctx, input)
},
InputMapperGet: reservedInstanceInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/route_table.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ func NewRouteTableSource(config aws.Config, accountID string, limit *sources.Lim
AccountID: accountID,
ItemType: "ec2-route-table",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeRouteTablesInput) (*ec2.DescribeRouteTablesOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeRouteTables(ctx, input)
},
InputMapperGet: routeTableInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/sg.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func NewSecurityGroupSource(config aws.Config, accountID string, limit *sources.
AccountID: accountID,
ItemType: "ec2-security-group",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeSecurityGroupsInput) (*ec2.DescribeSecurityGroupsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeSecurityGroups(ctx, input)
},
InputMapperGet: securityGroupInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/sg_rule.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func NewSecurityGroupRuleSource(config aws.Config, accountID string, limit *sour
AccountID: accountID,
ItemType: "ec2-security-group-rule",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeSecurityGroupRulesInput) (*ec2.DescribeSecurityGroupRulesOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeSecurityGroupRules(ctx, input)
},
InputMapperGet: securityGroupRuleInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/snapshot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ func NewSnapshotSource(config aws.Config, accountID string, limit *sources.Limit
AccountID: accountID,
ItemType: "ec2-snapshot",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeSnapshotsInput) (*ec2.DescribeSnapshotsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeSnapshots(ctx, input)
},
InputMapperGet: snapshotInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/subnet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func NewSubnetSource(config aws.Config, accountID string, limit *sources.LimitBu
AccountID: accountID,
ItemType: "ec2-subnet",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeSubnetsInput) (*ec2.DescribeSubnetsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeSubnets(ctx, input)
},
InputMapperGet: subnetInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/volume.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ func NewVolumeSource(config aws.Config, accountID string, limit *sources.LimitBu
AccountID: accountID,
ItemType: "ec2-volume",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeVolumesInput) (*ec2.DescribeVolumesOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeVolumes(ctx, input)
},
InputMapperGet: volumeInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/volume_status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func NewVolumeStatusSource(config aws.Config, accountID string, limit *sources.L
AccountID: accountID,
ItemType: "ec2-volume-status",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeVolumeStatusInput) (*ec2.DescribeVolumeStatusOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeVolumeStatus(ctx, input)
},
InputMapperGet: volumeStatusInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/vpc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func NewVpcSource(config aws.Config, accountID string, limit *sources.LimitBucke
AccountID: accountID,
ItemType: "ec2-vpc",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeVpcsInput) (*ec2.DescribeVpcsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeVpcs(ctx, input)
},
InputMapperGet: vpcInputMapperGet,
Expand Down
2 changes: 1 addition & 1 deletion sources/ec2/vpc_peering_connection.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ func NewVpcPeeringConnectionSource(config aws.Config, accountID string, limit *s
AccountID: accountID,
ItemType: "ec2-vpc-peering-connection",
DescribeFunc: func(ctx context.Context, client *ec2.Client, input *ec2.DescribeVpcPeeringConnectionsInput) (*ec2.DescribeVpcPeeringConnectionsOutput, error) {
<-limit.C // Wait for late limiting
limit.Wait(ctx) // Wait for rate limiting // Wait for late limiting
return client.DescribeVpcPeeringConnections(ctx, input)
},
InputMapperGet: func(scope, query string) (*ec2.DescribeVpcPeeringConnectionsInput, error) {
Expand Down
2 changes: 1 addition & 1 deletion sources/efs/access_point.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ func NewAccessPointSource(config aws.Config, accountID string, limit *sources.Li
AccountID: accountID,
DescribeFunc: func(ctx context.Context, client *efs.Client, input *efs.DescribeAccessPointsInput) (*efs.DescribeAccessPointsOutput, error) {
// Wait for rate limiting
<-limit.C
limit.Wait(ctx) // Wait for rate limiting
return client.DescribeAccessPoints(ctx, input)
},
PaginatorBuilder: func(client *efs.Client, params *efs.DescribeAccessPointsInput) sources.Paginator[*efs.DescribeAccessPointsOutput, *efs.Options] {
Expand Down
2 changes: 1 addition & 1 deletion sources/efs/backup_policy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func NewBackupPolicySource(config aws.Config, accountID string, limit *sources.L
AccountID: accountID,
DescribeFunc: func(ctx context.Context, client *efs.Client, input *efs.DescribeBackupPolicyInput) (*efs.DescribeBackupPolicyOutput, error) {
// Wait for rate limiting
<-limit.C
limit.Wait(ctx) // Wait for rate limiting
return client.DescribeBackupPolicy(ctx, input)
},
InputMapperGet: func(scope, query string) (*efs.DescribeBackupPolicyInput, error) {
Expand Down
2 changes: 1 addition & 1 deletion sources/efs/file_system.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ func NewFileSystemSource(config aws.Config, accountID string, limit *sources.Lim
AccountID: accountID,
DescribeFunc: func(ctx context.Context, client *efs.Client, input *efs.DescribeFileSystemsInput) (*efs.DescribeFileSystemsOutput, error) {
// Wait for rate limiting
<-limit.C
limit.Wait(ctx) // Wait for rate limiting
return client.DescribeFileSystems(ctx, input)
},
PaginatorBuilder: func(client *efs.Client, params *efs.DescribeFileSystemsInput) sources.Paginator[*efs.DescribeFileSystemsOutput, *efs.Options] {
Expand Down
2 changes: 1 addition & 1 deletion sources/efs/mount_target.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ func NewMountTargetSource(config aws.Config, accountID string, limit *sources.Li
AccountID: accountID,
DescribeFunc: func(ctx context.Context, client *efs.Client, input *efs.DescribeMountTargetsInput) (*efs.DescribeMountTargetsOutput, error) {
// Wait for rate limiting
<-limit.C
limit.Wait(ctx) // Wait for rate limiting
return client.DescribeMountTargets(ctx, input)
},
InputMapperGet: func(scope, query string) (*efs.DescribeMountTargetsInput, error) {
Expand Down
2 changes: 1 addition & 1 deletion sources/efs/replication_configuration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ func NewReplicationConfigurationSource(config aws.Config, accountID string, limi
AccountID: accountID,
DescribeFunc: func(ctx context.Context, client *efs.Client, input *efs.DescribeReplicationConfigurationsInput) (*efs.DescribeReplicationConfigurationsOutput, error) {
// Wait for rate limiting
<-limit.C
limit.Wait(ctx) // Wait for rate limiting
return client.DescribeReplicationConfigurations(ctx, input)
},
InputMapperGet: func(scope, query string) (*efs.DescribeReplicationConfigurationsInput, error) {
Expand Down
4 changes: 2 additions & 2 deletions sources/iam/group.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,11 @@ func NewGroupSource(config aws.Config, accountID string, region string, limit *s
CacheDuration: 1 * time.Hour, // IAM has very low rate limits, we need to cache for a long time
AccountID: accountID,
GetFunc: func(ctx context.Context, client *iam.Client, scope, query string) (*types.Group, error) {
<-limit.C
limit.Wait(ctx) // Wait for rate limiting
return groupGetFunc(ctx, client, scope, query)
},
ListFunc: func(ctx context.Context, client *iam.Client, scope string) ([]*types.Group, error) {
<-limit.C
limit.Wait(ctx) // Wait for rate limiting
return groupListFunc(ctx, client, scope)
},
ItemMapper: groupItemMapper,
Expand Down
Loading
Loading