From 7709c95b6b4b0c3e61307634f7dbda00499b91d7 Mon Sep 17 00:00:00 2001 From: Keep Focused Date: Tue, 17 Dec 2024 14:57:51 +0700 Subject: [PATCH] Optimized API call, query timing for the table aws_ec2_instance_type and added support to accept the wildcard pattern(`t2*`) for instance type. Closes #2292 (#2301) --- aws/table_aws_ec2_instance_type.go | 209 +++++++++++++++------------ docs/tables/aws_ec2_instance_type.md | 6 + 2 files changed, 122 insertions(+), 93 deletions(-) diff --git a/aws/table_aws_ec2_instance_type.go b/aws/table_aws_ec2_instance_type.go index 8d0665ddd..25d1d78e3 100644 --- a/aws/table_aws_ec2_instance_type.go +++ b/aws/table_aws_ec2_instance_type.go @@ -3,6 +3,8 @@ package aws import ( "context" "fmt" + "regexp" + "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" @@ -13,6 +15,7 @@ import ( "github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto" "github.com/turbot/steampipe-plugin-sdk/v5/plugin" "github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform" + "github.com/turbot/steampipe-plugin-sdk/v5/query_cache" ) //// TABLE DEFINITION @@ -21,29 +24,14 @@ func tableAwsInstanceType(_ context.Context) *plugin.Table { return &plugin.Table{ Name: "aws_ec2_instance_type", Description: "AWS EC2 Instance Type", - Get: &plugin.GetConfig{ - // We must have to include the region in the query parameter to make the gate API call. - // Otherwise we will get an Error: get call returned 9 results - the key column is not globally unique (SQLSTATE HV000) - KeyColumns: plugin.AllColumns([]string{"instance_type", "region"}), - IgnoreConfig: &plugin.IgnoreConfig{ - ShouldIgnoreErrorFunc: shouldIgnoreErrors([]string{"InvalidInstanceType"}), - }, - Hydrate: describeInstanceType, - Tags: map[string]string{"service": "ec2", "action": "DescribeInstanceTypes"}, - }, List: &plugin.ListConfig{ Hydrate: listAwsInstanceTypesOfferings, KeyColumns: plugin.KeyColumnSlice{ {Name: "instance_type", Require: plugin.Optional, Operators: []string{"="}}, + {Name: "instance_type_pattern", Require: plugin.Optional, Operators: []string{"="}, CacheMatch: query_cache.CacheMatchExact}, }, Tags: map[string]string{"service": "ec2", "action": "DescribeInstanceTypeOfferings"}, }, - HydrateConfig: []plugin.HydrateConfig{ - { - Func: describeInstanceType, - Tags: map[string]string{"service": "ec2", "action": "DescribeInstanceTypes"}, - }, - }, GetMatrixItemFunc: SupportedRegionMatrix(ec2v1.EndpointsID), Columns: awsRegionalColumns([]*plugin.Column{ { @@ -51,77 +39,74 @@ func tableAwsInstanceType(_ context.Context) *plugin.Table { Description: "The instance type. For more information, see [ Instance Types ](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html) in the Amazon Elastic Compute Cloud User Guide.", Type: proto.ColumnType_STRING, }, + // In the query "select * from aws_ec2_instance_type where instance_type = 't2*'", the API fetches the result but returns empty rows due to PostgreSQL-level filtering. + // The 'instance_type' column contains values like 't2.small', not 't2*', leading to a mismatch between the column value ('t2.small') and the wildcard pattern used in the query ('t2*'). + // To resolve this issue, the 'instance_type_pattern' column has been added, allowing for proper filtering using wildcard patterns. + { + Name: "instance_type_pattern", + Description: "The instance type pattern includes wildcards, such as 'c5-*', 't2*', and 'm5*'.", + Type: proto.ColumnType_STRING, + Transform: transform.FromQual("instance_type_pattern"), + }, { Name: "auto_recovery_supported", Description: "Indicates whether auto recovery is supported.", Type: proto.ColumnType_BOOL, - Hydrate: describeInstanceType, }, { Name: "bare_metal", Description: "Indicates whether the instance is a bare metal instance type.", Type: proto.ColumnType_BOOL, - Hydrate: describeInstanceType, }, { Name: "burstable_performance_supported", Description: "Indicates whether the instance type is a burstable performance instance type.", Type: proto.ColumnType_BOOL, - Hydrate: describeInstanceType, }, { Name: "current_generation", Description: "Indicates whether the instance type is current generation.", Type: proto.ColumnType_BOOL, - Hydrate: describeInstanceType, }, { Name: "dedicated_hosts_supported", Description: "Indicates whether Dedicated Hosts are supported on the instance type.", Type: proto.ColumnType_BOOL, - Hydrate: describeInstanceType, }, { Name: "free_tier_eligible", Description: "Indicates whether the instance type is eligible for the free tier.", Type: proto.ColumnType_BOOL, - Hydrate: describeInstanceType, }, { Name: "nitro_enclaves_support", Description: "Indicates whether Nitro Enclaves is supported.", Type: proto.ColumnType_STRING, - Hydrate: describeInstanceType, }, { Name: "nitro_tpm_support", Description: "Indicates whether NitroTPM is supported.", Type: proto.ColumnType_STRING, - Hydrate: describeInstanceType, }, { Name: "hibernation_supported", Description: "Indicates whether On-Demand hibernation is supported.", Type: proto.ColumnType_BOOL, - Hydrate: describeInstanceType, }, { Name: "hypervisor", Description: "The hypervisor for the instance type.", Type: proto.ColumnType_STRING, - Hydrate: describeInstanceType, }, { Name: "instance_storage_supported", Description: "Describes the instance storage for the instance type.", Type: proto.ColumnType_STRING, - Hydrate: describeInstanceType, }, { Name: "ebs_info", Description: "Describes the Amazon EBS settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "location_type", @@ -132,104 +117,87 @@ func tableAwsInstanceType(_ context.Context) *plugin.Table { Name: "memory_info", Description: "Describes the memory for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "network_info", Description: "Describes the network settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "placement_group_info", Description: "Describes the placement group settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "processor_info", Description: "Describes the processor.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "supported_root_device_types", Description: "The supported root device types.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "supported_usage_classes", Description: "Indicates whether the instance type is offered for spot or On-Demand.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "supported_virtualization_types", Description: "The supported virtualization types.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "v_cpu_info", Description: "Describes the vCPU configurations for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, Transform: transform.FromField("VCpuInfo"), }, { Name: "gpu_info", Description: "Describes the GPU accelerator settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "fpga_info", Description: "Describes the FPGA accelerator settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "inference_accelerator_info", Description: "Describes the Inference accelerator settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "instance_storage_info", Description: "Describes the instance storage for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "media_accelerator_info", Description: "Describes the media accelerator settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "neuron_info", Description: "Describes the Neuron accelerator settings for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "nitro_tpm_info", Description: "Describes the supported NitroTPM versions for the instance type.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "supported_boot_modes", Description: "The supported boot modes.", Type: proto.ColumnType_JSON, - Hydrate: describeInstanceType, }, { Name: "title", Description: resourceInterfaceDescription("title"), Type: proto.ColumnType_STRING, - Hydrate: describeInstanceType, Transform: transform.FromField("InstanceType"), }, { @@ -248,18 +216,19 @@ func tableAwsInstanceType(_ context.Context) *plugin.Table { func listAwsInstanceTypesOfferings(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { region := d.EqualsQualString(matrixKeyRegion) - // Create Session + // Create EC2 client svc, err := EC2Client(ctx, d) if err != nil { plugin.Logger(ctx).Error("aws_ec2_instance_type.listAwsInstanceTypesOfferings", "connection_error", err) return nil, err } + + // If the service is nil, the region is unsupported if svc == nil { - // Unsupported region, return no data return nil, nil } - // Limiting the results + // Set the maximum limit for results maxLimit := int32(1000) if d.QueryContext.Limit != nil { limit := int32(*d.QueryContext.Limit) @@ -272,99 +241,153 @@ func listAwsInstanceTypesOfferings(ctx context.Context, d *plugin.QueryData, h * } } - // First get all the types of instance + // Prepare the input for EC2 DescribeInstanceTypeOfferings API input := &ec2.DescribeInstanceTypeOfferingsInput{ LocationType: types.LocationTypeRegion, MaxResults: aws.Int32(maxLimit), + Filters: []types.Filter{{Name: aws.String("location"), Values: []string{region}}}, } - var filters []types.Filter - filters = append(filters, types.Filter{Name: aws.String("location"), Values: []string{region}}) + // Fetch instance type offering for a particular instance type if d.EqualsQualString("instance_type") != "" { - filters = append(filters, types.Filter{Name: aws.String("instance-type"), Values: []string{d.EqualsQualString("instance_type")}}) + input.Filters = append(input.Filters, types.Filter{Name: aws.String("instance-type"), Values: []string{d.EqualsQualString("instance_type")}}) } - input.Filters = filters + // Fetch instance types offerings + instanceTypes, err := fetchInstanceTypeOfferings(ctx, d, svc, input, maxLimit) + if err != nil { + return nil, err + } + + // Apply pattern matching on instance types if provided in query + filteredInstanceTypes := filterInstanceTypesByPattern(ctx, instanceTypes, d.EqualsQualString("instance_type_pattern")) + + // Batch process the instance types in groups of 100 + err = batchDescribeInstanceTypes(ctx, d, filteredInstanceTypes, region) + if err != nil { + plugin.Logger(ctx).Error("aws_ec2_instance_type.listAwsInstanceTypesOfferings", "batch_process_error", err) + return nil, err + } + + return nil, nil +} + +// Helper function to fetch instance type offerings using pagination +func fetchInstanceTypeOfferings(ctx context.Context, d *plugin.QueryData, svc *ec2.Client, input *ec2.DescribeInstanceTypeOfferingsInput, maxLimit int32) ([]types.InstanceType, error) { + var instanceTypes []types.InstanceType paginator := ec2.NewDescribeInstanceTypeOfferingsPaginator(svc, input, func(o *ec2.DescribeInstanceTypeOfferingsPaginatorOptions) { o.Limit = maxLimit o.StopOnDuplicateToken = true }) - // List call for paginator.HasMorePages() { - // apply rate limiting d.WaitForListRateLimit(ctx) output, err := paginator.NextPage(ctx) if err != nil { - plugin.Logger(ctx).Error("aws_ec2_instance_type.listAwsInstanceTypesOfferings", "api_error", err) + plugin.Logger(ctx).Error("aws_ec2_instance_type.fetchInstanceTypeOfferings", "api_error", err) return nil, err } - for _, items := range output.InstanceTypeOfferings { - d.StreamListItem(ctx, items) + for _, item := range output.InstanceTypeOfferings { + instanceTypes = append(instanceTypes, item.InstanceType) + } + } + + return instanceTypes, nil +} - // Context can be cancelled due to manual cancellation or the limit has been hit - if d.RowsRemaining(ctx) == 0 { - return nil, nil - } +// Helper function to filter instance types by a pattern like t2-*, m5-*, etc. +func filterInstanceTypesByPattern(_ context.Context, instanceTypes []types.InstanceType, pattern string) []types.InstanceType { + if pattern == "" { + return instanceTypes + } + + // The regex pattern "t3*" does not work as expected when matching the string "t3.small". This is because '*' in regex matches zero or more occurrences of the preceding character, + // allowing it to match "t3.small" due to the presence of the '.' character. To correct this, we replace '*' with '.+' to match any characters after "t3" appropriately. + // The following patterns were validated: + // - "c7*" matches (e.g., c7i.2xlarge, c7gn.xlarge, c7i-flex.2xlarge), + // - "c7i*" matches (e.g., c7i.2xlarge, c7i-flex.2xlarge), + // - "c7i.*" matches (e.g., c7i.8xlarge, c7i.2xlarge), + // - "c7i-*" matches (e.g., c7i-flex.2xlarge). + pattern = strings.ReplaceAll(pattern, ".", "\\.") + pattern = strings.ReplaceAll(pattern, "*", ".+") + var matchedInstanceTypes []types.InstanceType + re := regexp.MustCompile(pattern) + + for _, instanceType := range instanceTypes { + if re.MatchString(string(instanceType)) { + matchedInstanceTypes = append(matchedInstanceTypes, instanceType) } } - return nil, err + return matchedInstanceTypes } -//// HYDRATE FUNCTIONS +// Helper function to batch describe instance types in groups of 100 +func batchDescribeInstanceTypes(ctx context.Context, d *plugin.QueryData, instanceTypes []types.InstanceType, region string) error { + batchSize := 100 + if d.QueryContext.Limit != nil && *d.QueryContext.Limit > 0 && *d.QueryContext.Limit < 100 { + batchSize = int(*d.QueryContext.Limit) + } + for i := 0; i < len(instanceTypes); i += batchSize { + end := i + batchSize + if end > len(instanceTypes) { + end = len(instanceTypes) + } + batch := instanceTypes[i:end] -func describeInstanceType(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { - var instanceType types.InstanceType - if h.Item != nil { - data := h.Item.(types.InstanceTypeOffering) - instanceType = data.InstanceType - } else { - instanceType = types.InstanceType(d.EqualsQuals["instance_type"].GetStringValue()) + err := describeInstanceTypes(ctx, d, batch, region) + if err != nil { + return err + } } - // Create Session - svc, err := EC2Client(ctx, d) + return nil +} + +// Describe instance types and stream the results +func describeInstanceTypes(ctx context.Context, d *plugin.QueryData, instanceTypes []types.InstanceType, region string) error { + svc, err := EC2ClientForRegion(ctx, d, region) if err != nil { - plugin.Logger(ctx).Error("aws_ec2_instance_type.describeInstanceType", "connection_error", err) - return nil, err + plugin.Logger(ctx).Error("aws_ec2_instance_type.describeInstanceTypes", "connection_error", err) + return err } if svc == nil { // Unsupported region, return no data - return nil, nil + return nil } - // First get all the types of + // Create input for DescribeInstanceTypes API params := &ec2.DescribeInstanceTypesInput{ - InstanceTypes: []types.InstanceType{ - instanceType, - }, + InstanceTypes: instanceTypes, } - // execute get call + // Fetch the instance types op, err := svc.DescribeInstanceTypes(ctx, params) if err != nil { - plugin.Logger(ctx).Error("aws_ec2_instance_type.describeInstanceType", "api_error", err) - return nil, err + plugin.Logger(ctx).Error("aws_ec2_instance_type.describeInstanceTypes", "api_error", err) + return err } - if len(op.InstanceTypes) > 0 { - return op.InstanceTypes[0], nil + + // Stream each item from the response + for _, item := range op.InstanceTypes { + d.StreamListItem(ctx, item) + + // Context may get cancelled due to manual cancellation or if the limit has been reached + if d.RowsRemaining(ctx) == 0 { + return nil + } } - return nil, nil + return nil } +//// HYDRATE FUNCTION + func instanceTypeDataToAkas(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { - var instanceType types.InstanceType - switch h.Item.(type) { - case types.InstanceTypeOffering: - instanceType = h.Item.(types.InstanceTypeOffering).InstanceType - case types.InstanceTypeInfo: - instanceType = h.Item.(types.InstanceTypeInfo).InstanceType - } + instanceType := h.Item.(types.InstanceTypeInfo).InstanceType commonData, err := getCommonColumns(ctx, d, h) if err != nil { diff --git a/docs/tables/aws_ec2_instance_type.md b/docs/tables/aws_ec2_instance_type.md index 33ca38b9c..9e65f6114 100644 --- a/docs/tables/aws_ec2_instance_type.md +++ b/docs/tables/aws_ec2_instance_type.md @@ -11,6 +11,12 @@ The AWS EC2 Instance Type is a component of Amazon's Elastic Compute Cloud (EC2) The `aws_ec2_instance_type` table in Steampipe provides you with information about EC2 instance types within AWS Elastic Compute Cloud (EC2). This table allows you, as a DevOps engineer, to query instance type-specific details, including its name, current generation, vCPU, memory, storage, and network performance. You can utilize this table to gather insights on instance types, such as their capabilities, performance, and associated metadata. The schema outlines the various attributes of the EC2 instance type for you, including the instance type, current generation, vCPU, memory, storage, and network performance. +**Important Notes** +- This table supports the optional quals `instance_type` and `instance_type_pattern`. +- Queries with optional quals are optimised to use additional filtering provided by the AWS API function. +- To filter by a specific `instance_type`, you need to include it in the WHERE clause, such as `where instance_type = 't2.small'`, to retrieve a single instance type. +- If you want to fetch instance types using a wildcard pattern, you can use `instance_type_pattern` in the WHERE clause, like `where instance_type_pattern = 't2*'`. + ## Examples ### List of instance types which supports dedicated host