From 5b9f00e60c1668bd39f8441f4eaea8ad3033d626 Mon Sep 17 00:00:00 2001 From: liustve Date: Mon, 2 Dec 2024 21:15:38 +0000 Subject: [PATCH 1/4] contract tests for feature parity --- .../images/applications/aws-sdk/package.json | 6 + .../images/applications/aws-sdk/server.js | 416 +++++++++++++++++- .../tests/test/amazon/aws-sdk/aws_sdk_test.py | 382 ++++++++++++++-- .../test/amazon/base/contract_test_base.py | 16 +- .../utils/application_signals_constants.py | 1 + 5 files changed, 781 insertions(+), 40 deletions(-) diff --git a/contract-tests/images/applications/aws-sdk/package.json b/contract-tests/images/applications/aws-sdk/package.json index cef37d8..8596d51 100644 --- a/contract-tests/images/applications/aws-sdk/package.json +++ b/contract-tests/images/applications/aws-sdk/package.json @@ -15,10 +15,16 @@ "@aws-sdk/client-bedrock-agent-runtime": "^3.676.0", "@aws-sdk/client-bedrock-runtime": "^3.675.0", "@aws-sdk/client-dynamodb": "^3.658.1", + "@aws-sdk/client-iam": "^3.696.0", "@aws-sdk/client-kinesis": "^3.658.1", + "@aws-sdk/client-lambda": "^3.698.0", "@aws-sdk/client-s3": "^3.658.1", + "@aws-sdk/client-secrets-manager": "^3.696.0", + "@aws-sdk/client-sfn": "^3.696.0", + "@aws-sdk/client-sns": "^3.696.0", "@aws-sdk/client-sqs": "^3.658.1", "@smithy/node-http-handler": "^3.2.3", + "jszip": "^3.10.1", "node-fetch": "^2.7.0" } } diff --git a/contract-tests/images/applications/aws-sdk/server.js b/contract-tests/images/applications/aws-sdk/server.js index 6fd65c7..7c61039 100644 --- a/contract-tests/images/applications/aws-sdk/server.js +++ b/contract-tests/images/applications/aws-sdk/server.js @@ -5,15 +5,22 @@ const fs = require('fs'); const os = require('os'); const ospath = require('path'); const { NodeHttpHandler } =require('@smithy/node-http-handler'); +const fetch = require('node-fetch'); +const JSZip = require('jszip'); const { S3Client, CreateBucketCommand, PutObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3'); const { DynamoDBClient, CreateTableCommand, PutItemCommand } = require('@aws-sdk/client-dynamodb'); const { SQSClient, CreateQueueCommand, SendMessageCommand, ReceiveMessageCommand } = require('@aws-sdk/client-sqs'); const { KinesisClient, CreateStreamCommand, PutRecordCommand } = require('@aws-sdk/client-kinesis'); const { BedrockClient, GetGuardrailCommand } = require('@aws-sdk/client-bedrock'); -const { BedrockAgentClient, GetKnowledgeBaseCommand, GetDataSourceCommand, GetAgentCommand } = require('@aws-sdk/client-bedrock-agent'); +const { BedrockAgentClient, GetKnowledgeBaseCommand, GetDataSourceCommand, GetAgentCommand, AssociateAgentKnowledgeBaseCommand } = require('@aws-sdk/client-bedrock-agent'); const { BedrockRuntimeClient, InvokeModelCommand } = require('@aws-sdk/client-bedrock-runtime'); const { BedrockAgentRuntimeClient, InvokeAgentCommand, RetrieveCommand } = require('@aws-sdk/client-bedrock-agent-runtime'); +const { SNSClient, CreateTopicCommand, GetTopicAttributesCommand } = require('@aws-sdk/client-sns'); +const { SecretsManagerClient, CreateSecretCommand, DescribeSecretCommand } = require('@aws-sdk/client-secrets-manager'); +const { SFNClient, CreateStateMachineCommand, CreateActivityCommand, DescribeStateMachineCommand, DescribeActivityCommand } = require('@aws-sdk/client-sfn'); +const { IAMClient, AttachRolePolicyCommand, CreateRoleCommand } = require('@aws-sdk/client-iam') +const { LambdaClient, CreateFunctionCommand, GetEventSourceMappingCommand, CreateEventSourceMappingCommand, UpdateEventSourceMappingCommand } = require('@aws-sdk/client-lambda'); const _PORT = 8080; @@ -25,6 +32,7 @@ const _AWS_SDK_ENDPOINT = process.env.AWS_SDK_ENDPOINT; const _AWS_REGION = process.env.AWS_REGION; const _FAULT_ENDPOINT = 'http://fault.test:8080'; + process.env.AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID || 'testcontainers-localstack'; process.env.AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY || 'testcontainers-localstack'; @@ -63,6 +71,16 @@ async function prepareAwsServer() { region: _AWS_REGION, }); + const secretsClient = new SecretsManagerClient({ + endpoint: _AWS_SDK_ENDPOINT, + region: _AWS_REGION, + }) + + const snsClient = new SNSClient({ + endpoint: _AWS_SDK_ENDPOINT, + region: _AWS_REGION, + }) + // Set up S3 await s3Client.send( new CreateBucketCommand({ @@ -102,7 +120,7 @@ async function prepareAwsServer() { ); // Set up SQS - await sqsClient.send( + const sqsQueue = await sqsClient.send( new CreateQueueCommand({ QueueName: 'test_put_get_queue', }) @@ -113,11 +131,32 @@ async function prepareAwsServer() { new CreateStreamCommand({ StreamName: 'test_stream', ShardCount: 1, + })) + + // Set up SecretsManager + await secretsClient.send( + new CreateSecretCommand({ + "Description": "My test secret", + "Name": "MyTestSecret", + "SecretString": "{\"username\":\"user\",\"password\":\"password\"}" }) ); + + // Set up SNS + await snsClient.send(new CreateTopicCommand({ + "Name": "TestTopic" + })) + + // Set up Lambda + await setupLambda() + + // Set up StepFunctions + await setupSfn() + } catch (error) { console.error('Unexpected exception occurred', error); } + } const server = http.createServer(async (req, res) => { @@ -134,7 +173,7 @@ const server = http.createServer(async (req, res) => { res.writeHead(405); res.end(); } -}); +}) async function handleGetRequest(req, res, path) { if (path.includes('s3')) { @@ -147,6 +186,14 @@ async function handleGetRequest(req, res, path) { await handleKinesisRequest(req, res, path); } else if (path.includes('bedrock')) { await handleBedrockRequest(req, res, path); + } else if (path.includes('secretsmanager')) { + await handleSecretsRequest(req, res, path); + } else if (path.includes('stepfunctions')) { + await handleSfnRequest(req, res, path); + } else if (path.includes('sns')) { + await handleSnsRequest(req, res, path); + } else if (path.includes('lambda')) { + await handleLambdaRequest(req, res, path); } else { res.writeHead(404); res.end(); @@ -517,7 +564,9 @@ async function handleBedrockRequest(req, res, path) { await withInjected200Success( bedrockClient, ['GetGuardrailCommand'], - { guardrailId: 'bt4o77i015cu' }, + { guardrailId: 'bt4o77i015cu', + guardrailArn: 'arn:aws:bedrock:us-east-1:000000000000:guardrail/bt4o77i015cu' + }, async () => { await bedrockClient.send( new GetGuardrailCommand({ @@ -527,6 +576,17 @@ async function handleBedrockRequest(req, res, path) { } ); res.statusCode = 200; + } else if (path.includes('associateagent/associate_agent_knowledge_base')) { + await withInjected200Success(bedrockAgentRuntimeClient, ['AssociateAgentKnowledgeBaseCommand'], {}, async () => { + await bedrockAgentRuntimeClient.send( + new AssociateAgentKnowledgeBaseCommand({ + agentId: 'Q08WFRPHVL', + agentVersion: '1', + knowledgeBaseId: 'test-knowledge-base-id', + }) + ); + }); + res.statusCode = 200; } else if (path.includes('invokeagent/invoke_agent')) { await withInjected200Success(bedrockAgentRuntimeClient, ['InvokeAgentCommand'], {}, async () => { await bedrockAgentRuntimeClient.send( @@ -748,6 +808,349 @@ async function handleBedrockRequest(req, res, path) { res.end(); } + +async function handleSecretsRequest(req, res, path) { + const secretsClient = new SecretsManagerClient({ + endpoint: _AWS_SDK_ENDPOINT, + region: _AWS_REGION, + }); + + if (path.includes(_ERROR)) { + res.statusCode = 400; + + try { + await secretsClient.send( + new DescribeSecretCommand({ + SecretId: "arn:aws:secretsmanager:us-west-2:000000000000:secret:nonExistentSecret" + }) + ); + } catch (err) { + console.log('Expected exception occurred', err); + } + } + + if (path.includes(_FAULT)) { + res.statusCode = 500; + statusCodeForFault = 500; + + try { + const faultSecretsClient = new SecretsManagerClient({ + endpoint: _FAULT_ENDPOINT, + region: _AWS_REGION, + ...noRetryConfig, + }); + + await faultSecretsClient.send( + new DescribeSecretCommand({ + SecretId: "arn:aws:secretsmanager:us-west-2:000000000000:secret:nonExistentSecret" + }) + ); + } catch (err) { + console.log('Expected exception occurred', err); + } + } + + if (path.includes('describesecret/my-secret')) { + await secretsClient.send( + new DescribeSecretCommand({ + SecretId: "MyTestSecret" + }) + ); + } + + res.end(); +} + +async function handleSfnRequest(req, res, path) { + const sfnClient = new SFNClient({ + endpoint: _AWS_SDK_ENDPOINT, + region: _AWS_REGION, + }); + + if (path.includes(_ERROR)) { + res.statusCode = 400; + + try { + await sfnClient.send( + new DescribeStateMachineCommand({ + stateMachineArn: "arn:aws:states:us-west-2:000000000000:stateMachine:nonExistentStateMachine" + }) + ); + } catch (err) { + console.log('Expected exception occurred', err); + } + } + + if (path.includes(_FAULT)) { + res.statusCode = 500; + statusCodeForFault = 500; + + try { + + const faultSfnClient = new SFNClient({ + endpoint: _FAULT_ENDPOINT, + region: _AWS_REGION, + ...noRetryConfig, + }); + + await faultSfnClient.send( + new DescribeStateMachineCommand({ + stateMachineArn: "arn:aws:states:us-west-2:000000000000:stateMachine:invalid-state-machine" + }) + ); + } catch (err) { + console.log('Expected exception occurred', err); + } + } + + if (path.includes('describestatemachine/state-machine')) { + await sfnClient.send( + new DescribeStateMachineCommand({ + stateMachineArn: "arn:aws:states:us-west-2:000000000000:stateMachine:TestStateMachine" + }) + ); + } + + if (path.includes('describeactivity/activity')) { + await sfnClient.send( + new DescribeActivityCommand({ + activityArn: "arn:aws:states:us-west-2:000000000000:activity:TestActivity" + }) + ); + } + + res.end(); +} + +async function handleSnsRequest(req, res, path) { + const snsClient = new SNSClient({ + endpoint: _AWS_SDK_ENDPOINT, + region: _AWS_REGION, + }); + + if (path.includes(_ERROR)) { + res.statusCode = 404; + + try { + await snsClient.send( + new GetTopicAttributesCommand({ + TopicArn: "arn:aws:sns:us-west-2:000000000000:nonExistentTopic", + }) + ); + } catch (err) { + console.log('Expected exception occurred', err); + } + + } + + if (path.includes(_FAULT)) { + res.statusCode = 500; + statusCodeForFault = 500; + + try { + const faultSnsClient = new SNSClient({ + endpoint: _FAULT_ENDPOINT, + region: _AWS_REGION, + ...noRetryConfig, + }); + + await faultSnsClient.send( + new GetTopicAttributesCommand({ + TopicArn: "arn:aws:sns:us-west-2:000000000000:invalidTopic" + }) + ); + } catch (err) { + console.log('Expected exception occurred', err); + } + } + + if (path.includes('gettopicattributes/topic')) { + await snsClient.send( + new GetTopicAttributesCommand({ + TopicArn: "arn:aws:sns:us-west-2:000000000000:TestTopic" + }) + ); + } + + res.end(); +} + +async function handleLambdaRequest(req, res, path) { + const lambdaClient = new LambdaClient({ + endpoint: _AWS_SDK_ENDPOINT, + region: _AWS_REGION, + }); + + if (path.includes(_ERROR)) { + res.statusCode = 404; + + try { + await lambdaClient.send( + new GetEventSourceMappingCommand({ + UUID: "nonExistentUUID" + }) + ); + } + catch(err) { + console.log('Expected exception occurred', err); + } + + } + + if (path.includes(_FAULT)) { + res.statusCode = 500; + statusCodeForFault = 500; + + try { + const faultLambdaClient = new LambdaClient({ + endpoint: _FAULT_ENDPOINT, + region: _AWS_REGION, + ...noRetryConfig, + }); + + await faultLambdaClient.send( + new UpdateEventSourceMappingCommand({ + UUID: "123e4567-e89b-12d3-a456-426614174000" + }) + ); + } + catch(err) { + console.log('Expected exception occurred', err); + } + } + + + if (path.includes('geteventsourcemapping')) { + await lambdaClient.send( + new GetEventSourceMappingCommand({ + UUID: '' + }) + ); + } + res.end(); +} + +async function setupLambda() { + const lambdaClient = new LambdaClient({ + endpoint: _AWS_SDK_ENDPOINT, + region: _AWS_REGION, + }); + + const iamClient = new IAMClient({ + endpoint: _AWS_SDK_ENDPOINT, + region: _AWS_REGION, + }); + + const trustPolicy = { + Version: "2012-10-17", + Statement: [{ + Effect: "Allow", + Principal: { + Service: "lambda.amazonaws.com" + }, + Action: "sts:AssumeRole" + }] + }; + + const functionName = 'testFunction' + + const lambdaRoleParams = { + RoleName: "LambdaRole", + AssumeRolePolicyDocument: JSON.stringify(trustPolicy), + }; + + const policyParams = { + RoleName: "LambdaRole", + PolicyArn: "arn:aws:iam::aws:policy/AWSLambda_FullAccess" + }; + + const role = await iamClient.send(new CreateRoleCommand(lambdaRoleParams)); + await iamClient.send(new AttachRolePolicyCommand(policyParams)); + + const zip = new JSZip(); + zip.file('index.js', 'exports.handler = async (event) => { return { statusCode: 200 }; };'); + const zipBuffer = await zip.generateAsync({ type: 'nodebuffer' }); + + const functionParams = { + Code: { + ZipFile: zipBuffer + }, + FunctionName: functionName, + Handler: "index.handler", + Role: role.Role.Arn, + Runtime: "nodejs18.x" + }; + + const mappingParams = { + EventSourceArn: "arn:aws:sns:us-west-2:000000000000:TestTopic", + FunctionName: functionName, + Enabled: false + } + + await lambdaClient.send(new CreateFunctionCommand(functionParams)); + await lambdaClient.send(new CreateEventSourceMappingCommand(mappingParams)); +} + +async function setupSfn() { + const sfnClient = new SFNClient({ + endpoint: _AWS_SDK_ENDPOINT, + region: _AWS_REGION, + }); + + const iamClient = new IAMClient({ + endpoint: _AWS_SDK_ENDPOINT, + region: _AWS_REGION, + }); + + const trustPolicy = { + Version: "2012-10-17", + Statement: [{ + Effect: "Allow", + Principal: { + Service: "states.amazonaws.com" + }, + Action: "sts:AssumeRole" + }] +}; + +const roleName = 'testRole' + +const createRoleResponse = await iamClient.send(new CreateRoleCommand({ + RoleName: roleName, + AssumeRolePolicyDocument: JSON.stringify(trustPolicy), +})); + +await iamClient.send(new AttachRolePolicyCommand({ + RoleName: roleName, + PolicyArn: 'arn:aws:iam::aws:policy/AWSStepFunctionsFullAccess' +})); + +const roleArn = createRoleResponse.Role.Arn + +const definition = { + StartAt: "HelloWorld", + States: { + "HelloWorld": { + Type: "Pass", + Result: "Hello, World!", + End: true + } + } +}; + +await sfnClient.send(new CreateStateMachineCommand({ + name: 'TestStateMachine', + definition: JSON.stringify(definition), + roleArn: roleArn, + type: 'STANDARD' +})); + +await sfnClient.send( + new CreateActivityCommand({ + name: 'TestActivity', + })); +} + function inject200Success(client, commandNames, additionalResponse = {}, middlewareName = 'inject200SuccessMiddleware') { const middleware = (next, context) => async (args) => { const { commandName } = context; @@ -775,12 +1178,9 @@ async function withInjected200Success(client, commandNames, additionalResponse, client.middlewareStack.remove(middlewareName); } - - prepareAwsServer().then(() => { server.listen(_PORT, '0.0.0.0', () => { console.log('Server is listening on port', _PORT); console.log('Ready'); }); -}); - +}); \ No newline at end of file diff --git a/contract-tests/tests/test/amazon/aws-sdk/aws_sdk_test.py b/contract-tests/tests/test/amazon/aws-sdk/aws_sdk_test.py index 31f72d2..b67bed6 100644 --- a/contract-tests/tests/test/amazon/aws-sdk/aws_sdk_test.py +++ b/contract-tests/tests/test/amazon/aws-sdk/aws_sdk_test.py @@ -18,6 +18,7 @@ AWS_REMOTE_RESOURCE_TYPE, AWS_REMOTE_SERVICE, AWS_SPAN_KIND, + AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER ) from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue from opentelemetry.proto.metrics.v1.metrics_pb2 import ExponentialHistogramDataPoint, Metric @@ -32,8 +33,14 @@ _AWS_KINESIS_STREAM_NAME: str = "aws.kinesis.stream.name" _AWS_BEDROCK_AGENT_ID: str = "aws.bedrock.agent.id" _AWS_BEDROCK_GUARDRAIL_ID: str = "aws.bedrock.guardrail.id" +_AWS_BEDROCK_GUARDRAIL_ARN: str = "aws.bedrock.guardrail.arn" _AWS_BEDROCK_KNOWLEDGE_BASE_ID: str = "aws.bedrock.knowledge_base.id" _AWS_BEDROCK_DATA_SOURCE_ID: str = "aws.bedrock.data_source.id" +_AWS_SECRET_ARN: str = "aws.secretsmanager.secret.arn" +_AWS_SNS_TOPIC_ARN: str = 'aws.sns.topic.arn' +_AWS_LAMBDA_RESOURCE_MAPPING_ID: str = 'aws.lambda.resource_mapping.id' +_AWS_STATE_MACHINE_ARN: str = "aws.stepfunctions.state_machine.arn" +_AWS_ACTIVITY_ARN: str = "aws.stepfunctions.activity.arn" _GEN_AI_REQUEST_MODEL: str = "gen_ai.request.model" _GEN_AI_REQUEST_TEMPERATURE: str = "gen_ai.request.temperature" _GEN_AI_REQUEST_TOP_P: str = "gen_ai.request.top_p" @@ -77,8 +84,9 @@ def set_up_dependency_container(cls): cls._local_stack: LocalStackContainer = ( LocalStackContainer(image="localstack/localstack:3.5.0") .with_name("localstack") - .with_services("s3", "sqs", "dynamodb", "kinesis") + .with_services("s3", "sqs", "dynamodb", "kinesis", 'secretsmanager', 'stepfunctions', 'iam', 'sns', "lambda") .with_env("DEFAULT_REGION", "us-west-2") + .with_volume_mapping("/var/run/docker.sock", "/var/run/docker.sock") .with_kwargs(network=NETWORK_NAME, networking_config=local_stack_networking_config) ) cls._local_stack.start() @@ -104,6 +112,7 @@ def test_s3_create_bucket(self): remote_operation="CreateBucket", remote_resource_type="AWS::S3::Bucket", remote_resource_identifier="test-bucket-name", + cloudformation_primary_identifier="test-bucket-name", request_specific_attributes={ SpanAttributes.AWS_S3_BUCKET: "test-bucket-name", }, @@ -122,6 +131,7 @@ def test_s3_create_object(self): remote_operation="PutObject", remote_resource_type="AWS::S3::Bucket", remote_resource_identifier="test-put-object-bucket-name", + cloudformation_primary_identifier="test-put-object-bucket-name", request_specific_attributes={ SpanAttributes.AWS_S3_BUCKET: "test-put-object-bucket-name", }, @@ -140,6 +150,7 @@ def test_s3_get_object(self): remote_operation="GetObject", remote_resource_type="AWS::S3::Bucket", remote_resource_identifier="test-get-object-bucket-name", + cloudformation_primary_identifier="test-get-object-bucket-name", request_specific_attributes={ SpanAttributes.AWS_S3_BUCKET: "test-get-object-bucket-name", }, @@ -158,6 +169,7 @@ def test_s3_error(self): remote_operation="CreateBucket", remote_resource_type="AWS::S3::Bucket", remote_resource_identifier="-", + cloudformation_primary_identifier="-", request_specific_attributes={ SpanAttributes.AWS_S3_BUCKET: "-", }, @@ -178,6 +190,7 @@ def test_s3_fault(self): remote_operation="CreateBucket", remote_resource_type="AWS::S3::Bucket", remote_resource_identifier="valid-bucket-name", + cloudformation_primary_identifier="valid-bucket-name", request_specific_attributes={ SpanAttributes.AWS_S3_BUCKET: "valid-bucket-name", }, @@ -196,6 +209,7 @@ def test_dynamodb_create_table(self): remote_operation="CreateTable", remote_resource_type="AWS::DynamoDB::Table", remote_resource_identifier="test_table", + cloudformation_primary_identifier="test_table", request_specific_attributes={ SpanAttributes.AWS_DYNAMODB_TABLE_NAMES: ["test_table"], }, @@ -214,6 +228,7 @@ def test_dynamodb_put_item(self): remote_operation="PutItem", remote_resource_type="AWS::DynamoDB::Table", remote_resource_identifier="put_test_table", + cloudformation_primary_identifier="put_test_table", request_specific_attributes={ SpanAttributes.AWS_DYNAMODB_TABLE_NAMES: ["put_test_table"], }, @@ -232,6 +247,7 @@ def test_dynamodb_error(self): remote_operation="PutItem", remote_resource_type="AWS::DynamoDB::Table", remote_resource_identifier="invalid_table", + cloudformation_primary_identifier="invalid_table", request_specific_attributes={ SpanAttributes.AWS_DYNAMODB_TABLE_NAMES: ["invalid_table"], }, @@ -252,6 +268,7 @@ def test_dynamodb_fault(self): remote_operation="PutItem", remote_resource_type="AWS::DynamoDB::Table", remote_resource_identifier="invalid_table", + cloudformation_primary_identifier="invalid_table", request_specific_attributes={ SpanAttributes.AWS_DYNAMODB_TABLE_NAMES: ["invalid_table"], }, @@ -270,6 +287,7 @@ def test_sqs_create_queue(self): remote_operation="CreateQueue", remote_resource_type="AWS::SQS::Queue", remote_resource_identifier="test_queue", + cloudformation_primary_identifier="test_queue", request_specific_attributes={ _AWS_SQS_QUEUE_NAME: "test_queue", }, @@ -289,6 +307,7 @@ def test_sqs_send_message(self): remote_operation="SendMessage", remote_resource_type="AWS::SQS::Queue", remote_resource_identifier="test_put_get_queue", + cloudformation_primary_identifier="http://localstack:4566/000000000000/test_put_get_queue", request_specific_attributes={ _AWS_SQS_QUEUE_URL: "http://localstack:4566/000000000000/test_put_get_queue", }, @@ -309,6 +328,7 @@ def test_sqs_receive_message(self): remote_operation="ReceiveMessage", remote_resource_type="AWS::SQS::Queue", remote_resource_identifier="test_put_get_queue", + cloudformation_primary_identifier="http://localstack:4566/000000000000/test_put_get_queue", request_specific_attributes={ _AWS_SQS_QUEUE_URL: "http://localstack:4566/000000000000/test_put_get_queue", }, @@ -329,6 +349,7 @@ def test_sqs_error(self): remote_operation="SendMessage", remote_resource_type="AWS::SQS::Queue", remote_resource_identifier="sqserror", + cloudformation_primary_identifier="http://error.test:8080/000000000000/sqserror", request_specific_attributes={ _AWS_SQS_QUEUE_URL: "http://error.test:8080/000000000000/sqserror", }, @@ -350,6 +371,7 @@ def test_sqs_fault(self): remote_operation="CreateQueue", remote_resource_type="AWS::SQS::Queue", remote_resource_identifier="invalid_test", + cloudformation_primary_identifier="invalid_test", request_specific_attributes={ _AWS_SQS_QUEUE_NAME: "invalid_test", }, @@ -368,6 +390,7 @@ def test_kinesis_put_record(self): remote_operation="PutRecord", remote_resource_type="AWS::Kinesis::Stream", remote_resource_identifier="test_stream", + cloudformation_primary_identifier="test_stream", request_specific_attributes={ _AWS_KINESIS_STREAM_NAME: "test_stream", }, @@ -386,6 +409,7 @@ def test_kinesis_error(self): remote_operation="PutRecord", remote_resource_type="AWS::Kinesis::Stream", remote_resource_identifier="invalid_stream", + cloudformation_primary_identifier="invalid_stream", request_specific_attributes={ _AWS_KINESIS_STREAM_NAME: "invalid_stream", }, @@ -406,6 +430,7 @@ def test_kinesis_fault(self): remote_operation="PutRecord", remote_resource_type="AWS::Kinesis::Stream", remote_resource_identifier="test_stream", + cloudformation_primary_identifier="test_stream", request_specific_attributes={ _AWS_KINESIS_STREAM_NAME: "test_stream", }, @@ -425,6 +450,7 @@ def test_bedrock_runtime_invoke_model_amazon_titan(self): remote_operation="InvokeModel", remote_resource_type="AWS::Bedrock::Model", remote_resource_identifier='amazon.titan-text-premier-v1:0', + cloudformation_primary_identifier="amazon.titan-text-premier-v1:0", request_specific_attributes={ _GEN_AI_REQUEST_MODEL: 'amazon.titan-text-premier-v1:0', _GEN_AI_REQUEST_MAX_TOKENS: 3072, @@ -453,6 +479,7 @@ def test_bedrock_runtime_invoke_model_anthropic_claude(self): remote_operation="InvokeModel", remote_resource_type="AWS::Bedrock::Model", remote_resource_identifier='anthropic.claude-v2:1', + cloudformation_primary_identifier="anthropic.claude-v2:1", request_specific_attributes={ _GEN_AI_REQUEST_MODEL: 'anthropic.claude-v2:1', _GEN_AI_REQUEST_MAX_TOKENS: 1000, @@ -480,6 +507,7 @@ def test_bedrock_runtime_invoke_model_meta_llama(self): remote_operation="InvokeModel", remote_resource_type="AWS::Bedrock::Model", remote_resource_identifier='meta.llama2-13b-chat-v1', + cloudformation_primary_identifier="meta.llama2-13b-chat-v1", request_specific_attributes={ _GEN_AI_REQUEST_MODEL: 'meta.llama2-13b-chat-v1', _GEN_AI_REQUEST_MAX_TOKENS: 512, @@ -507,6 +535,7 @@ def test_bedrock_runtime_invoke_model_cohere_command_r(self): remote_operation="InvokeModel", remote_resource_type="AWS::Bedrock::Model", remote_resource_identifier='cohere.command-r-v1:0', + cloudformation_primary_identifier="cohere.command-r-v1:0", request_specific_attributes={ _GEN_AI_REQUEST_MODEL: 'cohere.command-r-v1:0', _GEN_AI_REQUEST_MAX_TOKENS: 512, @@ -535,6 +564,7 @@ def test_bedrock_runtime_invoke_model_cohere_command(self): remote_operation="InvokeModel", remote_resource_type="AWS::Bedrock::Model", remote_resource_identifier='cohere.command-light-text-v14', + cloudformation_primary_identifier="cohere.command-light-text-v14", request_specific_attributes={ _GEN_AI_REQUEST_MODEL: 'cohere.command-light-text-v14', _GEN_AI_REQUEST_MAX_TOKENS: 512, @@ -562,6 +592,7 @@ def test_bedrock_runtime_invoke_model_ai21_jamba(self): remote_operation="InvokeModel", remote_resource_type="AWS::Bedrock::Model", remote_resource_identifier='ai21.jamba-1-5-large-v1:0', + cloudformation_primary_identifier="ai21.jamba-1-5-large-v1:0", request_specific_attributes={ _GEN_AI_REQUEST_MODEL: 'ai21.jamba-1-5-large-v1:0', _GEN_AI_REQUEST_MAX_TOKENS: 512, @@ -589,6 +620,7 @@ def test_bedrock_runtime_invoke_model_mistral_mistral(self): remote_operation="InvokeModel", remote_resource_type="AWS::Bedrock::Model", remote_resource_identifier='mistral.mistral-7b-instruct-v0:2', + cloudformation_primary_identifier="mistral.mistral-7b-instruct-v0:2", request_specific_attributes={ _GEN_AI_REQUEST_MODEL: 'mistral.mistral-7b-instruct-v0:2', _GEN_AI_REQUEST_MAX_TOKENS: 4096, @@ -616,8 +648,10 @@ def test_bedrock_get_guardrail(self): remote_operation="GetGuardrail", remote_resource_type="AWS::Bedrock::Guardrail", remote_resource_identifier="bt4o77i015cu", + cloudformation_primary_identifier="bt4o77i015cu", request_specific_attributes={ _AWS_BEDROCK_GUARDRAIL_ID: "bt4o77i015cu", + _AWS_BEDROCK_GUARDRAIL_ARN: "arn:aws:bedrock:us-east-1:000000000000:guardrail/bt4o77i015cu" }, span_name="Bedrock.GetGuardrail", ) @@ -635,6 +669,7 @@ def test_bedrock_agent_runtime_invoke_agent(self): remote_operation="InvokeAgent", remote_resource_type="AWS::Bedrock::Agent", remote_resource_identifier="Q08WFRPHVL", + cloudformation_primary_identifier="Q08WFRPHVL", request_specific_attributes={ _AWS_BEDROCK_AGENT_ID: "Q08WFRPHVL", }, @@ -654,12 +689,34 @@ def test_bedrock_agent_runtime_retrieve(self): remote_operation="Retrieve", remote_resource_type="AWS::Bedrock::KnowledgeBase", remote_resource_identifier="test-knowledge-base-id", + cloudformation_primary_identifier="test-knowledge-base-id", request_specific_attributes={ _AWS_BEDROCK_KNOWLEDGE_BASE_ID: "test-knowledge-base-id", }, span_name="BedrockAgentRuntime.Retrieve", ) + def test_bedrock_agent_associate_agent_knowledge_base(self): + self.do_test_requests( + "bedrock/associateagent/associate_agent_knowledge_base", + "GET", + 200, + 0, + 0, + local_operation="GET /bedrock", + rpc_service="BedrockAgentRuntime", + remote_service="AWS::Bedrock", + remote_operation="AssociateAgentKnowledgeBase", + remote_resource_type="AWS::Bedrock::Agent", + remote_resource_identifier="Q08WFRPHVL", + cloudformation_primary_identifier="test-knowledge-base-id|Q08WFRPHVL", + request_specific_attributes={ + _AWS_BEDROCK_AGENT_ID: "Q08WFRPHVL", + _AWS_BEDROCK_KNOWLEDGE_BASE_ID: "test-knowledge-base-id", + }, + span_name="BedrockAgentRuntime.AssociateAgentKnowledgeBase", + ) + def test_bedrock_agent_get_agent(self): self.do_test_requests( "bedrock/getagent/get-agent", @@ -673,6 +730,7 @@ def test_bedrock_agent_get_agent(self): remote_operation="GetAgent", remote_resource_type="AWS::Bedrock::Agent", remote_resource_identifier="TESTAGENTID", + cloudformation_primary_identifier="TESTAGENTID", request_specific_attributes={ _AWS_BEDROCK_AGENT_ID: "TESTAGENTID", }, @@ -692,6 +750,7 @@ def test_bedrock_agent_get_knowledge_base(self): remote_operation="GetKnowledgeBase", remote_resource_type="AWS::Bedrock::KnowledgeBase", remote_resource_identifier="invalid-knowledge-base-id", + cloudformation_primary_identifier="invalid-knowledge-base-id", request_specific_attributes={ _AWS_BEDROCK_KNOWLEDGE_BASE_ID: "invalid-knowledge-base-id", }, @@ -711,12 +770,263 @@ def test_bedrock_agent_get_data_source(self): remote_operation="GetDataSource", remote_resource_type="AWS::Bedrock::DataSource", remote_resource_identifier="DATASURCID", + cloudformation_primary_identifier="DATASURCID", request_specific_attributes={ _AWS_BEDROCK_DATA_SOURCE_ID: "DATASURCID", }, span_name="BedrockAgent.GetDataSource", ) + def test_secretsmanager_fault(self): + self.do_test_requests( + "secretsmanager/fault", + "GET", + 500, + 0, + 1, + dp_count=3, + local_operation="GET /secretsmanager", + local_operation_2="POST /", + rpc_service="SecretsManager", + remote_service="AWS::SecretsManager", + remote_operation="DescribeSecret", + remote_resource_type="AWS::SecretsManager::Secret", + remote_resource_identifier="nonExistentSecret", + cloudformation_primary_identifier="arn:aws:secretsmanager:us-west-2:000000000000:secret:nonExistentSecret", + request_specific_attributes= { + _AWS_SECRET_ARN: "arn:aws:secretsmanager:us-west-2:000000000000:secret:nonExistentSecret", + }, + span_name="SecretsManager.DescribeSecret", + ) + + def test_secretsmanager_error(self): + self.do_test_requests( + "secretsmanager/error", + "GET", + 400, + 1, + 0, + local_operation="GET /secretsmanager", + rpc_service="SecretsManager", + remote_service="AWS::SecretsManager", + remote_operation="DescribeSecret", + remote_resource_type="AWS::SecretsManager::Secret", + remote_resource_identifier="nonExistentSecret", + cloudformation_primary_identifier="arn:aws:secretsmanager:us-west-2:000000000000:secret:nonExistentSecret", + request_specific_attributes= { + _AWS_SECRET_ARN: "arn:aws:secretsmanager:us-west-2:000000000000:secret:nonExistentSecret", + }, + span_name="SecretsManager.DescribeSecret", + ) + + def test_secretsmanager_describe_secret(self): + self.do_test_requests( + "secretsmanager/describesecret/my-secret", + "GET", + 200, + 0, + 0, + local_operation="GET /secretsmanager", + rpc_service="SecretsManager", + remote_service="AWS::SecretsManager", + remote_operation="DescribeSecret", + remote_resource_type="AWS::SecretsManager::Secret", + remote_resource_identifier=r'MyTestSecret-[a-zA-Z0-9]{6}$', + cloudformation_primary_identifier=r"arn:aws:secretsmanager:us-west-2:000000000000:secret:MyTestSecret-[a-zA-Z0-9]{6}$", + response_specific_attributes= { + _AWS_SECRET_ARN: r"arn:aws:secretsmanager:us-west-2:000000000000:secret:MyTestSecret-[a-zA-Z0-9]{6}$", + }, + span_name="SecretsManager.DescribeSecret", + ) + + def test_stepfunctions_fault(self): + self.do_test_requests( + "stepfunctions/fault", + "GET", + 500, + 0, + 1, + dp_count=3, + local_operation="GET /stepfunctions", + local_operation_2="POST /", + rpc_service="SFN", + remote_service="AWS::StepFunctions", + remote_operation="DescribeStateMachine", + remote_resource_type="AWS::StepFunctions::StateMachine", + remote_resource_identifier="invalid-state-machine", + cloudformation_primary_identifier="arn:aws:states:us-west-2:000000000000:stateMachine:invalid-state-machine", + request_specific_attributes= { + _AWS_STATE_MACHINE_ARN: "arn:aws:states:us-west-2:000000000000:stateMachine:invalid-state-machine", + }, + span_name="SFN.DescribeStateMachine", + ) + + def test_stepfunctions_error(self): + self.do_test_requests( + "stepfunctions/error", + "GET", + 400, + 1, + 0, + local_operation="GET /stepfunctions", + rpc_service="SFN", + remote_service="AWS::StepFunctions", + remote_operation="DescribeStateMachine", + remote_resource_type="AWS::StepFunctions::StateMachine", + remote_resource_identifier="nonExistentStateMachine", + cloudformation_primary_identifier="arn:aws:states:us-west-2:000000000000:stateMachine:nonExistentStateMachine", + request_specific_attributes= { + _AWS_STATE_MACHINE_ARN: "arn:aws:states:us-west-2:000000000000:stateMachine:nonExistentStateMachine", + }, + span_name="SFN.DescribeStateMachine", + ) + + def test_stepfunctions_describe_state_machine(self): + self.do_test_requests( + "stepfunctions/describestatemachine/state-machine", + "GET", + 200, + 0, + 0, + local_operation="GET /stepfunctions", + rpc_service="SFN", + remote_service="AWS::StepFunctions", + remote_operation="DescribeStateMachine", + remote_resource_type="AWS::StepFunctions::StateMachine", + remote_resource_identifier="TestStateMachine", + cloudformation_primary_identifier="arn:aws:states:us-west-2:000000000000:stateMachine:TestStateMachine", + request_specific_attributes= { + _AWS_STATE_MACHINE_ARN: "arn:aws:states:us-west-2:000000000000:stateMachine:TestStateMachine", + }, + span_name="SFN.DescribeStateMachine", + ) + + def test_stepfunctions_describe_activity(self): + self.do_test_requests( + "stepfunctions/describeactivity/activity", + "GET", + 200, + 0, + 0, + local_operation="GET /stepfunctions", + rpc_service="SFN", + remote_service="AWS::StepFunctions", + remote_operation="DescribeActivity", + remote_resource_type="AWS::StepFunctions::Activity", + remote_resource_identifier="TestActivity", + cloudformation_primary_identifier="arn:aws:states:us-west-2:000000000000:activity:TestActivity", + request_specific_attributes= { + _AWS_ACTIVITY_ARN: "arn:aws:states:us-west-2:000000000000:activity:TestActivity", + }, + span_name="SFN.DescribeActivity", + ) + + def test_sns_fault(self): + self.do_test_requests( + "sns/fault", + "GET", + 500, + 0, + 1, + dp_count=3, + local_operation="GET /sns", + local_operation_2="POST /", + rpc_service="SNS", + remote_service="AWS::SNS", + remote_operation="GetTopicAttributes", + remote_resource_type="AWS::SNS::Topic", + remote_resource_identifier="invalidTopic", + cloudformation_primary_identifier="arn:aws:sns:us-west-2:000000000000:invalidTopic", + request_specific_attributes= { + _AWS_SNS_TOPIC_ARN: "arn:aws:sns:us-west-2:000000000000:invalidTopic", + }, + span_name="SNS GetTopicAttributes", + ) + + def test_sns_error(self): + self.do_test_requests( + "sns/error", + "GET", + 404, # this is the expected status code error for sns + 1, + 0, + local_operation="GET /sns", + rpc_service="SNS", + remote_service="AWS::SNS", + remote_operation="GetTopicAttributes", + remote_resource_type="AWS::SNS::Topic", + remote_resource_identifier="nonExistentTopic", + cloudformation_primary_identifier="arn:aws:sns:us-west-2:000000000000:nonExistentTopic", + request_specific_attributes= { + _AWS_SNS_TOPIC_ARN: "arn:aws:sns:us-west-2:000000000000:nonExistentTopic", + }, + span_name="SNS GetTopicAttributes", + ) + + def test_sns_get_topic_attributes(self): + self.do_test_requests( + "sns/gettopicattributes/topic", + "GET", + 200, + 0, + 0, + local_operation="GET /sns", + rpc_service="SNS", + remote_service="AWS::SNS", + remote_operation="GetTopicAttributes", + remote_resource_type="AWS::SNS::Topic", + remote_resource_identifier="TestTopic", + cloudformation_primary_identifier="arn:aws:sns:us-west-2:000000000000:TestTopic", + request_specific_attributes= { + _AWS_SNS_TOPIC_ARN: "arn:aws:sns:us-west-2:000000000000:TestTopic", + }, + span_name="SNS GetTopicAttributes", + ) + + def test_lambda_fault(self): + self.do_test_requests( + "lambda/fault", + "GET", + 500, + 0, + 1, + dp_count=3, + local_operation="GET /lambda", + local_operation_2="PUT /2015-03-31", + rpc_service="Lambda", + remote_service="AWS::Lambda", + remote_operation="UpdateEventSourceMapping", + remote_resource_type="AWS::Lambda::EventSourceMapping", + remote_resource_identifier="123e4567-e89b-12d3-a456-426614174000", + cloudformation_primary_identifier="123e4567-e89b-12d3-a456-426614174000", + request_specific_attributes= { + _AWS_LAMBDA_RESOURCE_MAPPING_ID: "123e4567-e89b-12d3-a456-426614174000", + }, + span_name="Lambda.UpdateEventSourceMapping", + ) + + def test_lambda_error(self): + self.do_test_requests( + "lambda/error", + "GET", + 404, + 1, + 0, + local_operation="GET /lambda", + rpc_service="Lambda", + remote_service="AWS::Lambda", + remote_operation="GetEventSourceMapping", + remote_resource_type="AWS::Lambda::EventSourceMapping", + remote_resource_identifier="nonExistentUUID", + cloudformation_primary_identifier="nonExistentUUID", + request_specific_attributes= { + _AWS_LAMBDA_RESOURCE_MAPPING_ID: "nonExistentUUID", + }, + span_name="Lambda.GetEventSourceMapping", + ) + + #TODO: Need to add test_lambda_get_event_source_mapping once workaround is figured out for storing UUID between tests + @override def _assert_aws_span_attributes(self, resource_scope_spans: List[ResourceScopeSpan], path: str, **kwargs) -> None: target_spans: List[Span] = [] @@ -742,6 +1052,7 @@ def _assert_aws_span_attributes(self, resource_scope_spans: List[ResourceScopeSp span_kind, kwargs.get("remote_resource_type", "None"), kwargs.get("remote_resource_identifier", "None"), + kwargs.get("cloudformation_primary_identifier", "None"), ) def _assert_aws_attributes( @@ -753,6 +1064,7 @@ def _assert_aws_attributes( span_kind: str, remote_resource_type: str, remote_resource_identifier: str, + cloudformation_primary_identifier: str ) -> None: attributes_dict: Dict[str, AnyValue] = self._get_attributes_dict(attributes_list) self._assert_str_attribute(attributes_dict, AWS_LOCAL_SERVICE, self.get_application_otel_service_name()) @@ -760,16 +1072,19 @@ def _assert_aws_attributes( self._assert_str_attribute(attributes_dict, AWS_REMOTE_SERVICE, remote_service) self._assert_str_attribute(attributes_dict, AWS_REMOTE_OPERATION, remote_operation) if remote_resource_type != "None": - self._assert_str_attribute(attributes_dict, AWS_REMOTE_RESOURCE_TYPE, remote_resource_type) + self._assert_attribute(attributes_dict, AWS_REMOTE_RESOURCE_TYPE, remote_resource_type) if remote_resource_identifier != "None": - self._assert_str_attribute(attributes_dict, AWS_REMOTE_RESOURCE_IDENTIFIER, remote_resource_identifier) + self._assert_attribute(attributes_dict, AWS_REMOTE_RESOURCE_IDENTIFIER, remote_resource_identifier) + if cloudformation_primary_identifier != "None": + self._assert_attribute(attributes_dict, AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER, cloudformation_primary_identifier) + self._assert_str_attribute(attributes_dict, AWS_SPAN_KIND, span_kind) @override def _assert_semantic_conventions_span_attributes( self, resource_scope_spans: List[ResourceScopeSpan], method: str, path: str, status_code: int, **kwargs ) -> None: - + target_spans: List[Span] = [] for resource_scope_span in resource_scope_spans: # pylint: disable=no-member @@ -786,7 +1101,7 @@ def _assert_semantic_conventions_span_attributes( status_code, kwargs.get("request_specific_attributes", {}), kwargs.get("response_specific_attributes", {}), - ) + ) # pylint: disable=unidiomatic-typecheck def _assert_semantic_conventions_attributes( @@ -805,19 +1120,11 @@ def _assert_semantic_conventions_attributes( self._assert_int_attribute(attributes_dict, SpanAttributes.HTTP_STATUS_CODE, status_code) # TODO: aws sdk instrumentation is not respecting PEER_SERVICE # self._assert_str_attribute(attributes_dict, SpanAttributes.PEER_SERVICE, "backend:8080") - self._assert_specific_attributes(attributes_dict, request_specific_attributes) - self._assert_specific_attributes(attributes_dict, response_specific_attributes) - - def _assert_specific_attributes(self, attributes_dict: Dict[str, AnyValue], specific_attributes: Dict[str, AnyValue]) -> None: - for key, value in specific_attributes.items(): - if isinstance(value, str): - self._assert_str_attribute(attributes_dict, key, value) - elif isinstance(value, int): - self._assert_int_attribute(attributes_dict, key, value) - elif isinstance(value, float): - self._assert_float_attribute(attributes_dict, key, value) - else: - self._assert_array_value_ddb_table_name(attributes_dict, key, value) + for key, value in request_specific_attributes.items(): + self._assert_attribute(attributes_dict, key, value) + + for key, value in response_specific_attributes.items(): + self._assert_attribute(attributes_dict, key, value) @override def _assert_metric_attributes( @@ -845,23 +1152,23 @@ def _assert_metric_attributes( dependency_dp = dp_list[1] service_dp = dp_list[0] attribute_dict: Dict[str, AnyValue] = self._get_attributes_dict(dependency_dp.attributes) - self._assert_str_attribute(attribute_dict, AWS_LOCAL_SERVICE, self.get_application_otel_service_name()) - self._assert_str_attribute(attribute_dict, AWS_LOCAL_OPERATION, kwargs.get("local_operation")) - self._assert_str_attribute(attribute_dict, AWS_REMOTE_SERVICE, kwargs.get("remote_service")) - self._assert_str_attribute(attribute_dict, AWS_REMOTE_OPERATION, kwargs.get("remote_operation")) - self._assert_str_attribute(attribute_dict, AWS_SPAN_KIND, kwargs.get("dependency_metric_span_kind") or "CLIENT") + self._assert_attribute(attribute_dict, AWS_LOCAL_SERVICE, self.get_application_otel_service_name()) + self._assert_attribute(attribute_dict, AWS_LOCAL_OPERATION, kwargs.get("local_operation")) + self._assert_attribute(attribute_dict, AWS_REMOTE_SERVICE, kwargs.get("remote_service")) + self._assert_attribute(attribute_dict, AWS_REMOTE_OPERATION, kwargs.get("remote_operation")) + self._assert_attribute(attribute_dict, AWS_SPAN_KIND, kwargs.get("dependency_metric_span_kind") or "CLIENT") remote_resource_type = kwargs.get("remote_resource_type", "None") remote_resource_identifier = kwargs.get("remote_resource_identifier", "None") if remote_resource_type != "None": - self._assert_str_attribute(attribute_dict, AWS_REMOTE_RESOURCE_TYPE, remote_resource_type) + self._assert_attribute(attribute_dict, AWS_REMOTE_RESOURCE_TYPE, remote_resource_type) if remote_resource_identifier != "None": - self._assert_str_attribute(attribute_dict, AWS_REMOTE_RESOURCE_IDENTIFIER, remote_resource_identifier) + self._assert_attribute(attribute_dict, AWS_REMOTE_RESOURCE_IDENTIFIER, remote_resource_identifier) self.check_sum(metric_name, dependency_dp.sum, expected_sum) attribute_dict: Dict[str, AnyValue] = self._get_attributes_dict(service_dp.attributes) - self._assert_str_attribute(attribute_dict, AWS_LOCAL_SERVICE, self.get_application_otel_service_name()) - self._assert_str_attribute(attribute_dict, AWS_LOCAL_OPERATION, kwargs.get("local_operation")) - self._assert_str_attribute(attribute_dict, AWS_SPAN_KIND, "LOCAL_ROOT") + self._assert_attribute(attribute_dict, AWS_LOCAL_SERVICE, self.get_application_otel_service_name()) + self._assert_attribute(attribute_dict, AWS_LOCAL_OPERATION, kwargs.get("local_operation")) + self._assert_attribute(attribute_dict, AWS_SPAN_KIND, "LOCAL_ROOT") self.check_sum(metric_name, service_dp.sum, expected_sum) else: dependency_dp: ExponentialHistogramDataPoint = max(dp_list, key=lambda dp: len(dp.attributes)) @@ -878,9 +1185,9 @@ def _assert_metric_attributes( remote_resource_type = kwargs.get("remote_resource_type", "None") remote_resource_identifier = kwargs.get("remote_resource_identifier", "None") if remote_resource_type != "None": - self._assert_str_attribute(attribute_dict, AWS_REMOTE_RESOURCE_TYPE, remote_resource_type) + self._assert_attribute(attribute_dict, AWS_REMOTE_RESOURCE_TYPE, remote_resource_type) if remote_resource_identifier != "None": - self._assert_str_attribute(attribute_dict, AWS_REMOTE_RESOURCE_IDENTIFIER, remote_resource_identifier) + self._assert_attribute(attribute_dict, AWS_REMOTE_RESOURCE_IDENTIFIER, remote_resource_identifier) self.check_sum(metric_name, dependency_dp.sum, expected_sum) attribute_dict_service: Dict[str, AnyValue] = self._get_attributes_dict(service_dp.attributes) @@ -902,10 +1209,23 @@ def _assert_metric_attributes( self._assert_str_attribute(attribute_dict_other, AWS_SPAN_KIND, "LOCAL_ROOT") self.check_sum(metric_name, other_dp.sum, expected_sum) + def _assert_attribute(self, attributes_dict: Dict[str, AnyValue], key, value) -> None: + if isinstance(value, str): + if self._is_valid_regex(value): + self._assert_match_attribute(attributes_dict, key, value) + else: + self._assert_str_attribute(attributes_dict, key, value) + elif isinstance(value, int): + self._assert_int_attribute(attributes_dict, key, value) + elif isinstance(value, float): + self._assert_float_attribute(attributes_dict, key, value) + else: + self._assert_array_value_ddb_table_name(attributes_dict, key, value) + # pylint: disable=consider-using-enumerate def _assert_array_value_ddb_table_name(self, attributes_dict: Dict[str, AnyValue], key: str, expect_values: list): self.assertIn(key, attributes_dict) actual_values: [AnyValue] = attributes_dict[key].array_value self.assertEqual(len(actual_values.values), len(expect_values)) for index in range(len(actual_values.values)): - self.assertEqual(actual_values.values[index].string_value, expect_values[index]) + self.assertEqual(actual_values.values[index].string_value, expect_values[index]) \ No newline at end of file diff --git a/contract-tests/tests/test/amazon/base/contract_test_base.py b/contract-tests/tests/test/amazon/base/contract_test_base.py index ea522f8..f5f2590 100644 --- a/contract-tests/tests/test/amazon/base/contract_test_base.py +++ b/contract-tests/tests/test/amazon/base/contract_test_base.py @@ -1,6 +1,7 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 import time +import re from logging import INFO, Logger, getLogger from typing import Dict, List from unittest import TestCase @@ -230,6 +231,12 @@ def _assert_int_attribute(self, attributes_dict: Dict[str, AnyValue], key: str, self.assertIsNotNone(actual_value) self.assertEqual(expected_value, actual_value.int_value) + def _assert_match_attribute(self, attributes_dict: Dict[str, AnyValue], key: str, pattern: str) -> None: + self.assertIn(key, attributes_dict) + actual_value: AnyValue = attributes_dict[key] + self.assertIsNotNone(actual_value) + self.assertRegex(actual_value.string_value, pattern) + def _assert_float_attribute(self, attributes_dict: Dict[str, AnyValue], key: str, expected_value: float) -> None: self.assertIn(key, attributes_dict) actual_value: AnyValue = attributes_dict[key] @@ -242,6 +249,13 @@ def check_sum(self, metric_name: str, actual_sum: float, expected_sum: float) -> else: self.assertEqual(actual_sum, expected_sum) + def _is_valid_regex(self, pattern: str): + try: + re.compile(pattern) + return True + except (re.error, StopIteration, RuntimeError, KeyError): + return False + # pylint: disable=no-self-use # Methods that should be overridden in subclasses @classmethod @@ -285,4 +299,4 @@ def _assert_semantic_conventions_span_attributes( def _assert_metric_attributes( self, resource_scope_metrics: List[ResourceScopeMetric], metric_name: str, expected_sum: int, **kwargs ): - self.fail("Tests must implement this function") + self.fail("Tests must implement this function") \ No newline at end of file diff --git a/contract-tests/tests/test/amazon/utils/application_signals_constants.py b/contract-tests/tests/test/amazon/utils/application_signals_constants.py index 9f3a625..14b602e 100644 --- a/contract-tests/tests/test/amazon/utils/application_signals_constants.py +++ b/contract-tests/tests/test/amazon/utils/application_signals_constants.py @@ -17,4 +17,5 @@ AWS_REMOTE_OPERATION: str = "aws.remote.operation" AWS_REMOTE_RESOURCE_TYPE: str = "aws.remote.resource.type" AWS_REMOTE_RESOURCE_IDENTIFIER: str = "aws.remote.resource.identifier" +AWS_CLOUDFORMATION_PRIMARY_IDENTIFIER: str = 'aws.remote.resource.cfn.primary.identifier' AWS_SPAN_KIND: str = "aws.span.kind" From 7593a67af340d95cb8648e657293d0e6819703e6 Mon Sep 17 00:00:00 2001 From: liustve Date: Tue, 3 Dec 2024 23:51:35 +0000 Subject: [PATCH 2/4] fixed contract test string assertion and added knowledgebase id cfa assertion --- .../images/applications/aws-sdk/server.js | 13 +----- .../tests/test/amazon/aws-sdk/aws_sdk_test.py | 44 +++++++------------ .../test/amazon/base/contract_test_base.py | 15 +------ 3 files changed, 17 insertions(+), 55 deletions(-) diff --git a/contract-tests/images/applications/aws-sdk/server.js b/contract-tests/images/applications/aws-sdk/server.js index 7c61039..7352bfa 100644 --- a/contract-tests/images/applications/aws-sdk/server.js +++ b/contract-tests/images/applications/aws-sdk/server.js @@ -13,7 +13,7 @@ const { DynamoDBClient, CreateTableCommand, PutItemCommand } = require('@aws-sdk const { SQSClient, CreateQueueCommand, SendMessageCommand, ReceiveMessageCommand } = require('@aws-sdk/client-sqs'); const { KinesisClient, CreateStreamCommand, PutRecordCommand } = require('@aws-sdk/client-kinesis'); const { BedrockClient, GetGuardrailCommand } = require('@aws-sdk/client-bedrock'); -const { BedrockAgentClient, GetKnowledgeBaseCommand, GetDataSourceCommand, GetAgentCommand, AssociateAgentKnowledgeBaseCommand } = require('@aws-sdk/client-bedrock-agent'); +const { BedrockAgentClient, GetKnowledgeBaseCommand, GetDataSourceCommand, GetAgentCommand } = require('@aws-sdk/client-bedrock-agent'); const { BedrockRuntimeClient, InvokeModelCommand } = require('@aws-sdk/client-bedrock-runtime'); const { BedrockAgentRuntimeClient, InvokeAgentCommand, RetrieveCommand } = require('@aws-sdk/client-bedrock-agent-runtime'); const { SNSClient, CreateTopicCommand, GetTopicAttributesCommand } = require('@aws-sdk/client-sns'); @@ -576,17 +576,6 @@ async function handleBedrockRequest(req, res, path) { } ); res.statusCode = 200; - } else if (path.includes('associateagent/associate_agent_knowledge_base')) { - await withInjected200Success(bedrockAgentRuntimeClient, ['AssociateAgentKnowledgeBaseCommand'], {}, async () => { - await bedrockAgentRuntimeClient.send( - new AssociateAgentKnowledgeBaseCommand({ - agentId: 'Q08WFRPHVL', - agentVersion: '1', - knowledgeBaseId: 'test-knowledge-base-id', - }) - ); - }); - res.statusCode = 200; } else if (path.includes('invokeagent/invoke_agent')) { await withInjected200Success(bedrockAgentRuntimeClient, ['InvokeAgentCommand'], {}, async () => { await bedrockAgentRuntimeClient.send( diff --git a/contract-tests/tests/test/amazon/aws-sdk/aws_sdk_test.py b/contract-tests/tests/test/amazon/aws-sdk/aws_sdk_test.py index b67bed6..c7cac2a 100644 --- a/contract-tests/tests/test/amazon/aws-sdk/aws_sdk_test.py +++ b/contract-tests/tests/test/amazon/aws-sdk/aws_sdk_test.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from logging import INFO, Logger, getLogger import math +import re from typing import Dict, List from docker.types import EndpointConfig @@ -648,7 +649,7 @@ def test_bedrock_get_guardrail(self): remote_operation="GetGuardrail", remote_resource_type="AWS::Bedrock::Guardrail", remote_resource_identifier="bt4o77i015cu", - cloudformation_primary_identifier="bt4o77i015cu", + cloudformation_primary_identifier="arn:aws:bedrock:us-east-1:000000000000:guardrail/bt4o77i015cu", request_specific_attributes={ _AWS_BEDROCK_GUARDRAIL_ID: "bt4o77i015cu", _AWS_BEDROCK_GUARDRAIL_ARN: "arn:aws:bedrock:us-east-1:000000000000:guardrail/bt4o77i015cu" @@ -696,27 +697,6 @@ def test_bedrock_agent_runtime_retrieve(self): span_name="BedrockAgentRuntime.Retrieve", ) - def test_bedrock_agent_associate_agent_knowledge_base(self): - self.do_test_requests( - "bedrock/associateagent/associate_agent_knowledge_base", - "GET", - 200, - 0, - 0, - local_operation="GET /bedrock", - rpc_service="BedrockAgentRuntime", - remote_service="AWS::Bedrock", - remote_operation="AssociateAgentKnowledgeBase", - remote_resource_type="AWS::Bedrock::Agent", - remote_resource_identifier="Q08WFRPHVL", - cloudformation_primary_identifier="test-knowledge-base-id|Q08WFRPHVL", - request_specific_attributes={ - _AWS_BEDROCK_AGENT_ID: "Q08WFRPHVL", - _AWS_BEDROCK_KNOWLEDGE_BASE_ID: "test-knowledge-base-id", - }, - span_name="BedrockAgentRuntime.AssociateAgentKnowledgeBase", - ) - def test_bedrock_agent_get_agent(self): self.do_test_requests( "bedrock/getagent/get-agent", @@ -770,7 +750,7 @@ def test_bedrock_agent_get_data_source(self): remote_operation="GetDataSource", remote_resource_type="AWS::Bedrock::DataSource", remote_resource_identifier="DATASURCID", - cloudformation_primary_identifier="DATASURCID", + cloudformation_primary_identifier=r'TESTKBSEID\|DATASURCID', request_specific_attributes={ _AWS_BEDROCK_DATA_SOURCE_ID: "DATASURCID", }, @@ -832,9 +812,9 @@ def test_secretsmanager_describe_secret(self): remote_operation="DescribeSecret", remote_resource_type="AWS::SecretsManager::Secret", remote_resource_identifier=r'MyTestSecret-[a-zA-Z0-9]{6}$', - cloudformation_primary_identifier=r"arn:aws:secretsmanager:us-west-2:000000000000:secret:MyTestSecret-[a-zA-Z0-9]{6}$", + cloudformation_primary_identifier=r'arn:aws:secretsmanager:us-west-2:000000000000:secret:MyTestSecret-[a-zA-Z0-9]{6}$', response_specific_attributes= { - _AWS_SECRET_ARN: r"arn:aws:secretsmanager:us-west-2:000000000000:secret:MyTestSecret-[a-zA-Z0-9]{6}$", + _AWS_SECRET_ARN: r'arn:aws:secretsmanager:us-west-2:000000000000:secret:MyTestSecret-[a-zA-Z0-9]{6}$', }, span_name="SecretsManager.DescribeSecret", ) @@ -1211,10 +1191,7 @@ def _assert_metric_attributes( def _assert_attribute(self, attributes_dict: Dict[str, AnyValue], key, value) -> None: if isinstance(value, str): - if self._is_valid_regex(value): - self._assert_match_attribute(attributes_dict, key, value) - else: - self._assert_str_attribute(attributes_dict, key, value) + self._assert_str_attribute(attributes_dict, key, value) elif isinstance(value, int): self._assert_int_attribute(attributes_dict, key, value) elif isinstance(value, float): @@ -1222,6 +1199,15 @@ def _assert_attribute(self, attributes_dict: Dict[str, AnyValue], key, value) -> else: self._assert_array_value_ddb_table_name(attributes_dict, key, value) + @override + def _assert_str_attribute(self, attributes_dict: Dict[str, AnyValue], key: str, expected_value: str): + self.assertIn(key, attributes_dict) + actual_value: AnyValue = attributes_dict[key] + self.assertIsNotNone(actual_value) + pattern = re.compile(expected_value) + match = pattern.fullmatch(actual_value.string_value) + self.assertTrue(match is not None, f"Actual: {actual_value.string_value} does not match Expected: {expected_value}") + # pylint: disable=consider-using-enumerate def _assert_array_value_ddb_table_name(self, attributes_dict: Dict[str, AnyValue], key: str, expect_values: list): self.assertIn(key, attributes_dict) diff --git a/contract-tests/tests/test/amazon/base/contract_test_base.py b/contract-tests/tests/test/amazon/base/contract_test_base.py index f5f2590..07a59ab 100644 --- a/contract-tests/tests/test/amazon/base/contract_test_base.py +++ b/contract-tests/tests/test/amazon/base/contract_test_base.py @@ -230,13 +230,7 @@ def _assert_int_attribute(self, attributes_dict: Dict[str, AnyValue], key: str, actual_value: AnyValue = attributes_dict[key] self.assertIsNotNone(actual_value) self.assertEqual(expected_value, actual_value.int_value) - - def _assert_match_attribute(self, attributes_dict: Dict[str, AnyValue], key: str, pattern: str) -> None: - self.assertIn(key, attributes_dict) - actual_value: AnyValue = attributes_dict[key] - self.assertIsNotNone(actual_value) - self.assertRegex(actual_value.string_value, pattern) - + def _assert_float_attribute(self, attributes_dict: Dict[str, AnyValue], key: str, expected_value: float) -> None: self.assertIn(key, attributes_dict) actual_value: AnyValue = attributes_dict[key] @@ -249,13 +243,6 @@ def check_sum(self, metric_name: str, actual_sum: float, expected_sum: float) -> else: self.assertEqual(actual_sum, expected_sum) - def _is_valid_regex(self, pattern: str): - try: - re.compile(pattern) - return True - except (re.error, StopIteration, RuntimeError, KeyError): - return False - # pylint: disable=no-self-use # Methods that should be overridden in subclasses @classmethod From de117591d400b64ae263afa8835ebe172d92ce41 Mon Sep 17 00:00:00 2001 From: liustve Date: Tue, 3 Dec 2024 23:56:22 +0000 Subject: [PATCH 3/4] rerun --- contract-tests/images/applications/mysql2/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/contract-tests/images/applications/mysql2/package.json b/contract-tests/images/applications/mysql2/package.json index 0c1fe64..bb1bcbd 100644 --- a/contract-tests/images/applications/mysql2/package.json +++ b/contract-tests/images/applications/mysql2/package.json @@ -12,4 +12,5 @@ "dependencies": { "mysql2": "3.11.4" } + } From 34b6ea5dec8c62fefcc54385f94a46ee0bed6827 Mon Sep 17 00:00:00 2001 From: liustve Date: Wed, 4 Dec 2024 00:00:03 +0000 Subject: [PATCH 4/4] rerun --- contract-tests/images/applications/mysql2/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/contract-tests/images/applications/mysql2/package.json b/contract-tests/images/applications/mysql2/package.json index bb1bcbd..0c1fe64 100644 --- a/contract-tests/images/applications/mysql2/package.json +++ b/contract-tests/images/applications/mysql2/package.json @@ -12,5 +12,4 @@ "dependencies": { "mysql2": "3.11.4" } - }