From 3c10fe639727c3fd1f16024ca190cecb14f0e144 Mon Sep 17 00:00:00 2001 From: Momo Kornher Date: Fri, 13 Sep 2024 12:33:51 +0100 Subject: [PATCH] fix(package-sources): upgrade to AWS SDK v3 (#1452) Refactor the package-source lambda functions to use AWS SDK v3. Tests updated accordingly. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../__snapshots__/construct-hub.test.ts.snap | 30 +- .../__snapshots__/snapshot.test.ts.snap | 6 +- .../__snapshots__/code-artifact.test.ts.snap | 4 +- .../code-artifact-forwarder.lambda.test.ts | 363 ++++++++---------- .../npmjs/stage-and-notify.lambda.test.ts | 101 ++++- .../code-artifact-forwarder.lambda.ts | 40 +- .../canary/npmjs-package-canary.lambda.ts | 55 +-- .../npmjs/npm-js-follower.lambda.ts | 83 ++-- .../npmjs/stage-and-notify.lambda.ts | 18 +- 9 files changed, 386 insertions(+), 314 deletions(-) diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index df9845bec..374dc9f11 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -9398,7 +9398,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "9daa8d4a97f3b2b8f5a8902b59d725d97c286de1a0c6a342f310d5a1c60e7591.zip", + "S3Key": "455dc8070938b6c14e85478c54597fbcdd4f5b1fe07673be6f0b8918329ff2c6.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -9452,7 +9452,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "741a47347e5b7f4fb51631f474a7f89e7ee84b6eec5d1abec623a65f92b3c3f4.zip", + "S3Key": "4a3c5d0d0a2826e02448be301d54f2e7e86b2be27b33abcada9c2d5e1d0f92eb.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs/PackageCanary] Monitors construct-hub-probe versions availability", "Environment": { @@ -10157,7 +10157,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "99987e97a1ddafe10ed59318a559ebf8453b1d30732c70f4e29578136335928d.zip", + "S3Key": "50b6d529a3a7020ad49e5365ec7d79e8f1d8f7c874a80e13a74f12fde65ccab3.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -22397,7 +22397,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "9daa8d4a97f3b2b8f5a8902b59d725d97c286de1a0c6a342f310d5a1c60e7591.zip", + "S3Key": "455dc8070938b6c14e85478c54597fbcdd4f5b1fe07673be6f0b8918329ff2c6.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -22451,7 +22451,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "741a47347e5b7f4fb51631f474a7f89e7ee84b6eec5d1abec623a65f92b3c3f4.zip", + "S3Key": "4a3c5d0d0a2826e02448be301d54f2e7e86b2be27b33abcada9c2d5e1d0f92eb.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs/PackageCanary] Monitors construct-hub-probe versions availability", "Environment": { @@ -23156,7 +23156,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "99987e97a1ddafe10ed59318a559ebf8453b1d30732c70f4e29578136335928d.zip", + "S3Key": "50b6d529a3a7020ad49e5365ec7d79e8f1d8f7c874a80e13a74f12fde65ccab3.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -34968,7 +34968,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "9daa8d4a97f3b2b8f5a8902b59d725d97c286de1a0c6a342f310d5a1c60e7591.zip", + "S3Key": "455dc8070938b6c14e85478c54597fbcdd4f5b1fe07673be6f0b8918329ff2c6.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -35022,7 +35022,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "741a47347e5b7f4fb51631f474a7f89e7ee84b6eec5d1abec623a65f92b3c3f4.zip", + "S3Key": "4a3c5d0d0a2826e02448be301d54f2e7e86b2be27b33abcada9c2d5e1d0f92eb.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs/PackageCanary] Monitors construct-hub-probe versions availability", "Environment": { @@ -35727,7 +35727,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "99987e97a1ddafe10ed59318a559ebf8453b1d30732c70f4e29578136335928d.zip", + "S3Key": "50b6d529a3a7020ad49e5365ec7d79e8f1d8f7c874a80e13a74f12fde65ccab3.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -47688,7 +47688,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "9daa8d4a97f3b2b8f5a8902b59d725d97c286de1a0c6a342f310d5a1c60e7591.zip", + "S3Key": "455dc8070938b6c14e85478c54597fbcdd4f5b1fe07673be6f0b8918329ff2c6.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -47742,7 +47742,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "741a47347e5b7f4fb51631f474a7f89e7ee84b6eec5d1abec623a65f92b3c3f4.zip", + "S3Key": "4a3c5d0d0a2826e02448be301d54f2e7e86b2be27b33abcada9c2d5e1d0f92eb.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs/PackageCanary] Monitors construct-hub-probe versions availability", "Environment": { @@ -48434,7 +48434,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "99987e97a1ddafe10ed59318a559ebf8453b1d30732c70f4e29578136335928d.zip", + "S3Key": "50b6d529a3a7020ad49e5365ec7d79e8f1d8f7c874a80e13a74f12fde65ccab3.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -60389,7 +60389,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "9daa8d4a97f3b2b8f5a8902b59d725d97c286de1a0c6a342f310d5a1c60e7591.zip", + "S3Key": "455dc8070938b6c14e85478c54597fbcdd4f5b1fe07673be6f0b8918329ff2c6.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -60443,7 +60443,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "741a47347e5b7f4fb51631f474a7f89e7ee84b6eec5d1abec623a65f92b3c3f4.zip", + "S3Key": "4a3c5d0d0a2826e02448be301d54f2e7e86b2be27b33abcada9c2d5e1d0f92eb.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs/PackageCanary] Monitors construct-hub-probe versions availability", "Environment": { @@ -61148,7 +61148,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "99987e97a1ddafe10ed59318a559ebf8453b1d30732c70f4e29578136335928d.zip", + "S3Key": "50b6d529a3a7020ad49e5365ec7d79e8f1d8f7c874a80e13a74f12fde65ccab3.zip", }, "DeadLetterConfig": { "TargetArn": { diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 182dbe449..28a04fe1b 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -10680,7 +10680,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "9daa8d4a97f3b2b8f5a8902b59d725d97c286de1a0c6a342f310d5a1c60e7591.zip", + "S3Key": "455dc8070938b6c14e85478c54597fbcdd4f5b1fe07673be6f0b8918329ff2c6.zip", }, "Description": "[dev/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -10734,7 +10734,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "741a47347e5b7f4fb51631f474a7f89e7ee84b6eec5d1abec623a65f92b3c3f4.zip", + "S3Key": "4a3c5d0d0a2826e02448be301d54f2e7e86b2be27b33abcada9c2d5e1d0f92eb.zip", }, "Description": "[dev/ConstructHub/Sources/NpmJs/PackageCanary] Monitors construct-hub-probe versions availability", "Environment": { @@ -11433,7 +11433,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "99987e97a1ddafe10ed59318a559ebf8453b1d30732c70f4e29578136335928d.zip", + "S3Key": "50b6d529a3a7020ad49e5365ec7d79e8f1d8f7c874a80e13a74f12fde65ccab3.zip", }, "DeadLetterConfig": { "TargetArn": { diff --git a/src/__tests__/package-sources/__snapshots__/code-artifact.test.ts.snap b/src/__tests__/package-sources/__snapshots__/code-artifact.test.ts.snap index 5319f896a..159fd40c4 100644 --- a/src/__tests__/package-sources/__snapshots__/code-artifact.test.ts.snap +++ b/src/__tests__/package-sources/__snapshots__/code-artifact.test.ts.snap @@ -243,7 +243,7 @@ exports[`default configuration 1`] = ` "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ec30c61650234004be8dc0d77a47229fbdae737cf9a4d2dbc8acb91741c42a73.zip", + "S3Key": "cef554c64ae6cb679b47371ce4759963a74eabba08cc65b570afce94c1821f0e.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -727,7 +727,7 @@ exports[`user-provided staging bucket 1`] = ` "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ec30c61650234004be8dc0d77a47229fbdae737cf9a4d2dbc8acb91741c42a73.zip", + "S3Key": "cef554c64ae6cb679b47371ce4759963a74eabba08cc65b570afce94c1821f0e.zip", }, "DeadLetterConfig": { "TargetArn": { diff --git a/src/__tests__/package-sources/codeartifact/code-artifact-forwarder.lambda.test.ts b/src/__tests__/package-sources/codeartifact/code-artifact-forwarder.lambda.test.ts index 01f0a06eb..7dd712484 100644 --- a/src/__tests__/package-sources/codeartifact/code-artifact-forwarder.lambda.test.ts +++ b/src/__tests__/package-sources/codeartifact/code-artifact-forwarder.lambda.test.ts @@ -1,16 +1,30 @@ import { pseudoRandomBytes } from 'crypto'; +import { + CodeartifactClient, + GetPackageVersionAssetCommand, + GetPackageVersionAssetCommandOutput, +} from '@aws-sdk/client-codeartifact'; +import { + PutObjectCommand, + PutObjectCommandOutput, + S3Client, +} from '@aws-sdk/client-s3'; +import { + SendMessageCommand, + SendMessageResult, + SQSClient, +} from '@aws-sdk/client-sqs'; import { SPEC_FILE_NAME } from '@jsii/spec'; import type { Context } from 'aws-lambda'; -import * as AWS from 'aws-sdk'; -import * as AWSMock from 'aws-sdk-mock'; +import { mockClient } from 'aws-sdk-client-mock'; import * as denyListClient from '../../../backend/deny-list/client.lambda-shared'; import * as licenseListClient from '../../../backend/license-list/client.lambda-shared'; -import { reset } from '../../../backend/shared/aws.lambda-shared'; import * as env from '../../../backend/shared/env.lambda-shared'; import { integrity } from '../../../backend/shared/integrity.lambda-shared'; import * as tarball from '../../../backend/shared/tarball.lambda-shared'; import { handler } from '../../../package-sources/codeartifact/code-artifact-forwarder.lambda'; import { safeMock } from '../../safe-mock'; +import { stringToStream } from '../../streams'; const mockBucketName = 'mock-bucket-name'; const mockQueueUrl = 'https://mock-queue-url/dummy'; @@ -69,13 +83,14 @@ const mockExtractObjects = ( tarball.extractObjects as jest.MockedFunction ).mockName('tarball.extractObjects'); -beforeEach(() => { - AWSMock.setSDKInstance(AWS); -}); +const mockCodeartifact = mockClient(CodeartifactClient); +const mockS3 = mockClient(S3Client); +const mockSQS = mockClient(SQSClient); -afterEach(() => { - AWSMock.restore(); - reset(); +beforeEach(() => { + mockCodeartifact.reset(); + mockS3.reset(); + mockSQS.reset(); }); type RequestType = Parameters[0]; @@ -103,46 +118,33 @@ test('happy path', async () => { )}`, }); - const mockGetPackageVersionAssetResult: AWS.CodeArtifact.GetPackageVersionAssetResult = - { - asset: 'mock-asset-content', - assetName: 'package.tgz', - packageVersion: '1.2.3-dev.1337', - packageVersionRevision: pseudoRandomBytes(10).toString('base64'), - }; - AWSMock.mock( - 'CodeArtifact', - 'getPackageVersionAsset', - ( - request: AWS.CodeArtifact.GetPackageVersionAssetRequest, - cb: Response - ) => { - try { - expect(request).toEqual({ - asset: 'package.tgz', - format: 'npm', - domainOwner: detail.domainOwner, - domain: detail.domainName, - repository: detail.repositoryName, - namespace: detail.packageNamespace, - package: detail.packageName, - packageVersion: detail.packageVersion, - }); - cb(null, mockGetPackageVersionAssetResult); - } catch (e: any) { - cb(e); - } - } - ); + const mockAsset = 'mock-asset-content'; + const mockGetPackageVersionAssetResult: GetPackageVersionAssetResponse = { + asset: stringToStream(mockAsset), + assetName: 'package.tgz', + packageVersion: '1.2.3-dev.1337', + packageVersionRevision: pseudoRandomBytes(10).toString('base64'), + }; + + mockCodeartifact + .on(GetPackageVersionAssetCommand, { + asset: 'package.tgz', + format: 'npm', + domainOwner: detail.domainOwner, + domain: detail.domainName, + repository: detail.repositoryName, + namespace: detail.packageNamespace, + package: detail.packageName, + packageVersion: detail.packageVersion, + }) + .resolves(mockGetPackageVersionAssetResult); const mockAssembly = Buffer.from('mock-assembly-content'); const mockPackageJson = safeMock('package.json', { license: 'Apache-2.0', }); mockExtractObjects.mockImplementationOnce(async (tgz, selector) => { - expect(tgz).toEqual( - Buffer.from(mockGetPackageVersionAssetResult.asset! as any) - ); + expect(tgz).toEqual(Buffer.from(mockAsset)); expect(selector).toHaveProperty('assemblyJson', { path: `package/${SPEC_FILE_NAME}`, }); @@ -160,75 +162,56 @@ test('happy path', async () => { mockLicenseListLookup.mockReturnValueOnce('Apache-2.0'); const stagingKey = `@${detail.packageNamespace}/${detail.packageName}/${detail.packageVersion}/${mockGetPackageVersionAssetResult.packageVersionRevision}/package.tgz`; - AWSMock.mock( - 'S3', - 'putObject', - (req: AWS.S3.PutObjectRequest, cb: Response) => { - try { - expect(req).toEqual({ - Bucket: mockBucketName, - Key: stagingKey, - Body: mockGetPackageVersionAssetResult.asset!, - ContentType: 'application/octet-stream', - Metadata: { - 'Lambda-Log-Group': mockContext.logGroupName, - 'Lambda-Log-Stream': mockContext.logStreamName, - 'Lambda-Run-Id': mockContext.awsRequestId, - }, - }); - cb(null, safeMock('mockS3PutObjectOutput', {})); - } catch (e: any) { - cb(e); - } - } - ); - const mockSendMessageResult: AWS.SQS.SendMessageResult = { + mockS3.on(PutObjectCommand).callsFake((req) => { + expect(req).toEqual({ + Bucket: mockBucketName, + Key: stagingKey, + Body: mockGetPackageVersionAssetResult.asset!, + ContentType: 'application/octet-stream', + Metadata: { + 'Lambda-Log-Group': mockContext.logGroupName, + 'Lambda-Log-Stream': mockContext.logStreamName, + 'Lambda-Run-Id': mockContext.awsRequestId, + }, + }); + return safeMock('mockS3PutObjectOutput', {}); + }); + + const mockSendMessageResult: SendMessageResult = { MessageId: pseudoRandomBytes(10).toString('base64'), }; const time = new Date().toISOString(); const resources = ['arn:obviously:made:up']; - AWSMock.mock( - 'SQS', - 'sendMessage', - ( - req: AWS.SQS.SendMessageRequest, - cb: Response - ) => { - try { - expect(req).toEqual({ - MessageAttributes: { - AWS_REQUEST_ID: { - DataType: 'String', - StringValue: mockContext.awsRequestId, - }, - LOG_GROUP_NAME: { - DataType: 'String', - StringValue: mockContext.logGroupName, - }, - LOG_STREAM_NAME: { - DataType: 'String', - StringValue: mockContext.logStreamName, - }, + mockSQS + .on(SendMessageCommand, { + MessageAttributes: { + AWS_REQUEST_ID: { + DataType: 'String', + StringValue: mockContext.awsRequestId, + }, + LOG_GROUP_NAME: { + DataType: 'String', + StringValue: mockContext.logGroupName, + }, + LOG_STREAM_NAME: { + DataType: 'String', + StringValue: mockContext.logStreamName, + }, + }, + MessageBody: JSON.stringify( + integrity( + { + tarballUri: `s3://${mockBucketName}/${stagingKey}`, + metadata: { resources: resources.join(', ') }, + time, }, - MessageBody: JSON.stringify( - integrity( - { - tarballUri: `s3://${mockBucketName}/${stagingKey}`, - metadata: { resources: resources.join(', ') }, - time, - }, - Buffer.from(mockGetPackageVersionAssetResult.asset! as any) - ) - ), - QueueUrl: mockQueueUrl, - }); - cb(null, mockSendMessageResult); - } catch (e: any) { - cb(e); - } - } - ); + Buffer.from(mockAsset) + ) + ), + QueueUrl: mockQueueUrl, + }) + .resolves(mockSendMessageResult); // WHEN const request: RequestType = safeMock('request', { @@ -260,44 +243,31 @@ test('no license (i.e: UNLICENSED)', async () => { )}`, }); - const mockGetPackageVersionAssetResult: AWS.CodeArtifact.GetPackageVersionAssetResult = - { - asset: 'mock-asset-content', - assetName: 'package.tgz', - packageVersion: '1.2.3-dev.1337', - packageVersionRevision: pseudoRandomBytes(10).toString('base64'), - }; - AWSMock.mock( - 'CodeArtifact', - 'getPackageVersionAsset', - ( - request: AWS.CodeArtifact.GetPackageVersionAssetRequest, - cb: Response - ) => { - try { - expect(request).toEqual({ - asset: 'package.tgz', - format: 'npm', - domainOwner: detail.domainOwner, - domain: detail.domainName, - repository: detail.repositoryName, - namespace: detail.packageNamespace, - package: detail.packageName, - packageVersion: detail.packageVersion, - }); - cb(null, mockGetPackageVersionAssetResult); - } catch (e: any) { - cb(e); - } - } - ); + const mockAsset = 'mock-asset-content'; + const mockGetPackageVersionAssetResult: GetPackageVersionAssetResponse = { + asset: stringToStream(mockAsset), + assetName: 'package.tgz', + packageVersion: '1.2.3-dev.1337', + packageVersionRevision: pseudoRandomBytes(10).toString('base64'), + }; + + mockCodeartifact + .on(GetPackageVersionAssetCommand, { + asset: 'package.tgz', + format: 'npm', + domainOwner: detail.domainOwner, + domain: detail.domainName, + repository: detail.repositoryName, + namespace: detail.packageNamespace, + package: detail.packageName, + packageVersion: detail.packageVersion, + }) + .resolves(mockGetPackageVersionAssetResult); const mockAssembly = Buffer.from('mock-assembly-content'); const mockPackageJson = safeMock('package.json', { license: undefined }); mockExtractObjects.mockImplementationOnce(async (tgz, selector) => { - expect(tgz).toEqual( - Buffer.from(mockGetPackageVersionAssetResult.asset! as any) - ); + expect(tgz).toEqual(Buffer.from(mockAsset)); expect(selector).toHaveProperty('assemblyJson', { path: `package/${SPEC_FILE_NAME}`, }); @@ -335,46 +305,32 @@ test('ineligible license', async () => { )}`, }); - const mockGetPackageVersionAssetResult: AWS.CodeArtifact.GetPackageVersionAssetResult = - { - asset: 'mock-asset-content', - assetName: 'package.tgz', - packageVersion: '1.2.3-dev.1337', - packageVersionRevision: pseudoRandomBytes(10).toString('base64'), - }; - AWSMock.mock( - 'CodeArtifact', - 'getPackageVersionAsset', - ( - request: AWS.CodeArtifact.GetPackageVersionAssetRequest, - cb: Response - ) => { - try { - expect(request).toEqual({ - asset: 'package.tgz', - format: 'npm', - domainOwner: detail.domainOwner, - domain: detail.domainName, - repository: detail.repositoryName, - namespace: detail.packageNamespace, - package: detail.packageName, - packageVersion: detail.packageVersion, - }); - cb(null, mockGetPackageVersionAssetResult); - } catch (e: any) { - cb(e); - } - } - ); + const mockAsset = 'mock-asset-content'; + const mockGetPackageVersionAssetResult: GetPackageVersionAssetResponse = { + asset: stringToStream(mockAsset), + assetName: 'package.tgz', + packageVersion: '1.2.3-dev.1337', + packageVersionRevision: pseudoRandomBytes(10).toString('base64'), + }; + mockCodeartifact + .on(GetPackageVersionAssetCommand, { + asset: 'package.tgz', + format: 'npm', + domainOwner: detail.domainOwner, + domain: detail.domainName, + repository: detail.repositoryName, + namespace: detail.packageNamespace, + package: detail.packageName, + packageVersion: detail.packageVersion, + }) + .resolves(mockGetPackageVersionAssetResult); const mockAssembly = Buffer.from('mock-assembly-content'); const mockPackageJson = safeMock('package.json', { license: 'Phony-MOCK', }); mockExtractObjects.mockImplementationOnce(async (tgz, selector) => { - expect(tgz).toEqual( - Buffer.from(mockGetPackageVersionAssetResult.asset! as any) - ); + expect(tgz).toEqual(Buffer.from(mockAsset)); expect(selector).toHaveProperty('assemblyJson', { path: `package/${SPEC_FILE_NAME}`, }); @@ -412,45 +368,31 @@ test('not a jsii package', async () => { )}`, }); - const mockGetPackageVersionAssetResult: AWS.CodeArtifact.GetPackageVersionAssetResult = - { - asset: 'mock-asset-content', - assetName: 'package.tgz', - packageVersion: '1.2.3-dev.1337', - packageVersionRevision: pseudoRandomBytes(10).toString('base64'), - }; - AWSMock.mock( - 'CodeArtifact', - 'getPackageVersionAsset', - ( - request: AWS.CodeArtifact.GetPackageVersionAssetRequest, - cb: Response - ) => { - try { - expect(request).toEqual({ - asset: 'package.tgz', - format: 'npm', - domainOwner: detail.domainOwner, - domain: detail.domainName, - repository: detail.repositoryName, - namespace: detail.packageNamespace, - package: detail.packageName, - packageVersion: detail.packageVersion, - }); - cb(null, mockGetPackageVersionAssetResult); - } catch (e: any) { - cb(e); - } - } - ); + const mockAsset = 'mock-asset-content'; + const mockGetPackageVersionAssetResult: GetPackageVersionAssetResponse = { + asset: stringToStream(mockAsset), + assetName: 'package.tgz', + packageVersion: '1.2.3-dev.1337', + packageVersionRevision: pseudoRandomBytes(10).toString('base64'), + }; + mockCodeartifact + .on(GetPackageVersionAssetCommand, { + asset: 'package.tgz', + format: 'npm', + domainOwner: detail.domainOwner, + domain: detail.domainName, + repository: detail.repositoryName, + namespace: detail.packageNamespace, + package: detail.packageName, + packageVersion: detail.packageVersion, + }) + .resolves(mockGetPackageVersionAssetResult); const mockPackageJson = safeMock('package.json', { license: 'Apache-2.0', }); mockExtractObjects.mockImplementationOnce(async (tgz, selector) => { - expect(tgz).toEqual( - Buffer.from(mockGetPackageVersionAssetResult.asset! as any) - ); + expect(tgz).toEqual(Buffer.from(mockAsset)); expect(selector).toHaveProperty('assemblyJson', { path: `package/${SPEC_FILE_NAME}`, }); @@ -512,4 +454,7 @@ test('deleted package', async () => { return expect(handler(request, mockContext)).resolves.toBeUndefined(); }); -type Response = (err: AWS.AWSError | null, data?: T) => void; +type GetPackageVersionAssetResponse = Omit< + GetPackageVersionAssetCommandOutput, + '$metadata' +>; diff --git a/src/__tests__/package-sources/npmjs/stage-and-notify.lambda.test.ts b/src/__tests__/package-sources/npmjs/stage-and-notify.lambda.test.ts index 0b6c30044..c19d41207 100644 --- a/src/__tests__/package-sources/npmjs/stage-and-notify.lambda.test.ts +++ b/src/__tests__/package-sources/npmjs/stage-and-notify.lambda.test.ts @@ -1,23 +1,37 @@ -import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; -import { Context } from 'aws-lambda'; +import { + GetObjectCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import { SendMessageCommand, SQSClient } from '@aws-sdk/client-sqs'; +import type { Context } from 'aws-lambda'; import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest'; import * as nock from 'nock'; import { ENV_DENY_LIST_BUCKET_NAME, ENV_DENY_LIST_OBJECT_KEY, } from '../../../backend/deny-list/constants'; +import { S3KeyPrefix } from '../../../package-sources/npmjs/constants.lambda-shared'; import { handler, PackageVersion, } from '../../../package-sources/npmjs/stage-and-notify.lambda'; import { stringToStream } from '../../streams'; +const MOCK_STAGING_BUCKET = 'foo'; +const MOCK_QUEUE_URL = 'bar'; const MOCK_DENY_LIST_BUCKET = 'deny-list-bucket-name'; const MOCK_DENY_LIST_OBJECT = 'my-deny-list.json'; +const mockS3 = mockClient(S3Client); +const mockSQS = mockClient(SQSClient); + beforeEach(() => { - process.env.BUCKET_NAME = 'foo'; - process.env.QUEUE_URL = 'bar'; + mockS3.reset(); + mockSQS.reset(); + process.env.BUCKET_NAME = MOCK_STAGING_BUCKET; + process.env.QUEUE_URL = MOCK_QUEUE_URL; process.env[ENV_DENY_LIST_BUCKET_NAME] = MOCK_DENY_LIST_BUCKET; process.env[ENV_DENY_LIST_OBJECT_KEY] = MOCK_DENY_LIST_OBJECT; }); @@ -29,16 +43,85 @@ afterEach(() => { delete process.env[ENV_DENY_LIST_OBJECT_KEY]; }); -test('ignores 404', async () => { +test('happy path', async () => { const basePath = 'https://registry.npmjs.org'; - const uri = '/@pepperize/cdk-vpc/-/cdk-vpc-0.0.785.tgz'; + const uri = '@pepperize/cdk-vpc/-/cdk-vpc-0.0.785.tgz'; + const stagingKey = `${S3KeyPrefix.STAGED_KEY_PREFIX}${uri}`; + const tarball = 'tarball'; + + // deny list + mockS3 + .on(GetObjectCommand, { + Bucket: MOCK_DENY_LIST_BUCKET, + Key: MOCK_DENY_LIST_OBJECT, + }) + .resolves({ + Body: stringToStream(JSON.stringify({})), + }); + + // registry response + nock(basePath).get(`/${uri}`).reply(200, tarball); + + const event: PackageVersion = { + tarballUrl: `${basePath}/${uri}`, + integrity: '09d37ec93c5518bf4842ac8e381a5c06452500e5', + modified: '2023-09-22T15:48:10.381Z', + name: '@pepper/cdk-vpc', + seq: '26437963', + version: '0.0.785', + }; + + const context: Context = { + logGroupName: 'group', + logStreamName: 'stream', + awsRequestId: 'request-id', + } as any; - const s3Mock = mockClient(S3Client); + await expect(handler(event, context)).resolves.toBe(undefined); + + expect(mockS3).toHaveReceivedCommandTimes(PutObjectCommand, 1); + expect(mockS3).toHaveReceivedCommandWith(PutObjectCommand, { + Bucket: MOCK_STAGING_BUCKET, + Key: stagingKey, + Body: Buffer.from(tarball), + ContentType: 'application/octet-stream', + Metadata: expect.anything(), + }); - s3Mock.on(GetObjectCommand).resolves({ - Body: stringToStream(JSON.stringify({})), + expect(mockSQS).toHaveReceivedCommandTimes(SendMessageCommand, 1); + expect(mockSQS).toHaveReceivedCommandWith(SendMessageCommand, { + MessageBody: JSON.stringify({ + tarballUri: `s3://${MOCK_STAGING_BUCKET}/${stagingKey}`, + metadata: { + dist: event.tarballUrl, + integrity: event.integrity, + modified: event.modified, + seq: event.seq, + }, + time: event.modified, + integrity: + 'sha384-Ebfvd5xY6T7bTyV20TsEHDVZPrWk2boggPbYA7vTwum0xhDLIDx+tOU0wTcVUnHy', + }), + MessageAttributes: expect.anything(), + QueueUrl: MOCK_QUEUE_URL, }); +}); + +test('ignores 404', async () => { + const basePath = 'https://registry.npmjs.org'; + const uri = '/@pepperize/cdk-vpc/-/cdk-vpc-0.0.785.tgz'; + + // deny list + mockS3 + .on(GetObjectCommand, { + Bucket: MOCK_DENY_LIST_BUCKET, + Key: MOCK_DENY_LIST_OBJECT, + }) + .resolves({ + Body: stringToStream(JSON.stringify({})), + }); + // registry response nock(basePath).get(uri).reply(404); const event: PackageVersion = { diff --git a/src/package-sources/codeartifact/code-artifact-forwarder.lambda.ts b/src/package-sources/codeartifact/code-artifact-forwarder.lambda.ts index d0804ed50..bd49c85ab 100644 --- a/src/package-sources/codeartifact/code-artifact-forwarder.lambda.ts +++ b/src/package-sources/codeartifact/code-artifact-forwarder.lambda.ts @@ -1,7 +1,14 @@ +import { + CodeartifactClient, + GetPackageVersionAssetCommand, + PackageFormat, +} from '@aws-sdk/client-codeartifact'; +import { PutObjectCommand } from '@aws-sdk/client-s3'; +import { SendMessageCommand } from '@aws-sdk/client-sqs'; import { SPEC_FILE_NAME } from '@jsii/spec'; import { metricScope, Unit } from 'aws-embedded-metrics'; import type { Context, EventBridgeEvent } from 'aws-lambda'; - +import { captureAWSv3Client } from 'aws-xray-sdk-core'; import { METRICS_NAMESPACE, MetricName, @@ -11,13 +18,17 @@ import { } from './constants.lambda-shared'; import { DenyListClient } from '../../backend/deny-list/client.lambda-shared'; import { LicenseListClient } from '../../backend/license-list/client.lambda-shared'; -import * as aws from '../../backend/shared/aws.lambda-shared'; +import { S3_CLIENT, SQS_CLIENT } from '../../backend/shared/aws.lambda-shared'; import { requireEnv } from '../../backend/shared/env.lambda-shared'; import { integrity } from '../../backend/shared/integrity.lambda-shared'; import { extractObjects } from '../../backend/shared/tarball.lambda-shared'; const DETAIL_TYPE = 'CodeArtifact Package Version State Change' as const; +const CODEARTIFACT_CLIENT: CodeartifactClient = captureAWSv3Client( + new CodeartifactClient() +); + export const handler = metricScope( (metrics) => async ( @@ -60,20 +71,19 @@ export const handler = metricScope( return; } - const { asset, packageVersionRevision } = await aws - .codeArtifact() - .getPackageVersionAsset({ + const { asset, packageVersionRevision } = await CODEARTIFACT_CLIENT.send( + new GetPackageVersionAssetCommand({ asset: 'package.tgz', // Always named this way for npm packages! domainOwner: event.detail.domainOwner, domain: event.detail.domainName, repository: event.detail.repositoryName, - format: event.detail.packageFormat, + format: event.detail.packageFormat as PackageFormat, // input is provided by EventBridge and should be correct, if not a hard fail seems fine namespace: event.detail.packageNamespace, package: event.detail.packageName, packageVersion: event.detail.packageVersion, }) - .promise(); - const tarball = Buffer.from(asset! as any); + ); + const tarball = Buffer.from(await asset!.transformToByteArray()); const { assemblyJson, packageJson } = await extractObjects(tarball, { assemblyJson: { path: `package/${SPEC_FILE_NAME}` }, @@ -113,9 +123,8 @@ export const handler = metricScope( } const stagingKey = `${packageName}/${event.detail.packageVersion}/${packageVersionRevision}/package.tgz`; - await aws - .s3() - .putObject({ + await S3_CLIENT.send( + new PutObjectCommand({ Bucket: stagingBucket, Key: stagingKey, Body: asset, @@ -126,7 +135,7 @@ export const handler = metricScope( 'Lambda-Run-Id': context.awsRequestId, }, }) - .promise(); + ); const message = integrity( { @@ -136,9 +145,8 @@ export const handler = metricScope( }, tarball ); - return aws - .sqs() - .sendMessage({ + return SQS_CLIENT.send( + new SendMessageCommand({ MessageAttributes: { AWS_REQUEST_ID: { DataType: 'String', @@ -156,7 +164,7 @@ export const handler = metricScope( MessageBody: JSON.stringify(message), QueueUrl: queueUrl, }) - .promise(); + ); } ); diff --git a/src/package-sources/npmjs/canary/npmjs-package-canary.lambda.ts b/src/package-sources/npmjs/canary/npmjs-package-canary.lambda.ts index d408357fe..bb9352d98 100644 --- a/src/package-sources/npmjs/canary/npmjs-package-canary.lambda.ts +++ b/src/package-sources/npmjs/canary/npmjs-package-canary.lambda.ts @@ -1,8 +1,12 @@ import * as https from 'https'; import { Readable } from 'stream'; import { createGunzip } from 'zlib'; +import { + GetObjectCommand, + NoSuchKey, + PutObjectCommand, +} from '@aws-sdk/client-s3'; import { metricScope, Configuration, Unit } from 'aws-embedded-metrics'; -import type { AWSError, S3 } from 'aws-sdk'; import * as JSONStream from 'JSONStream'; import { METRICS_NAMESPACE, @@ -11,7 +15,7 @@ import { ObjectKey, } from './constants'; import { CatalogModel } from '../../../backend'; -import * as aws from '../../../backend/shared/aws.lambda-shared'; +import { S3_CLIENT } from '../../../backend/shared/aws.lambda-shared'; import { requireEnv } from '../../../backend/shared/env.lambda-shared'; Configuration.namespace = METRICS_NAMESPACE; @@ -302,15 +306,14 @@ export class CanaryStateService { const url = this.url(packageName); console.log(`Saving to ${url}: ${JSON.stringify(state, null, 2)}`); - await aws - .s3() - .putObject({ + await S3_CLIENT.send( + new PutObjectCommand({ Bucket: this.bucketName, Key: this.key(packageName), Body: JSON.stringify(state, null, 2), ContentType: 'application/json', }) - .promise(); + ); } /** @@ -323,30 +326,30 @@ export class CanaryStateService { const url = this.url(packageName); console.log(`Fetching: ${url}`); - const data = await aws - .s3() - .getObject({ Bucket: this.bucketName, Key: objectKey }) - .promise() - .catch((err: AWSError) => - err.code !== 'NoSuchKey' - ? Promise.reject(err) - : Promise.resolve({ - /* no data */ - } as S3.GetObjectOutput) + try { + const res = await S3_CLIENT.send( + new GetObjectCommand({ Bucket: this.bucketName, Key: objectKey }) ); - if (!data?.Body) { - console.log(`Not found: ${url}`); - return undefined; - } + const content = await res?.Body?.transformToString('utf-8'); + if (!content) { + console.log(`Not found: ${url}`); + return undefined; + } - console.log(`Loaded: ${url}`); - return JSON.parse(data.Body.toString('utf-8'), (key, value) => { - if (key === 'publishedAt' || key === 'availableAt') { - return new Date(value); + console.log(`Loaded: ${url}`); + return JSON.parse(content, (key, value) => { + if (key === 'publishedAt' || key === 'availableAt') { + return new Date(value); + } + return value; + }); + } catch (error: any) { + if (error instanceof NoSuchKey || error.name === 'NoSuchKey') { + return undefined; } - return value; - }); + throw error; + } } /** diff --git a/src/package-sources/npmjs/npm-js-follower.lambda.ts b/src/package-sources/npmjs/npm-js-follower.lambda.ts index 1ad7f8866..8c8e1d212 100644 --- a/src/package-sources/npmjs/npm-js-follower.lambda.ts +++ b/src/package-sources/npmjs/npm-js-follower.lambda.ts @@ -1,6 +1,18 @@ import * as console from 'console'; -import { gunzipSync } from 'zlib'; +import { text } from 'node:stream/consumers'; +import { createGunzip } from 'zlib'; +import { InvokeCommand } from '@aws-sdk/client-lambda'; +import { + GetObjectCommand, + NoSuchKey, + PutObjectCommand, + PutObjectCommandInput, +} from '@aws-sdk/client-s3'; +import { + StreamingBlobPayloadInputTypes, + StreamingBlobPayloadOutputTypes, +} from '@smithy/types'; import { metricScope, Configuration, @@ -19,7 +31,10 @@ import { CouchChanges, DatabaseChange } from './couch-changes.lambda-shared'; import { PackageVersion } from './stage-and-notify.lambda'; import { DenyListClient } from '../../backend/deny-list/client.lambda-shared'; import { LicenseListClient } from '../../backend/license-list/client.lambda-shared'; -import * as aws from '../../backend/shared/aws.lambda-shared'; +import { + LAMBDA_CLIENT, + S3_CLIENT, +} from '../../backend/shared/aws.lambda-shared'; import { requireEnv } from '../../backend/shared/env.lambda-shared'; // eslint-disable-next-line @typescript-eslint/no-require-imports const normalizeNPMMetadata = require('normalize-registry-metadata'); @@ -161,13 +176,13 @@ export async function handler(event: ScheduledEvent, context: Context) { }; // "Fire-and-forget" invocation here. console.log(`Sending ${invokeArgs.tarballUrl} for staging`); - await aws - .lambda() - .invokeAsync({ + await LAMBDA_CLIENT.send( + new InvokeCommand({ FunctionName: stagingFunction, - InvokeArgs: JSON.stringify(invokeArgs, null, 2), + InvocationType: 'Event', + Payload: Buffer.from(JSON.stringify(invokeArgs)), }) - .promise(); + ); // Record that this is now a "known" version (no need to re-discover) knownVersions.set(`${infos.name}@${infos.version}`, modified); }) @@ -230,17 +245,17 @@ async function loadLastTransactionMarker( registry: CouchChanges ): Promise<{ marker: string | number; knownVersions: Map }> { try { - const response = await aws - .s3() - .getObject({ + const response = await S3_CLIENT.send( + new GetObjectCommand({ Bucket: stagingBucket, Key: MARKER_FILE_NAME, }) - .promise(); - if (response.ContentEncoding === 'gzip') { - response.Body = gunzipSync(Buffer.from(response.Body! as any)); + ); + if (!response.Body) { + throw new Error('Transaction Marker Response Body is empty'); } - let data = JSON.parse(response.Body!.toString('utf-8'), (key, value) => { + let body = await transformBody(response.Body, response.ContentEncoding); + let data = JSON.parse(body, (key, value) => { if (key !== 'knownVersions') { return value; } @@ -269,16 +284,33 @@ async function loadLastTransactionMarker( return data; } catch (error: any) { - if (error.code !== 'NoSuchKey') { - throw error; + if (error instanceof NoSuchKey || error.name === 'NoSuchKey') { + console.warn( + `Marker object (s3://${stagingBucket}/${MARKER_FILE_NAME}) does not exist, starting from scratch` + ); + return { marker: '0', knownVersions: new Map() }; } - console.warn( - `Marker object (s3://${stagingBucket}/${MARKER_FILE_NAME}) does not exist, starting from scratch` - ); - return { marker: '0', knownVersions: new Map() }; + // re-throw unexpected errors + throw error; } } +/** + * Helper function to transform a possibly gzip'ed blob stream into a string. + * @returns string + */ +async function transformBody( + body: StreamingBlobPayloadOutputTypes, + encoding?: string +): Promise { + if (encoding === 'gzip') { + const gunzip = createGunzip(); + return text(body.pipe(gunzip)); + } + + return body.transformToString('utf-8'); +} + /** * Updates the last transaction marker in S3. * @@ -330,12 +362,11 @@ function putObject( context: Context, bucket: string, key: string, - body: AWS.S3.Body, - opts: Omit = {} + body: StreamingBlobPayloadInputTypes, + opts: Omit = {} ) { - return aws - .s3() - .putObject({ + return S3_CLIENT.send( + new PutObjectCommand({ Bucket: bucket, Key: key, Body: body, @@ -347,7 +378,7 @@ function putObject( }, ...opts, }) - .promise(); + ); } //#endregion diff --git a/src/package-sources/npmjs/stage-and-notify.lambda.ts b/src/package-sources/npmjs/stage-and-notify.lambda.ts index b6908dd3f..dd1da6365 100644 --- a/src/package-sources/npmjs/stage-and-notify.lambda.ts +++ b/src/package-sources/npmjs/stage-and-notify.lambda.ts @@ -1,9 +1,11 @@ import * as https from 'https'; import { URL } from 'url'; +import { PutObjectCommand } from '@aws-sdk/client-s3'; +import { SendMessageCommand } from '@aws-sdk/client-sqs'; import type { Context, SQSEvent } from 'aws-lambda'; import { S3KeyPrefix } from './constants.lambda-shared'; import { DenyListClient } from '../../backend/deny-list/client.lambda-shared'; -import { s3, sqs } from '../../backend/shared/aws.lambda-shared'; +import { S3_CLIENT, SQS_CLIENT } from '../../backend/shared/aws.lambda-shared'; import { requireEnv } from '../../backend/shared/env.lambda-shared'; import { integrity } from '../../backend/shared/integrity.lambda-shared'; @@ -65,8 +67,8 @@ export async function handler( new URL(event.tarballUrl).pathname }`.replace(/\/{2,}/g, '/'); console.log(`Storing tarball in staging bucket with key ${stagingKey}`); - await s3() - .putObject({ + await S3_CLIENT.send( + new PutObjectCommand({ Bucket: stagingBucket, Key: stagingKey, Body: tarball, @@ -82,7 +84,7 @@ export async function handler( ...(event.seq ? { Sequence: event.seq } : {}), }, }) - .promise(); + ); // Prepare ingestion request const message = integrity( @@ -108,9 +110,9 @@ export async function handler( 2 )}` ); - await sqs() - .sendMessage({ - MessageBody: JSON.stringify(message, null, 2), + await SQS_CLIENT.send( + new SendMessageCommand({ + MessageBody: JSON.stringify(message), MessageAttributes: { 'Lambda-Log-Group': { DataType: 'String', @@ -127,7 +129,7 @@ export async function handler( }, QueueUrl: queueUrl, }) - .promise(); + ); } export interface PackageVersion {