chore(cb2-11292): upgrade aws sdk v2 to v3 (#32)

package.json

@@ -35,10 +35,11 @@
   ],
   "license": "MIT",
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.565.0",
+    "@aws-sdk/client-secrets-manager": "^3.565.0",
     "@types/md5": "^2.3.2",
     "@types/ssh2-sftp-client": "^7.0.1",
     "@types/tar": "^6.1.1",
-    "aws-sdk": "^2.1153.0",
     "md5": "^2.3.0",
     "ssh2-sftp-client": "^9.0.0",
     "tar": "^6.1.11",
@@ -55,6 +56,7 @@
     "@typescript-eslint/eslint-plugin": "^5.27.1",
     "@typescript-eslint/parser": "^5.27.1",
     "archiver": "^5.3.0",
+    "aws-sdk-client-mock": "^4.0.0",
     "commitlint-plugin-function-rules": "^1.7.1",
     "current-git-branch": "^1.1.0",
     "dotenv": "^16.0.0",
@@ -73,7 +75,7 @@
     "serverless": "^3.19.0",
     "serverless-offline": "^8.8.0",
     "serverless-plugin-typescript": "^2.1.2",
-    "serverless-s3-local": "^0.6.22",
+    "serverless-s3-local": "^0.8.4",
     "sonar-scanner": "^3.1.0",
     "source-map-support": "^0.5.19",
     "ts-jest": "^27.0.5",

src/filePull/fromS3.ts

@@ -1,13 +1,23 @@
+/* eslint-disable @typescript-eslint/no-unsafe-return */
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+/* eslint-disable @typescript-eslint/no-unsafe-assignment */
 import logger from '../util/logger';
-import { GetObjectOutput } from 'aws-sdk/clients/s3';
-import { S3 } from 'aws-sdk';
+import {
+  GetObjectCommand,
+  GetObjectCommandOutput,
+  S3Client,
+} from '@aws-sdk/client-s3';
 import { S3EventRecord } from 'aws-lambda';
+import { Stream } from 'stream';
+import { IncomingMessage } from 'http';
 
-const s3 = new S3(
+const s3Client = new S3Client(
   (process.env.IS_LOCAL || process.env.IS_OFFLINE) && {
-    s3ForcePathStyle: true,
-    accessKeyId: 'S3RVER',
-    secretAccessKey: 'S3RVER',
+    forcePathStyle: true,
+    credentials: {
+      accessKeyId: 'S3RVER',
+      secretAccessKey: 'S3RVER',
+    },
     endpoint: 'http://localhost:4569',
   },
 );
@@ -20,19 +30,24 @@ export const filePull = async (record: S3EventRecord) => {
     Key: key,
   };
   try {
-    const s3Object: GetObjectOutput = await s3.getObject(params).promise();
+    const s3Object: GetObjectCommandOutput = await s3Client.send(
+      new GetObjectCommand(params),
+    );
 
-    logger.debug(`s3Object: ${JSON.stringify(s3Object, null, 2)}`);
-    if (!Buffer.isBuffer(s3Object.Body)) {
-      throw new Error(
-        `Body of object with ETag ${s3Object.ETag} is not a Buffer.`,
-      );
+    const chunks: Buffer[] = [];
+    for await (const chunk of Stream.Readable.from(
+      s3Object.Body as IncomingMessage,
+    )) {
+      chunks.push(chunk as Buffer);
     }
+    const buffer = Buffer.concat(chunks);
+
+    logger.debug(`s3Object: ${JSON.stringify(cleanForLogging(s3Object))}`);
 
     logger.info(`${key} pulled successfully.`);
 
     return {
-      data: s3Object.Body,
+      data: buffer,
       filename: key,
     };
   } catch (err) {
@@ -43,3 +58,12 @@ export const filePull = async (record: S3EventRecord) => {
     throw err;
   }
 };
+
+const cleanForLogging = (input) => {
+  const retVal = { ...input };
+  retVal.Body = { redacted: true };
+  if (retVal.$response) {
+    delete retVal.$response;
+  }
+  return retVal;
+};

src/util/getSecret.ts

@@ -1,4 +1,8 @@
-import { SecretsManager } from 'aws-sdk';
+/* eslint-disable @typescript-eslint/no-unsafe-return */
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+/* eslint-disable @typescript-eslint/no-unsafe-call */
+/* eslint-disable @typescript-eslint/no-unsafe-assignment */
+import { SecretsManager } from '@aws-sdk/client-secrets-manager';
 
 async function getSecret(secretName: string): Promise<string> {
   const envVariable: string | undefined =
@@ -8,9 +12,9 @@ async function getSecret(secretName: string): Promise<string> {
   }
 
   const secretsManager = new SecretsManager();
-  const secretValue = await secretsManager
-    .getSecretValue({ SecretId: secretName })
-    .promise();
+  const secretValue = await secretsManager.getSecretValue({
+    SecretId: secretName,
+  });
   return secretValue.SecretString;
 }
 

tests/filePull/fromS3.test.ts

@@ -1,24 +1,21 @@
-const mockS3 = {
-  getObject: jest.fn().mockReturnThis(),
-  promise: jest.fn(),
-};
-
-jest.mock('aws-sdk', () => {
-  return { S3: jest.fn(() => mockS3) };
-});
-
-import event from '../resources/s3event.json';
-import type { S3Event } from 'aws-lambda';
-import { GetObjectOutput } from 'aws-sdk/clients/s3';
+/* eslint-disable @typescript-eslint/no-unsafe-call */
+/* eslint-disable @typescript-eslint/no-unsafe-member-access */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/* eslint-disable @typescript-eslint/no-unsafe-assignment */
+import { S3Event } from 'aws-lambda';
+import { mockClient } from 'aws-sdk-client-mock';
+import { GetObjectOutput, S3Client } from '@aws-sdk/client-s3';
 import { filePull } from '../../src/filePull/fromS3';
+import event from '../resources/s3event.json';
 
 describe('Test pull file from S3', () => {
   test('should return file content', async () => {
     const getObjectOutput: GetObjectOutput = {
       ContentType: 'text/csv',
-      Body: Buffer.from('File content'),
+      Body: Buffer.from('File content') as any,
     };
-    mockS3.promise.mockResolvedValueOnce(getObjectOutput);
+    mockClient(S3Client).resolves(getObjectOutput);
+
     const eventMock: S3Event = event as S3Event;
     const evlFileData = await filePull(eventMock.Records[0]);
 
@@ -29,20 +26,4 @@ describe('Test pull file from S3', () => {
 
     expect(evlFileData).toStrictEqual(expectedEvlFileData);
   });
-
-  test('should return error if body not a Buffer', async () => {
-    const getObjectOutput: GetObjectOutput = {
-      ContentType: 'text/csv',
-      ETag: 'c4c7b60167b533a5eae07b5ce38d7368',
-      Body: 'File content',
-    };
-    mockS3.promise.mockResolvedValueOnce(getObjectOutput);
-    const eventMock: S3Event = event as S3Event;
-
-    await expect(async () => {
-      await filePull(eventMock.Records[0]);
-    }).rejects.toThrow(
-      'Body of object with ETag c4c7b60167b533a5eae07b5ce38d7368 is not a Buffer.',
-    );
-  });
 });

tests/handler/s3Event.test.ts

@@ -1,36 +1,25 @@
-/* eslint-disable security/detect-non-literal-fs-filename */
-const mockS3 = {
-  getObject: jest.fn().mockReturnThis(),
-  promise: jest.fn(),
-};
-const mockConnect = jest.fn();
-const mockFastPut = jest.fn();
-const mockEnd = jest.fn();
-
-jest.mock('aws-sdk', () => {
-  return { S3: jest.fn(() => mockS3) };
-});
-
-jest.mock('ssh2-sftp-client', () => {
-  return {
-    __esModule: true,
-    default: jest.fn().mockImplementation(() => {
-      return {
-        connect: mockConnect,
-        fastPut: mockFastPut,
-        end: mockEnd,
-      };
-    }),
-  };
-});
-
+import type { S3Event } from 'aws-lambda';
+import { mockClient } from 'aws-sdk-client-mock';
+import { S3Client } from '@aws-sdk/client-s3';
 import event from '../resources/s3event.json';
 import eventTwo from '../resources/s3eventTwo.json';
-import type { S3Event } from 'aws-lambda';
-import { GetObjectOutput } from 'aws-sdk/clients/s3';
 import { handler } from '../../src/handler/s3Event';
 import * as filePush from '../../src/filePush/filePush';
 
+const mockConnect = jest.fn();
+const mockFastPut = jest.fn();
+const mockEnd = jest.fn();
+const mockGetObjectCommand = jest.fn();
+
+jest.mock('ssh2-sftp-client', () => ({
+  __esModule: true,
+  default: jest.fn().mockImplementation(() => ({
+    connect: mockConnect,
+    fastPut: mockFastPut,
+    end: mockEnd,
+  })),
+}));
+
 describe('Test S3 Event Lambda Function', () => {
   test('should return 204', async () => {
     jest.spyOn(filePush, 'createConfig').mockImplementation(() => {
@@ -45,11 +34,11 @@ describe('Test S3 Event Lambda Function', () => {
     mockConnect.mockReturnValue(Promise.resolve(true));
     mockFastPut.mockReturnValue(Promise.resolve('uploaded'));
     mockEnd.mockReturnValue(Promise.resolve(void 0));
-    const getObjectOutput: GetObjectOutput = {
+    const getObjectOutput = {
       ContentType: 'text/csv',
       Body: Buffer.from('File content'),
     };
-    mockS3.promise.mockResolvedValue(getObjectOutput);
+    mockClient(S3Client).resolves(getObjectOutput);
     const eventMock: S3Event = event as S3Event;
 
     const res: string = await handler(eventMock);
@@ -70,11 +59,11 @@ describe('Test S3 Event Lambda Function', () => {
     mockConnect.mockReturnValue(Promise.resolve(true));
     mockFastPut.mockReturnValue(Promise.resolve('uploaded'));
     mockEnd.mockReturnValue(Promise.resolve(void 0));
-    const getObjectOutput: GetObjectOutput = {
+    const getObjectOutput = {
       ContentType: 'text/csv',
       Body: Buffer.from('File content'),
     };
-    mockS3.promise.mockResolvedValue(getObjectOutput);
+    mockClient(S3Client).resolves(getObjectOutput);
     const eventMock: S3Event = event as S3Event;
     eventMock.Records[0].s3.object.key = 'VOSA_1235.csv';
     process.env.TFL_SFTP_SEND = 'true';
@@ -97,11 +86,11 @@ describe('Test S3 Event Lambda Function', () => {
     mockConnect.mockReturnValue(Promise.resolve(true));
     mockFastPut.mockReturnValue(Promise.resolve('uploaded'));
     mockEnd.mockReturnValue(Promise.resolve(void 0));
-    const getObjectOutput: GetObjectOutput = {
+    const getObjectOutput = {
       ContentType: 'text/csv',
       Body: Buffer.from('File content'),
     };
-    mockS3.promise.mockResolvedValue(getObjectOutput);
+    mockClient(S3Client).resolves(getObjectOutput);
     const eventMock: S3Event = eventTwo as S3Event;
 
     const res: string = await handler(eventMock);
@@ -123,16 +112,16 @@ describe('Test S3 Event Lambda Function', () => {
     mockConnect.mockReturnValue(Promise.resolve(true));
     mockFastPut.mockReturnValue(Promise.resolve('uploaded'));
     mockEnd.mockReturnValue(Promise.resolve(void 0));
-    const getObjectOutput: GetObjectOutput = {
+    const getObjectOutput = {
       ContentType: 'text/csv',
       Body: Buffer.from('File content'),
     };
-    const getObjectOutputBroken: GetObjectOutput = {
+    const getObjectOutputBroken = {
       ContentType: 'text',
       Body: Buffer.from('File content'),
     };
-    mockS3.promise.mockResolvedValueOnce(getObjectOutput);
-    mockS3.promise.mockResolvedValueOnce(getObjectOutputBroken);
+    mockGetObjectCommand.mockResolvedValueOnce(getObjectOutput);
+    mockGetObjectCommand.mockResolvedValueOnce(getObjectOutputBroken);
     const eventMock: S3Event = eventTwo as S3Event;
 
     await expect(handler(eventMock)).rejects.toBe(

tests/util/getSecrets.test.ts

@@ -1,19 +1,15 @@
+/* eslint-disable @typescript-eslint/no-unsafe-argument */
+import {
+  GetSecretValueCommand,
+  SecretsManager,
+} from '@aws-sdk/client-secrets-manager';
+import { mockClient } from 'aws-sdk-client-mock';
 import { getSecret } from '../../src/util/getSecret';
 
-jest.mock('aws-sdk', () => {
-  const mSecretsManagerInstance = {
-    getSecretValue: () => {
-      return {
-        promise: jest
-          .fn()
-          .mockReturnValue({ SecretString: 'Secret From Secrets Manager' }),
-      };
-    },
-  };
-  const mSecretsManager = jest.fn(() => mSecretsManagerInstance);
-
-  return { SecretsManager: mSecretsManager };
-});
+const mockSecretsManager = mockClient(SecretsManager);
+mockSecretsManager
+  .on(GetSecretValueCommand)
+  .resolves({ SecretString: 'Secret From Secrets Manager' });
 
 describe('getSecret functions', () => {
   it('GIVEN a variable WHEN variable not local THEN get variable from Secrets Manager.', async () => {