Skip to content

Commit

Permalink
chore(cb2-12276): upgrade packages (#36)
Browse files Browse the repository at this point in the history
  • Loading branch information
Ruairidh-BJSS authored May 28, 2024
1 parent ab1d05b commit fb8795e
Show file tree
Hide file tree
Showing 13 changed files with 12,232 additions and 11,483 deletions.
23,526 changes: 12,148 additions & 11,378 deletions package-lock.json

Large diffs are not rendered by default.

87 changes: 40 additions & 47 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
"version": "1.1.0",
"description": "Gets, packages, and sends the EVL data file.",
"main": "index.js",
"engines": {
"node": "^18.*"
},
"scripts": {
"start": "sls offline start",
"build": "tsc --rootDir ./ --outDir .build --sourceMap false && npm run build:copy",
Expand All @@ -27,67 +30,57 @@
"predeploy": "npm install && npm run prepush",
"prepush": "npm run test && npm run build && npm run test-i"
},
"contributors": [
{
"name": "Shaun Hare",
"email": "[email protected]"
}
],
"license": "MIT",
"dependencies": {
"@aws-sdk/client-s3": "^3.565.0",
"@aws-sdk/client-secrets-manager": "^3.565.0",
"@types/md5": "^2.3.2",
"@types/ssh2-sftp-client": "^7.0.1",
"@types/tar": "^6.1.1",
"@aws-sdk/client-s3": "^3.577.0",
"@aws-sdk/client-secrets-manager": "^3.580.0",
"@types/md5": "^2.3.5",
"@types/ssh2-sftp-client": "^9.0.3",
"@types/tar": "^6.1.13",
"md5": "^2.3.0",
"ssh2-sftp-client": "^9.0.0",
"tar": "^6.1.11",
"winston": "^3.7.2"
"ssh2-sftp-client": "^10.0.3",
"tar": "^7.1.0",
"winston": "^3.13.0"
},
"devDependencies": {
"@commitlint/cli": "^17.0.2",
"@commitlint/config-conventional": "^17.0.2",
"@dvsa/eslint-config-ts": "^2.4.0",
"@types/aws-lambda": "^8.10.78",
"@types/jest": "^27.0.2",
"@types/node": "^16.10.2",
"@types/uuid": "^8.3.1",
"@typescript-eslint/eslint-plugin": "^5.27.1",
"@typescript-eslint/parser": "^5.27.1",
"archiver": "^5.3.0",
"@commitlint/cli": "^19.3.0",
"@commitlint/config-conventional": "^19.2.2",
"@dvsa/eslint-config-ts": "^3.0.1",
"@types/aws-lambda": "^8.10.138",
"@types/jest": "^29.5.12",
"@types/node": "^20.12.12",
"@types/uuid": "^9.0.8",
"@typescript-eslint/eslint-plugin": "^7.10.0",
"@typescript-eslint/parser": "^7.10.0",
"archiver": "^7.0.1",
"aws-sdk-client-mock": "^4.0.0",
"commitlint-plugin-function-rules": "^1.7.1",
"commitlint-plugin-function-rules": "^4.0.0",
"current-git-branch": "^1.1.0",
"dotenv": "^16.0.0",
"esbuild": "^0.14.43",
"eslint": "^7.32.0",
"eslint-config-airbnb-typescript": "^17.0.0",
"eslint-import-resolver-typescript": "^2.5.0",
"eslint-plugin-import": "^2.26.0",
"eslint-plugin-jest": "^26.5.3",
"eslint-plugin-security": "^1.5.0",
"fs-extra": "^10.0.0",
"husky": "^8.0.1",
"jest": "^27.2.4",
"dotenv": "^16.4.5",
"esbuild": "^0.21.3",
"eslint-config-airbnb-typescript": "^18.0.0",
"eslint-import-resolver-typescript": "^3.6.1",
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-jest": "^28.5.0",
"eslint-plugin-security": "^3.0.0",
"fs-extra": "^11.2.0",
"husky": "^9.0.11",
"jest": "^29.7.0",
"jest-sonar-reporter": "^2.0.0",
"prettier": "^2.6.2",
"serverless": "^3.19.0",
"serverless-offline": "^8.8.0",
"serverless-plugin-typescript": "^2.1.2",
"prettier": "^3.2.5",
"serverless": "^3.38.0",
"serverless-offline": "^13.6.0",
"serverless-plugin-typescript": "^2.1.5",
"serverless-s3-local": "^0.8.4",
"sonar-scanner": "^3.1.0",
"source-map-support": "^0.5.19",
"ts-jest": "^27.0.5",
"ts-loader": "^9.2.6",
"typescript": "^4.7.4"
"source-map-support": "^0.5.21",
"ts-jest": "^29.1.3",
"ts-loader": "^9.5.1",
"typescript": "^5.4.5"
},
"overrides": {
"got": "^11.8.5"
},
"engines": {
"node": "^18.*"
},
"jestSonar": {
"reportPath": ".reports",
"reportFile": "test-report.xml",
Expand Down
20 changes: 8 additions & 12 deletions src/fileConvert/fileConvert.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
/* eslint-disable security/detect-non-literal-fs-filename */
import md5 from 'md5';
import tar from 'tar';
import * as tar from 'tar';
import * as fs from 'fs';
import logger from '../util/logger';

Expand All @@ -26,23 +25,20 @@ export const configureEvlFile = async (
const textFilenamePrefix = 'crc32_';
const archiveNamePrefix = 'EVL_GVT_';
const dateFromFilename = filename.split('_')[2].split('.')[0];
const csvFilename = archiveNamePrefix + dateFromFilename + '.csv';
const textFilename = textFilenamePrefix + dateFromFilename + '.txt';
const archiveName =
workingDir + archiveNamePrefix + dateFromFilename + '.tar.gz';
const csvFilename = `${archiveNamePrefix + dateFromFilename}.csv`;
const textFilename = `${textFilenamePrefix + dateFromFilename}.txt`;
const archiveName = `${workingDir + archiveNamePrefix + dateFromFilename}.tar.gz`;

try {
const data = bufferInput.toString().split('\n');

if (data.length == 1 && data[0] === '') {
if (data.length === 1 && data[0] === '') {
throw new Error('No data provided');
}

const numberOfRecords = data.length.toString();
const headerLine =
recordInfoHeader + description + dateFileCreation + ', , ';
const trailerLine =
recordInfoTrailer + numberOfRecords + ' ' + vrmsExtracted + ', , ';
const headerLine = `${recordInfoHeader + description + dateFileCreation}, , `;
const trailerLine = `${recordInfoTrailer + numberOfRecords} ${vrmsExtracted}, , `;

data.splice(0, 0, headerLine);
data.splice(data.length, 0, trailerLine);
Expand All @@ -53,7 +49,7 @@ export const configureEvlFile = async (
logger.info('Written csv file');

const md5sum = md5(csvData);
const md5string = 'MD5 (' + csvFilename + ') = ' + md5sum;
const md5string = `MD5 (${csvFilename}) = ${md5sum}`;
fs.writeFileSync(workingDir + textFilename, md5string);

Check warning on line 53 in src/fileConvert/fileConvert.ts

View workflow job for this annotation

GitHub Actions / build-test (18.x)

Found writeFileSync from package "fs" with non literal argument at index 0
logger.info('Written txt checksum file');

Expand Down
3 changes: 2 additions & 1 deletion src/filePull/fromS3.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* eslint-disable @typescript-eslint/no-unsafe-return */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import logger from '../util/logger';
/* eslint-disable no-restricted-syntax */
import {
GetObjectCommand,
GetObjectCommandOutput,
Expand All @@ -10,6 +10,7 @@ import {
import { S3EventRecord } from 'aws-lambda';
import { Stream } from 'stream';
import { IncomingMessage } from 'http';
import logger from '../util/logger';

const s3Client = new S3Client(
(process.env.IS_LOCAL || process.env.IS_OFFLINE) && {
Expand Down
6 changes: 2 additions & 4 deletions src/filePush/filePush.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
/* eslint-disable security/detect-object-injection */
import Client from 'ssh2-sftp-client';
import logger from '../util/logger';
import path from 'path';
import logger from '../util/logger';
import { getSecret } from '../util/getSecret';

export interface Config {
Expand Down Expand Up @@ -45,8 +44,7 @@ export const filePush = async (filepath: string, eventType: Feeds) => {
evl: process.env.EVL_SFTP_PATH,
tfl: process.env.TFL_SFTP_PATH,
};
const remoteFileLocation =
(sftpPath[eventType] ?? '') + path.basename(filepath);
const remoteFileLocation = (sftpPath[eventType] ?? '') + path.basename(filepath);

Check warning on line 47 in src/filePush/filePush.ts

View workflow job for this annotation

GitHub Actions / build-test (18.x)

Generic Object Injection Sink

try {
logger.info('Attempt connection');
Expand Down
10 changes: 6 additions & 4 deletions src/handler/s3Event.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
/* eslint-disable security/detect-non-literal-fs-filename */
/* eslint-disable prefer-promise-reject-errors */
/* eslint-disable no-await-in-loop */
/* eslint-disable no-restricted-syntax */
import * as fs from 'fs';
import type { S3Event, S3EventRecord } from 'aws-lambda';
import { randomUUID } from 'crypto';
import { configureEvlFile } from '../fileConvert/fileConvert';
import { filePull } from '../filePull/fromS3';
import { filePush } from '../filePush/filePush';
import { randomUUID } from 'crypto';
import logger from '../util/logger';

const handleEvlEvent = async (record: S3EventRecord) => {
Expand Down Expand Up @@ -52,8 +54,8 @@ export const handler = async (event: S3Event): Promise<string> => {
if (fileName.startsWith('EVL_') && process.env.EVL_SFTP_SEND === 'true') {
await handleEvlEvent(record);
} else if (
fileName.startsWith('VOSA') &&
process.env.TFL_SFTP_SEND === 'true'
fileName.startsWith('VOSA')
&& process.env.TFL_SFTP_SEND === 'true'
) {
await handleTflEvent(record);
} else {
Expand Down
7 changes: 1 addition & 6 deletions src/util/getSecret.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,7 @@
/* eslint-disable @typescript-eslint/no-unsafe-return */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { SecretsManager } from '@aws-sdk/client-secrets-manager';

async function getSecret(secretName: string): Promise<string> {
const envVariable: string | undefined =
process.env[secretName.replace(/\//g, '-')];
const envVariable: string | undefined = process.env[secretName.replace(/\//g, '-')];
if (envVariable !== undefined) {
return envVariable.replace(/\\n/g, '\n');
}
Expand Down
10 changes: 5 additions & 5 deletions tests/fileConvert/fileConvert.test.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
/* eslint-disable security/detect-non-literal-fs-filename */
import { configureEvlFile } from '../../src/fileConvert/fileConvert';
import * as fs from 'fs';
import md5 from 'md5';
import { configureEvlFile } from '../../src/fileConvert/fileConvert';

describe('test the file config', () => {
const testFilename = 'tests/resources/EVL_GVT_20220621.csv';
const dateFromFilename = testFilename
.split('/')[2]
.split('_')[2]
.split('.')[0];
const txtFilename = 'crc32_' + dateFromFilename + '.txt';
const csvFilename = 'EVL_GVT_' + dateFromFilename + '.csv';
const finalFilename = 'EVL_GVT_' + dateFromFilename + '.tar.gz';
const txtFilename = `crc32_${dateFromFilename}.txt`;
const csvFilename = `EVL_GVT_${dateFromFilename}.csv`;
const finalFilename = `EVL_GVT_${dateFromFilename}.tar.gz`;

let buffer: Buffer;

Expand Down Expand Up @@ -55,7 +55,7 @@ describe('test the file config', () => {
await configureEvlFile('', buffer, 'EVL_GVT_20220621.csv');
const csvFile = fs.readFileSync(csvFilename);
const textFile = fs.readFileSync(txtFilename);
const hash = 'MD5 (EVL_GVT_20220621.csv) = ' + md5(csvFile);
const hash = `MD5 (EVL_GVT_20220621.csv) = ${md5(csvFile)}`;
expect(textFile.toString()).toBe(hash);
});
});
Expand Down
2 changes: 0 additions & 2 deletions tests/filePull/fromS3.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-explicit-any */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { S3Event } from 'aws-lambda';
Expand Down
33 changes: 14 additions & 19 deletions tests/filePush/filePush.test.ts
Original file line number Diff line number Diff line change
@@ -1,30 +1,25 @@
/* eslint-disable no-void */
import { filePush, createConfig, Config } from '../../src/filePush/filePush';

const mockConnect = jest.fn();
const mockFastPut = jest.fn();
const mockEnd = jest.fn();

jest.mock('ssh2-sftp-client', () => {
return {
__esModule: true,
default: jest.fn().mockImplementation(() => {
return {
connect: mockConnect,
fastPut: mockFastPut,
end: mockEnd,
};
}),
};
});
jest.mock('ssh2-sftp-client', () => ({
__esModule: true,
default: jest.fn().mockImplementation(() => ({
connect: mockConnect,
fastPut: mockFastPut,
end: mockEnd,
})),
}));

jest.mock('../../src/util/getSecret', () => ({
getSecret: jest.fn(() =>
Promise.resolve(
'{ "host" : "test", "username" : "user", "retries" : 3, "password": "psswrd" }',
),
),
getSecret: jest.fn(() => Promise.resolve(
'{ "host" : "test", "username" : "user", "retries" : 3, "password": "psswrd" }',
)),
}));

import { filePush, createConfig, Config } from '../../src/filePush/filePush';

describe('test the push to SFTP server', () => {
process.env.EVL_SFTP_CONFIG = 'mockString';
process.env.TFL_SFTP_CONFIG = 'mockString';
Expand Down
1 change: 1 addition & 0 deletions tests/handler/s3Event.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* eslint-disable no-void */
import type { S3Event } from 'aws-lambda';
import { mockClient } from 'aws-sdk-client-mock';
import { S3Client } from '@aws-sdk/client-s3';
Expand Down
1 change: 0 additions & 1 deletion tests/util/getSecrets.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
/* eslint-disable @typescript-eslint/no-unsafe-argument */
import {
GetSecretValueCommand,
SecretsManager,
Expand Down
9 changes: 5 additions & 4 deletions tests/util/logger.test.ts
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
/* eslint-disable import/first */
/* eslint-disable no-underscore-dangle */
/* eslint-disable @typescript-eslint/ban-ts-comment */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
import { EOL } from 'os';

process.env.LOG_LEVEL = 'debug';
// eslint-disable-next-line import/first
import logger from '../../src/util/logger';

describe('logger functions', () => {
it('GIVEN a logger WHEN an info is logged THEN the console message is correct.', () => {
// @ts-ignore
// @ts-expect-error
const consoleSpy = jest.spyOn(console._stdout, 'write');
logger.info('I am an info message!');
expect(consoleSpy).toHaveBeenCalledWith(
Expand All @@ -17,7 +18,7 @@ describe('logger functions', () => {
});

it('GIVEN a logger WHEN a debug is logged THEN the console message is correct.', () => {
// @ts-ignore
// @ts-expect-error
const consoleSpy = jest.spyOn(console._stdout, 'write');
logger.debug('I am a debug message!');
expect(consoleSpy).toHaveBeenCalledWith(
Expand All @@ -26,7 +27,7 @@ describe('logger functions', () => {
});

it('GIVEN a logger WHEN an error is logged THEN the console message is correct.', () => {
// @ts-ignore
// @ts-expect-error
const consoleSpy = jest.spyOn(console._stdout, 'write');
const error = new Error('AN ERROR');
logger.error('', error);
Expand Down

0 comments on commit fb8795e

Please sign in to comment.