diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9e6b67e1ae..9598810657 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,7 +55,7 @@ jobs: run: npm ci - name: Extract branch name shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + run: echo "branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT id: extract_branch - name: Run versioning script run: . ./scripts/github/deploy.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index f32636adbb..1e649f1ca3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,66 @@ +## v1.55.0-rc.6 (2024-11-25) + +#### :rocket: Enhancement +* `fulfilment-job` + * [#2070](https://github.com/DEFRA/rod-licensing/pull/2070) Remove ftp image and build ([@ScottDormand96](https://github.com/ScottDormand96)) + +#### Committers: 1 +- Scott Dormand ([@ScottDormand96](https://github.com/ScottDormand96)) + +## v1.55.0-rc.5 (2024-11-25) + +#### :rocket: Enhancement +* `fulfilment-job`, `pocl-job` + * [#2064](https://github.com/DEFRA/rod-licensing/pull/2064) Remove ftp functionality ([@ScottDormand96](https://github.com/ScottDormand96)) + +#### Committers: 1 +- Scott Dormand ([@ScottDormand96](https://github.com/ScottDormand96)) + +## v1.55.0-rc.4 (2024-11-21) + +#### :rocket: Enhancement +* `connectors-lib`, `gafl-webapp-service`, `sales-api-service` + * [#2074](https://github.com/DEFRA/rod-licensing/pull/2074) Add agreement_id to existing GOV.UK Pay request ([@irisfaraway](https://github.com/irisfaraway)) + +#### Committers: 1 +- Iris Faraway ([@irisfaraway](https://github.com/irisfaraway)) + +## v1.55.0-rc.3 (2024-11-21) + +#### :rocket: Enhancement +* [#2078](https://github.com/DEFRA/rod-licensing/pull/2078) Set-output removed from versioning script ([@ScottDormand96](https://github.com/ScottDormand96)) + +#### Committers: 1 +- Scott Dormand ([@ScottDormand96](https://github.com/ScottDormand96)) + +## v1.55.0-rc.2 (2024-11-19) + +#### :rocket: Enhancement +* `gafl-webapp-service` + * [#2077](https://github.com/DEFRA/rod-licensing/pull/2077) Change links for three rod licence ([@ScottDormand96](https://github.com/ScottDormand96)) + +#### Committers: 1 +- Scott Dormand ([@ScottDormand96](https://github.com/ScottDormand96)) + +## v1.55.0-rc.1 (2024-11-19) + +#### :rocket: Enhancement +* `gafl-webapp-service` + * [#2079](https://github.com/DEFRA/rod-licensing/pull/2079) Licence summary licence type capitals ([@ScottDormand96](https://github.com/ScottDormand96)) + +#### Committers: 1 +- Scott Dormand ([@ScottDormand96](https://github.com/ScottDormand96)) + +## v1.55.0-rc.0 (2024-11-13) + +#### :rocket: Enhancement +* `dynamics-lib` + * [#2072](https://github.com/DEFRA/rod-licensing/pull/2072) Create activity in CRM ([@nabeelamir-defra](https://github.com/nabeelamir-defra)) + +#### Committers: 1 +- Nabeel Amir ([@nabeelamir-defra](https://github.com/nabeelamir-defra)) + diff --git a/docker/env/fulfilment_job.env.example b/docker/env/fulfilment_job.env.example index 36356b4102..a616195603 100644 --- a/docker/env/fulfilment_job.env.example +++ b/docker/env/fulfilment_job.env.example @@ -20,11 +20,6 @@ FULFILMENT_S3_BUCKET=fulfilment-audit FULFILMENT_FILE_SIZE=5000 # FTP Settings -FULFILMENT_FTP_HOST=host.docker.internal -FULFILMENT_FTP_PORT=2222 -FULFILMENT_FTP_PATH=/share/fulfilment -FULFILMENT_FTP_USERNAME=test_sftp_user -FULFILMENT_FTP_KEY_SECRET_ID=/dev/fsh/local/sftp/ssh_ed25519_host_key FULFILMENT_SEND_UNENCRYPTED_FILE=true FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID=/dev/fsh/local/ diff --git a/docker/env/gafl_webapp.secrets.env.example b/docker/env/gafl_webapp.secrets.env.example index 4b8818c26c..4075008506 100644 --- a/docker/env/gafl_webapp.secrets.env.example +++ b/docker/env/gafl_webapp.secrets.env.example @@ -3,5 +3,6 @@ #################################################################################### GOV_PAY_APIKEY= +GOV_PAY_RECURRING_APIKEY= SESSION_COOKIE_PASSWORD= ANALYTICS_PROPERTY_API= diff --git a/docker/infrastructure.yml b/docker/infrastructure.yml index 89d387f4fb..acaf6a6ec2 100644 --- a/docker/infrastructure.yml +++ b/docker/infrastructure.yml @@ -107,30 +107,3 @@ services: deploy: restart_policy: condition: on-failure - - ####################################################### - # Test SFTP server - ####################################################### - ftp: - image: rod_licensing/ftp:${TAG:-latest} - build: - dockerfile: Dockerfile - context: ./resources/infrastructure/sftp - depends_on: - - localstack - ports: - - '2222:22' - volumes: - - ./volumes/sftp:/home/test_sftp_user/share - environment: - SFTP_USER: test_sftp_user - SFTP_FOLDERS: share/pocl;share/fulfilment - AWS_SECRETSMANAGER_ENDPOINT: http://host.docker.internal:4566 - AWS_DEFAULT_REGION: eu-west-2 - AWS_ACCESS_KEY_ID: local - AWS_SECRET_ACCESS_KEY: local - SSH_HOST_ED25519_SECRET_ID: /dev/fsh/local/sftp/ssh_ed25519_host_key - SSH_HOST_RSA_SECRET_ID: /dev/fsh/local/sftp/ssh_rsa_host_key - deploy: - restart_policy: - condition: on-failure diff --git a/docker/resources/infrastructure/sftp/Dockerfile b/docker/resources/infrastructure/sftp/Dockerfile deleted file mode 100644 index 2c9f4ff96b..0000000000 --- a/docker/resources/infrastructure/sftp/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM alpine:latest - -RUN apk --no-cache -U -v upgrade \ - && apk --no-cache -U -v add bash openssh openssh-sftp-server aws-cli \ - && mkdir -p /var/run/sshd \ - && rm -f /etc/ssh/ssh_host_*key* - -COPY ./files/sshd_config /etc/ssh/sshd_config -COPY ./files/entrypoint.sh / -EXPOSE 22 -ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/resources/infrastructure/sftp/files/entrypoint.sh b/docker/resources/infrastructure/sftp/files/entrypoint.sh deleted file mode 100755 index 7e853335a0..0000000000 --- a/docker/resources/infrastructure/sftp/files/entrypoint.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -############################################################################### -# SFTP Initialisation script -############################################################################### -set -e -trap 'exit 1' INT - -ED25519_KEY_PATH="/etc/ssh/ssh_host_ed25519_key" -RSA_KEY_PATH="/etc/ssh/ssh_host_rsa_key" -SFTP_USER=${SFTP_USER:=test} -SFTP_FOLDERS=${SFTP_FOLDERS:=share} - -AWS_CLI_ARGS=() -if [ -n "${AWS_SECRETSMANAGER_ENDPOINT}" ]; then - AWS_CLI_ARGS+=('--endpoint' "${AWS_SECRETSMANAGER_ENDPOINT}") -fi - -######################### -# Create SSH keys -######################### -if [ -n "${SSH_HOST_ED25519_SECRET_ID}" ]; then - echo "Retrieving SSH_HOST_ED25519_KEY from aws secrets manager" - aws "${AWS_CLI_ARGS[@]}" secretsmanager get-secret-value --secret-id "${SSH_HOST_ED25519_SECRET_ID}" --query SecretString --output text | (umask 177; cat > "${ED25519_KEY_PATH}") -elif [ -n "${SSH_HOST_ED25519_KEY}" ]; then - echo "Using SSH_HOST_ED25519_KEY defined in environment" - echo "${SSH_HOST_ED25519_KEY}" | (umask 177; cat > "${ED25519_KEY_PATH}") -else - echo "Generating new SSH_HOST_ED25519_KEY" - ssh-keygen -t ed25519 -f "${ED25519_KEY_PATH}" -N '' - cat "${ED25519_KEY_PATH}" -fi -ssh-keygen -y -f "${ED25519_KEY_PATH}" > "${ED25519_KEY_PATH}.pub" -ssh-keygen -lvf "${ED25519_KEY_PATH}" - -if [ -n "${SSH_HOST_RSA_SECRET_ID}" ]; then - echo "Retrieving SSH_HOST_RSA_SECRET_ID from aws secrets manager" - aws "${AWS_CLI_ARGS[@]}" secretsmanager get-secret-value --secret-id "${SSH_HOST_RSA_SECRET_ID}" --query SecretString --output text | (umask 177; cat > "${RSA_KEY_PATH}") -elif [ -n "${SSH_HOST_RSA_KEY}" ]; then - echo "Using SSH_HOST_RSA_KEY defined in environment" - echo "${SSH_HOST_RSA_KEY}" | (umask 177; cat > "${RSA_KEY_PATH}") -else - echo "Generating new SSH_HOST_RSA_KEY" - ssh-keygen -t rsa -b 4096 -f "${RSA_KEY_PATH}" -N '' - cat "${RSA_KEY_PATH}" -fi -ssh-keygen -y -f "${RSA_KEY_PATH}" > "${RSA_KEY_PATH}.pub" -ssh-keygen -lvf "${RSA_KEY_PATH}" - -######################### -# Create test user -######################### -echo "Creating user ${SFTP_USER} with random password" -adduser "${SFTP_USER}" > /dev/null 2>&1 || true -echo "${SFTP_USER}:$(base64 /dev/urandom | tr -d '/+' | fold -w 32 | head -n1)" | chpasswd -e > /dev/null 2>&1 - -######################### -# Add authorised keys -######################### -echo "Adding authorised keys" -mkdir -p "/home/${SFTP_USER}/.ssh/keys/" -cp "${ED25519_KEY_PATH}.pub" "/home/${SFTP_USER}/.ssh/keys/id_ed25519.pub" -cp "${RSA_KEY_PATH}.pub" "/home/${SFTP_USER}/.ssh/keys/id_rsa.pub" -for publickey in "/home/${SFTP_USER}/.ssh/keys"/*; do - cat "${publickey}" >> "/home/${SFTP_USER}/.ssh/authorized_keys" -done - - -######################### -# Create default folders -######################### -IFS=';' -read -ra FOLDERS <<< "${SFTP_FOLDERS}" -for folder in "${FOLDERS[@]}"; do - echo "Creating folder /home/${SFTP_USER}/${folder}" - mkdir -p "/home/${SFTP_USER}/${folder}" -done - -######################### -# Set permissions -######################### -chown -R "${SFTP_USER}" "/home/${SFTP_USER}/" -chown root:root "/home/${SFTP_USER}" -chmod 755 "/home/${SFTP_USER}" - -exec /usr/sbin/sshd -D -e diff --git a/docker/resources/infrastructure/sftp/files/sshd_config b/docker/resources/infrastructure/sftp/files/sshd_config deleted file mode 100644 index 96e7861832..0000000000 --- a/docker/resources/infrastructure/sftp/files/sshd_config +++ /dev/null @@ -1,24 +0,0 @@ -# Secure defaults -Protocol 2 -HostKey /etc/ssh/ssh_host_ed25519_key -HostKey /etc/ssh/ssh_host_rsa_key - -# IPv4 Only -AddressFamily inet - -# Faster connection -UseDNS no - -# Limit access -PermitRootLogin no -X11Forwarding no -AllowTcpForwarding no -PasswordAuthentication no - -# Force sftp and chroot jail -Subsystem sftp internal-sftp -ForceCommand internal-sftp -ChrootDirectory %h - -# Enable this for more logs -LogLevel VERBOSE diff --git a/docker/volumes/sftp/fulfilment/README.md b/docker/volumes/sftp/fulfilment/README.md deleted file mode 100644 index c919ee9d42..0000000000 --- a/docker/volumes/sftp/fulfilment/README.md +++ /dev/null @@ -1 +0,0 @@ -> Used to provide a remote FTP server folder for fulfilment - do not remove diff --git a/docker/volumes/sftp/pocl/README.md b/docker/volumes/sftp/pocl/README.md deleted file mode 100644 index fa717a9d65..0000000000 --- a/docker/volumes/sftp/pocl/README.md +++ /dev/null @@ -1 +0,0 @@ -> Used to provide a remote FTP server folder for POCL - do not remove diff --git a/packages/connectors-lib/README.md b/packages/connectors-lib/README.md index bd2c7dd54c..40a6b3ef18 100644 --- a/packages/connectors-lib/README.md +++ b/packages/connectors-lib/README.md @@ -18,6 +18,7 @@ Provides connectivity to the resources/infrastructure used in the rod licensing | SALES_API_TIMEOUT_MS | Request timeout for the requests to the sales API | no | 20000 (20s) | | | | GOV_PAY_API_URL | The GOV.UK Pay API base url | yes | | | | | GOV_PAY_APIKEY | GOV pay access identifier | yes | | | | +| GOV_PAY_RECURRING_APIKEY | GOV pay access identifier for recurring payments | yes | | | | | GOV_PAY_REQUEST_TIMEOUT_MS | Timeout in milliseconds for API requests | no | 10000 | | | | GOV_PAY_RCP_API_URL | The GOV.UK Pay API url for agreements | yes | | diff --git a/packages/connectors-lib/src/__tests__/govuk-pay-api.spec.js b/packages/connectors-lib/src/__tests__/govuk-pay-api.spec.js index de26640df6..43f4b3e085 100644 --- a/packages/connectors-lib/src/__tests__/govuk-pay-api.spec.js +++ b/packages/connectors-lib/src/__tests__/govuk-pay-api.spec.js @@ -5,6 +5,7 @@ const fetch = require('node-fetch') process.env.GOV_PAY_API_URL = 'http://0.0.0.0/payment' process.env.GOV_PAY_RCP_API_URL = 'http://0.0.0.0/agreement' process.env.GOV_PAY_APIKEY = 'key' +process.env.GOV_PAY_RECURRING_APIKEY = 'recurringkey' const headers = { accept: 'application/json', @@ -12,6 +13,12 @@ const headers = { 'content-type': 'application/json' } +const recurringHeaders = { + accept: 'application/json', + authorization: `Bearer ${process.env.GOV_PAY_RECURRING_APIKEY}`, + 'content-type': 'application/json' +} + describe('govuk-pay-api-connector', () => { beforeEach(jest.clearAllMocks) @@ -41,6 +48,17 @@ describe('govuk-pay-api-connector', () => { }) expect(consoleErrorSpy).toHaveBeenCalled() }) + + it('uses the correct API key if recurring arg is set to true', async () => { + fetch.mockReturnValue({ ok: true, status: 200 }) + await expect(govUkPayApi.createPayment({ cost: 0 }, true)).resolves.toEqual({ ok: true, status: 200 }) + expect(fetch).toHaveBeenCalledWith('http://0.0.0.0/payment', { + body: JSON.stringify({ cost: 0 }), + headers: recurringHeaders, + method: 'post', + timeout: 10000 + }) + }) }) describe('fetchPaymentStatus', () => { @@ -63,6 +81,16 @@ describe('govuk-pay-api-connector', () => { expect(fetch).toHaveBeenCalledWith('http://0.0.0.0/payment/123', { headers, method: 'get', timeout: 10000 }) expect(consoleErrorSpy).toHaveBeenCalled() }) + + it('uses the correct API key if recurring arg is set to true', async () => { + fetch.mockReturnValue({ ok: true, status: 200, json: () => {} }) + await expect(govUkPayApi.fetchPaymentStatus(123, true)).resolves.toEqual(expect.objectContaining({ ok: true, status: 200 })) + expect(fetch).toHaveBeenCalledWith('http://0.0.0.0/payment/123', { + headers: recurringHeaders, + method: 'get', + timeout: 10000 + }) + }) }) describe('fetchPaymentEvents', () => { @@ -80,6 +108,16 @@ describe('govuk-pay-api-connector', () => { await expect(govUkPayApi.fetchPaymentEvents(123)).rejects.toEqual(Error('test event error')) expect(consoleErrorSpy).toHaveBeenCalled() }) + + it('uses the correct API key if recurring arg is set to true', async () => { + fetch.mockReturnValue({ ok: true, status: 200, json: () => {} }) + await expect(govUkPayApi.fetchPaymentEvents(123, true)).resolves.toEqual(expect.objectContaining({ ok: true, status: 200 })) + expect(fetch).toHaveBeenCalledWith('http://0.0.0.0/payment/123/events', { + headers: recurringHeaders, + method: 'get', + timeout: 10000 + }) + }) }) describe('createRecurringPayment', () => { @@ -88,7 +126,7 @@ describe('govuk-pay-api-connector', () => { await expect(govUkPayApi.createRecurringPayment({ cost: 0 })).resolves.toEqual({ ok: true, status: 200 }) expect(fetch).toHaveBeenCalledWith('http://0.0.0.0/agreement', { body: JSON.stringify({ cost: 0 }), - headers, + headers: recurringHeaders, method: 'post', timeout: 10000 }) @@ -102,7 +140,7 @@ describe('govuk-pay-api-connector', () => { expect(govUkPayApi.createRecurringPayment({ reference: '123' })).rejects.toEqual(Error('')) expect(fetch).toHaveBeenCalledWith('http://0.0.0.0/agreement', { body: JSON.stringify({ reference: '123' }), - headers, + headers: recurringHeaders, method: 'post', timeout: 10000 }) diff --git a/packages/connectors-lib/src/govuk-pay-api.js b/packages/connectors-lib/src/govuk-pay-api.js index 6bd913d897..16657c2135 100644 --- a/packages/connectors-lib/src/govuk-pay-api.js +++ b/packages/connectors-lib/src/govuk-pay-api.js @@ -4,9 +4,9 @@ import fetch from 'node-fetch' const GOV_PAY_REQUEST_TIMEOUT_MS_DEFAULT = 10000 -const headers = () => ({ +const headers = recurring => ({ accept: 'application/json', - authorization: `Bearer ${process.env.GOV_PAY_APIKEY}`, + authorization: `Bearer ${recurring ? process.env.GOV_PAY_RECURRING_APIKEY : process.env.GOV_PAY_APIKEY}`, 'content-type': 'application/json' }) @@ -18,7 +18,7 @@ const headers = () => ({ export const createRecurringPayment = async preparedPayment => { try { return fetch(process.env.GOV_PAY_RCP_API_URL, { - headers: headers(), + headers: headers(true), method: 'post', body: JSON.stringify(preparedPayment), timeout: process.env.GOV_PAY_REQUEST_TIMEOUT_MS || GOV_PAY_REQUEST_TIMEOUT_MS_DEFAULT @@ -37,10 +37,10 @@ export const createRecurringPayment = async preparedPayment => { * @param preparedPayment - see the GOV.UK pay API reference for details * @returns {Promise<*>} */ -export const createPayment = async preparedPayment => { +export const createPayment = async (preparedPayment, recurring = false) => { try { return fetch(process.env.GOV_PAY_API_URL, { - headers: headers(), + headers: headers(recurring), method: 'post', body: JSON.stringify(preparedPayment), timeout: process.env.GOV_PAY_REQUEST_TIMEOUT_MS || GOV_PAY_REQUEST_TIMEOUT_MS_DEFAULT @@ -56,10 +56,10 @@ export const createPayment = async preparedPayment => { * @param paymentId * @returns {Promise} */ -export const fetchPaymentStatus = async paymentId => { +export const fetchPaymentStatus = async (paymentId, recurring = false) => { try { return fetch(`${process.env.GOV_PAY_API_URL}/${paymentId}`, { - headers: headers(), + headers: headers(recurring), method: 'get', timeout: process.env.GOV_PAY_REQUEST_TIMEOUT_MS || GOV_PAY_REQUEST_TIMEOUT_MS_DEFAULT }) @@ -74,10 +74,10 @@ export const fetchPaymentStatus = async paymentId => { * @param paymentId * @returns {Promise} */ -export const fetchPaymentEvents = async paymentId => { +export const fetchPaymentEvents = async (paymentId, recurring = false) => { try { return fetch(`${process.env.GOV_PAY_API_URL}/${paymentId}/events`, { - headers: headers(), + headers: headers(recurring), method: 'get', timeout: process.env.GOV_PAY_REQUEST_TIMEOUT_MS || GOV_PAY_REQUEST_TIMEOUT_MS_DEFAULT }) diff --git a/packages/dynamics-lib/src/index.js b/packages/dynamics-lib/src/index.js index a55c7a1cdb..e896f46651 100644 --- a/packages/dynamics-lib/src/index.js +++ b/packages/dynamics-lib/src/index.js @@ -28,6 +28,7 @@ export * from './queries/concession-proof.queries.js' export * from './queries/pocl-validation-error.queries.js' export * from './queries/recurring-payments.queries.js' export * from './queries/contact.queries.js' +export * from './queries/activity.queries.js' // Framework functionality export * from './client/util.js' diff --git a/packages/dynamics-lib/src/queries/__tests__/activity.queries.spec.js b/packages/dynamics-lib/src/queries/__tests__/activity.queries.spec.js new file mode 100644 index 0000000000..66290b415a --- /dev/null +++ b/packages/dynamics-lib/src/queries/__tests__/activity.queries.spec.js @@ -0,0 +1,72 @@ +import { createActivity } from '../activity.queries.js' +import { dynamicsClient } from '../../client/dynamics-client.js' + +jest.mock('dynamics-web-api', () => { + return jest.fn().mockImplementation(() => { + return { + executeUnboundAction: jest.fn() + } + }) +}) + +describe('Activity Service', () => { + describe('createActivity', () => { + const mockResponse = { + '@odata.context': 'https://dynamics.com/api/data/v9.1/defra_CreateRCRActivityResponse', + RCRActivityId: 'abc123', + ReturnStatus: 'success', + SuccessMessage: 'RCR Activity - created successfully', + ErrorMessage: null, + oDataContext: 'https://dynamics.com/api/data/v9.1/defra_CreateRCRActivityResponse' + } + + const errorResponse = { + '@odata.context': 'https://dynamics.com/api/data/v9.1/.defra_CreateRCRActivityResponse', + RCRActivityId: null, + ReturnStatus: 'error', + SuccessMessage: '', + ErrorMessage: 'Failed to create activity', + oDataContext: 'https://dynamics.com/api/data/v9.1/$metadata#Microsoft.Dynamics.CRM.defra_CreateRCRActivityResponse' + } + + it('should call dynamicsClient with correct parameters', async () => { + dynamicsClient.executeUnboundAction.mockResolvedValue(mockResponse) + + await createActivity('contact-identifier-123', 2023) + + expect(dynamicsClient.executeUnboundAction).toHaveBeenCalledWith('defra_CreateRCRActivity', { + ContactId: 'contact-identifier-123', + ActivityStatus: 'STARTED', + Season: 2023 + }) + }) + + it('should return the CRM response correctly', async () => { + dynamicsClient.executeUnboundAction.mockResolvedValue(mockResponse) + + const result = await createActivity('contact-identifier-123', 2024) + + expect(result).toEqual(mockResponse) + }) + + it('should handle error in dynamicsClient response', async () => { + const error = new Error('Failed to create activity') + dynamicsClient.executeUnboundAction.mockRejectedValue(error) + + await expect(createActivity('contact-identifier-123', 2024)).rejects.toThrow('Failed to create activity') + }) + + it('should handle the case where activity creation fails', async () => { + dynamicsClient.executeUnboundAction.mockResolvedValue(errorResponse) + + const result = await createActivity('invalid-contact-id', 2024) + + expect(result).toMatchObject({ + RCRActivityId: null, + ReturnStatus: 'error', + SuccessMessage: '', + ErrorMessage: 'Failed to create activity' + }) + }) + }) +}) diff --git a/packages/dynamics-lib/src/queries/activity.queries.js b/packages/dynamics-lib/src/queries/activity.queries.js new file mode 100644 index 0000000000..3a125c6f4f --- /dev/null +++ b/packages/dynamics-lib/src/queries/activity.queries.js @@ -0,0 +1,24 @@ +import { dynamicsClient } from '../client/dynamics-client.js' + +/** + * Creates an RCR Activity in Microsoft Dynamics CRM. + * + * @param {string} contactId - The ID of the contact associated with the activity. + * @param {number} season - The season year for which the activity is being created. + * @returns {Promise} - A promise that resolves to the response from Dynamics CRM. + * @property {string} response.@odata.context - The OData context URL of the response. + * @property {string} response.RCRActivityId - The unique identifier of the created RCR activity. + * @property {string} response.ReturnStatus - The status of the activity creation operation (e.g., 'success'). + * @property {string} response.SuccessMessage - A message indicating successful creation of the activity. + * @property {string|null} response.ErrorMessage - An error message if the activity creation failed, otherwise null. + * @property {string} response.oDataContext - The OData context URL of the response. + */ +export const createActivity = (contactId, season) => { + const request = { + ContactId: contactId, + ActivityStatus: 'STARTED', + Season: season + } + + return dynamicsClient.executeUnboundAction('defra_CreateRCRActivity', request) +} diff --git a/packages/fulfilment-job/README.md b/packages/fulfilment-job/README.md index 566f9b5ff7..393cc1eca7 100644 --- a/packages/fulfilment-job/README.md +++ b/packages/fulfilment-job/README.md @@ -19,21 +19,16 @@ provider. # Environment variables -| name | description | required | default | valid | notes | -| ----------------------------------- | ----------------------------------------------------------------------------------------- | :------: | ------- | ----------------------------------------------------------------------- | ----- | -| NODE_ENV | Node environment | no | | development, test, production | | -| FULFILMENT_FILE_SIZE | The maximum number of records written to an aggregated fulfilment file | yes | | | | -| FULFILMENT_FTP_HOST | The hostname of the target FTP server | yes | | | | -| FULFILMENT_FTP_PORT | The port of the FTP service on the target server | yes | | | | -| FULFILMENT_FTP_PATH | The base path under which files should be written to the FTP server | yes | | | | -| FULFILMENT_FTP_USERNAME | The username used to authenticate with the FTP server | yes | | | | -| FULFILMENT_FTP_KEY_SECRET_ID | The ID of the secret in AWS secrets manager which contains the SSH key for authentication | yes | | | | -| FULFILMENT_S3_BUCKET | The name of the AWS S3 bucket in which to stage and aggregate fulfilment data | yes | | | | -| FULFILMENT_SEND_UNENCRYPTED_FILE | Flag for whether to send the unencrypted fulfilment file | no | false | true, false, 0, 1 | | -| FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID | The secret id for the file encryption public key | yes | | | | -| DEBUG | Use to enable output of debug information to the console | yes | | fulfilment:\*, fulfilment:staging, fulfilment:transport, fulfilment:ftp | | -| AIRBRAKE_HOST | URL of airbrake host | no | | | | -| AIRBRAKE_PROJECT_KEY | Project key for airbrake logging | no | | | | +| name | description | required | default | valid | notes | +| ----------------------------------- | ----------------------------------------------------------------------------- | :------: | ------- | ----------------------------------------------------------------------- | ----- | +| NODE_ENV | Node environment | no | | development, test, production | | +| FULFILMENT_FILE_SIZE | The maximum number of records written to an aggregated fulfilment file | yes | | | | +| FULFILMENT_S3_BUCKET | The name of the AWS S3 bucket in which to stage and aggregate fulfilment data | yes | | | | +| FULFILMENT_SEND_UNENCRYPTED_FILE | Flag for whether to send the unencrypted fulfilment file | no | false | true, false, 0, 1 | | +| FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID | The secret id for the file encryption public key | yes | | | | +| DEBUG | Use to enable output of debug information to the console | yes | | fulfilment:\*, fulfilment:staging, fulfilment:transport, fulfilment:ftp | | +| AIRBRAKE_HOST | URL of airbrake host | no | | | | +| AIRBRAKE_PROJECT_KEY | Project key for airbrake logging | no | | | | ### See also: diff --git a/packages/fulfilment-job/package-lock.json b/packages/fulfilment-job/package-lock.json index d14617d9b7..abb0a474b8 100644 --- a/packages/fulfilment-job/package-lock.json +++ b/packages/fulfilment-job/package-lock.json @@ -16,8 +16,7 @@ "merge2": "^1.4.1", "moment": "^2.29.1", "openpgp": "^5.0.0-1", - "pluralize": "^8.0.0", - "ssh2-sftp-client": "^6.0.1" + "pluralize": "^8.0.0" }, "engines": { "node": ">=18.17" @@ -50,20 +49,10 @@ "node": ">=10" } }, - "node_modules/@dabh/diagnostics": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz", - "integrity": "sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==", - "dependencies": { - "colorspace": "1.1.x", - "enabled": "2.0.x", - "kuler": "^2.0.0" - } - }, "node_modules/@defra-fish/connectors-lib": { - "version": "1.50.0-rc.7", - "resolved": "https://registry.npmjs.org/@defra-fish/connectors-lib/-/connectors-lib-1.50.0-rc.7.tgz", - "integrity": "sha512-DV8J4CnxRuYZTfbX8JCkGhm1gQ4hgScIxdRuoy/X5THMIPaLgDSxBkOH90UT/2D61U8A5QGVfgKjPuKSisfGrA==", + "version": "1.50.0-rc.8", + "resolved": "https://registry.npmjs.org/@defra-fish/connectors-lib/-/connectors-lib-1.50.0-rc.8.tgz", + "integrity": "sha512-aXPMRNfXBF6xytV+TD7C3fbeEmMb0m2VPsaDiq4uN4ShqqTg2q3OuAo6YUZypKOkX2tH6/Hj4Qpb3UMerNJ6hg==", "dependencies": { "@airbrake/node": "^2.1.7", "aws-sdk": "^2.1074.0", @@ -77,9 +66,9 @@ } }, "node_modules/@defra-fish/dynamics-lib": { - "version": "1.50.0-rc.7", - "resolved": "https://registry.npmjs.org/@defra-fish/dynamics-lib/-/dynamics-lib-1.50.0-rc.7.tgz", - "integrity": "sha512-6ttk/rlLlHR12L5Jz4Gn61f+hUer9IEp0LxczOqXyEvQqGwKiivJHZkh80jcL4OlmuFqOOj4Ft1neS1puZ0mwQ==", + "version": "1.50.0-rc.8", + "resolved": "https://registry.npmjs.org/@defra-fish/dynamics-lib/-/dynamics-lib-1.50.0-rc.8.tgz", + "integrity": "sha512-K0d15rfayclGfqcoIrrv9XaHRTXqnvhPu+1cnRmoWHgaDBeKr/YrGB/H2Y0S5Mq4tk78E1QdYIupJY9oXAWuxA==", "dependencies": { "cache-manager": "^3.6.0", "cache-manager-ioredis": "^2.1.0", @@ -202,14 +191,6 @@ "node": ">= 6.0.0" } }, - "node_modules/asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "dependencies": { - "safer-buffer": "~2.1.0" - } - }, "node_modules/asn1.js": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", @@ -293,14 +274,6 @@ } ] }, - "node_modules/bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "dependencies": { - "tweetnacl": "^0.14.3" - } - }, "node_modules/bintrees": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", @@ -326,11 +299,6 @@ "isarray": "^1.0.0" } }, - "node_modules/buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" - }, "node_modules/cache-manager": { "version": "3.6.3", "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-3.6.3.tgz", @@ -378,54 +346,6 @@ "node": ">=0.10.0" } }, - "node_modules/color": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", - "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==", - "dependencies": { - "color-convert": "^1.9.1", - "color-string": "^1.5.2" - } - }, - "node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" - }, - "node_modules/color-string": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.4.tgz", - "integrity": "sha512-57yF5yt8Xa3czSEW1jfQDE79Idk0+AkN/4KWad6tbdxUmAs3MvjxlWSWD4deYytcRfoZ9nhKyFl1kj5tBvidbw==", - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, - "node_modules/colors": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", - "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/colorspace": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz", - "integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==", - "dependencies": { - "color": "3.0.x", - "text-hex": "1.0.x" - } - }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -445,25 +365,6 @@ "node": ">= 10" } }, - "node_modules/concat-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", - "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "engines": [ - "node >= 6.0" - ], - "dependencies": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.0.2", - "typedarray": "^0.0.6" - } - }, - "node_modules/core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, "node_modules/cross-fetch": { "version": "3.1.8", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", @@ -529,16 +430,6 @@ "https-proxy-agent": "^5.0.0" } }, - "node_modules/enabled": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", - "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" - }, - "node_modules/err-code": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", - "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==" - }, "node_modules/error-stack-parser": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.1.4.tgz", @@ -574,21 +465,6 @@ "node": ">=0.4.x" } }, - "node_modules/fast-safe-stringify": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", - "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" - }, - "node_modules/fecha": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.0.tgz", - "integrity": "sha512-aN3pcx/DSmtyoovUudctc8+6Hl4T+hI9GBBHLjA76jdZl7+b1sgh5g4k+u/GL3dTy1/pnYzKp69FpJ0OicE3Wg==" - }, - "node_modules/fn.name": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", - "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" - }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -780,11 +656,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -810,14 +681,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", - "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==", - "engines": { - "node": ">=8" - } - }, "node_modules/is-typed-array": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", @@ -857,11 +720,6 @@ "@sideway/pinpoint": "^2.0.0" } }, - "node_modules/kuler": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", - "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" - }, "node_modules/lodash.clonedeep": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", @@ -882,18 +740,6 @@ "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" }, - "node_modules/logform": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz", - "integrity": "sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg==", - "dependencies": { - "colors": "^1.2.1", - "fast-safe-stringify": "^2.0.4", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "triple-beam": "^1.3.0" - } - }, "node_modules/lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -969,14 +815,6 @@ } } }, - "node_modules/one-time": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", - "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", - "dependencies": { - "fn.name": "1.x.x" - } - }, "node_modules/openpgp": { "version": "5.0.0-1", "resolved": "https://registry.npmjs.org/openpgp/-/openpgp-5.0.0-1.tgz", @@ -1012,28 +850,11 @@ "node": ">= 0.4" } }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, "node_modules/promise-polyfill": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/promise-polyfill/-/promise-polyfill-8.3.0.tgz", "integrity": "sha512-H5oELycFml5yto/atYqmjyigJoAo3+OXwolYiH7OfQuYlAqhxNvTfiNMbV9hsC6Yp83yE5r2KTVmtrG6R9i6Pg==" }, - "node_modules/promise-retry": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", - "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", - "dependencies": { - "err-code": "^2.0.2", - "retry": "^0.12.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -1048,19 +869,6 @@ "node": ">=0.4.x" } }, - "node_modules/readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/redis-commands": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", @@ -1096,33 +904,6 @@ "node": ">=8.0.0" } }, - "node_modules/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=", - "engines": { - "node": ">= 4" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -1160,57 +941,6 @@ "joi": "^17.3.0" } }, - "node_modules/simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=", - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, - "node_modules/ssh2": { - "version": "0.8.9", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-0.8.9.tgz", - "integrity": "sha512-GmoNPxWDMkVpMFa9LVVzQZHF6EW3WKmBwL+4/GeILf2hFmix5Isxm7Amamo8o7bHiU0tC+wXsGcUXOxp8ChPaw==", - "dependencies": { - "ssh2-streams": "~0.4.10" - }, - "engines": { - "node": ">=5.2.0" - } - }, - "node_modules/ssh2-sftp-client": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-6.0.1.tgz", - "integrity": "sha512-Glut2SmK/XpNOBiEuzqlKZGKkIyha2XMbuWVXR2hFUJkNsbyl/wmlZSeUEPxKFp/dC9UEvUKzanKydgLmNdfkw==", - "dependencies": { - "concat-stream": "^2.0.0", - "promise-retry": "^2.0.1", - "ssh2": "^0.8.9", - "winston": "^3.3.3" - } - }, - "node_modules/ssh2-streams": { - "version": "0.4.10", - "resolved": "https://registry.npmjs.org/ssh2-streams/-/ssh2-streams-0.4.10.tgz", - "integrity": "sha512-8pnlMjvnIZJvmTzUIIA5nT4jr2ZWNNVHwyXfMGdRJbug9TpI3kd99ffglgfSWqujVv/0gxwMsDn9j9RVst8yhQ==", - "dependencies": { - "asn1": "~0.2.0", - "bcrypt-pbkdf": "^1.0.2", - "streamsearch": "~0.1.2" - }, - "engines": { - "node": ">=5.2.0" - } - }, - "node_modules/stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=", - "engines": { - "node": "*" - } - }, "node_modules/stackframe": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.3.4.tgz", @@ -1221,22 +951,6 @@ "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" }, - "node_modules/streamsearch": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", - "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, "node_modules/tdigest": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", @@ -1245,31 +959,11 @@ "bintrees": "1.0.2" } }, - "node_modules/text-hex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", - "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" - }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, - "node_modules/triple-beam": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", - "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" - }, - "node_modules/tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" - }, - "node_modules/typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" - }, "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", @@ -1296,11 +990,6 @@ "which-typed-array": "^1.1.2" } }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, "node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -1341,64 +1030,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/winston": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/winston/-/winston-3.3.3.tgz", - "integrity": "sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw==", - "dependencies": { - "@dabh/diagnostics": "^2.0.2", - "async": "^3.1.0", - "is-stream": "^2.0.0", - "logform": "^2.2.0", - "one-time": "^1.0.0", - "readable-stream": "^3.4.0", - "stack-trace": "0.0.x", - "triple-beam": "^1.3.0", - "winston-transport": "^4.4.0" - }, - "engines": { - "node": ">= 6.4.0" - } - }, - "node_modules/winston-transport": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.0.tgz", - "integrity": "sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw==", - "dependencies": { - "readable-stream": "^2.3.7", - "triple-beam": "^1.2.0" - }, - "engines": { - "node": ">= 6.4.0" - } - }, - "node_modules/winston-transport/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/winston-transport/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/winston-transport/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, "node_modules/xml2js": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", diff --git a/packages/fulfilment-job/package.json b/packages/fulfilment-job/package.json index 3a65e1cda4..3698eb1010 100644 --- a/packages/fulfilment-job/package.json +++ b/packages/fulfilment-job/package.json @@ -42,7 +42,6 @@ "merge2": "^1.4.1", "moment": "^2.29.1", "openpgp": "^5.0.0-1", - "pluralize": "^8.0.0", - "ssh2-sftp-client": "^6.0.1" + "pluralize": "^8.0.0" } } diff --git a/packages/fulfilment-job/src/__mocks__/ssh2-sftp-client.js b/packages/fulfilment-job/src/__mocks__/ssh2-sftp-client.js deleted file mode 100644 index 124d4777bb..0000000000 --- a/packages/fulfilment-job/src/__mocks__/ssh2-sftp-client.js +++ /dev/null @@ -1,9 +0,0 @@ -const ssh2sftpClient = jest.genMockFromModule('ssh2-sftp-client') - -export const mockedFtpMethods = { - connect: jest.fn(async () => {}), - put: jest.fn(async () => {}), - end: jest.fn() -} -ssh2sftpClient.mockImplementation(() => mockedFtpMethods) -export default ssh2sftpClient diff --git a/packages/fulfilment-job/src/__tests__/config.spec.js b/packages/fulfilment-job/src/__tests__/config.spec.js index 1cbecefe04..883b6c86c6 100644 --- a/packages/fulfilment-job/src/__tests__/config.spec.js +++ b/packages/fulfilment-job/src/__tests__/config.spec.js @@ -17,11 +17,6 @@ const clearEnvVars = () => { const envVars = Object.freeze({ FULFILMENT_FILE_SIZE: 1234, - FULFILMENT_FTP_HOST: 'test-host', - FULFILMENT_FTP_PORT: 2222, - FULFILMENT_FTP_PATH: '/remote/share', - FULFILMENT_FTP_USERNAME: 'test-user', - FULFILMENT_FTP_KEY_SECRET_ID: 'test-secret-id', FULFILMENT_S3_BUCKET: 'test-bucket', FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID: 'pgp-key-secret-id', FULFILMENT_SEND_UNENCRYPTED_FILE: 'false' @@ -44,32 +39,6 @@ describe('config', () => { }) }) - describe('ftp', () => { - it('provides properties relating the use of SFTP', async () => { - expect(config.ftp).toEqual( - expect.objectContaining({ - host: 'test-host', - port: '2222', - path: '/remote/share', - username: 'test-user', - privateKey: 'test-ssh-key', - algorithms: { cipher: expect.any(Array), kex: expect.any(Array) }, - // Wait up to 60 seconds for the SSH handshake - readyTimeout: expect.any(Number), - // Retry 5 times over a minute - retries: expect.any(Number), - retry_minTimeout: expect.any(Number), - debug: expect.any(Function) - }) - ) - }) - it('defaults the sftp port to 22 if the environment variable is not configured', async () => { - delete process.env.FULFILMENT_FTP_PORT - await config.initialise() - expect(config.ftp.port).toEqual('22') - }) - }) - describe('s3', () => { it('provides properties relating the use of Amazon S3', async () => { expect(config.s3.bucket).toEqual('test-bucket') @@ -79,7 +48,7 @@ describe('config', () => { describe('pgp config', () => { const init = async (samplePublicKey = 'sample-pgp-key') => { - AwsMock.SecretsManager.__setNextResponses('getSecretValue', { SecretString: 'test-ssh-key' }, { SecretString: samplePublicKey }) + AwsMock.SecretsManager.__setNextResponses('getSecretValue', { SecretString: samplePublicKey }) await config.initialise() } beforeAll(setEnvVars) diff --git a/packages/fulfilment-job/src/config.js b/packages/fulfilment-job/src/config.js index bc0620fe45..e0f9c01315 100644 --- a/packages/fulfilment-job/src/config.js +++ b/packages/fulfilment-job/src/config.js @@ -1,45 +1,6 @@ import { AWS } from '@defra-fish/connectors-lib' -import db from 'debug' -const { secretsManager } = AWS() -/** - * Key exchange algorithms for public key authentication - in descending order of priority - * @type {string[]} - */ -export const SFTP_KEY_EXCHANGE_ALGORITHMS = [ - 'curve25519-sha256@libssh.org', - 'curve25519-sha256', - 'ecdh-sha2-nistp521', - 'ecdh-sha2-nistp384', - 'ecdh-sha2-nistp256', - 'diffie-hellman-group-exchange-sha256', - 'diffie-hellman-group14-sha256', - 'diffie-hellman-group16-sha512', - 'diffie-hellman-group18-sha512', - 'diffie-hellman-group14-sha1', - 'diffie-hellman-group-exchange-sha1', - 'diffie-hellman-group1-sha1' -] -/** - * Ciphers for SFTP support - in descending order of priority - * @type {string[]} - */ -export const SFTP_CIPHERS = [ - // http://tools.ietf.org/html/rfc4344#section-4 - 'aes256-ctr', - 'aes192-ctr', - 'aes128-ctr', - 'aes256-gcm', - 'aes256-gcm@openssh.com', - 'aes128-gcm', - 'aes128-gcm@openssh.com', - 'aes256-cbc', - 'aes192-cbc', - 'aes128-cbc', - 'blowfish-cbc', - '3des-cbc', - 'cast128-cbc' -] +const { secretsManager } = AWS() const falseRegEx = /(false|0)/i const trueRegEx = /(true|1)/i const toBoolean = val => { @@ -54,7 +15,6 @@ const toBoolean = val => { class Config { _file - _ftp _s3 _pgp @@ -68,20 +28,6 @@ class Config { */ partFileSize: Math.min(Number.parseInt(process.env.FULFILMENT_FILE_SIZE), 999) } - this.ftp = { - host: process.env.FULFILMENT_FTP_HOST, - port: process.env.FULFILMENT_FTP_PORT || '22', - path: process.env.FULFILMENT_FTP_PATH, - username: process.env.FULFILMENT_FTP_USERNAME, - privateKey: (await secretsManager.getSecretValue({ SecretId: process.env.FULFILMENT_FTP_KEY_SECRET_ID }).promise()).SecretString, - algorithms: { cipher: SFTP_CIPHERS, kex: SFTP_KEY_EXCHANGE_ALGORITHMS }, - // Wait up to 60 seconds for the SSH handshake - readyTimeout: 60000, - // Retry 5 times over a minute - retries: 5, - retry_minTimeout: 12000, - debug: db('fulfilment:ftp') - } this.s3 = { bucket: process.env.FULFILMENT_S3_BUCKET } @@ -104,18 +50,6 @@ class Config { this._file = cfg } - /** - * FTP configuration settings - * @type {object} - */ - get ftp () { - return this._ftp - } - - set ftp (cfg) { - this._ftp = cfg - } - /** * S3 configuration settings * @type {object} diff --git a/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js b/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js index 83e2145335..ca157ec115 100644 --- a/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js +++ b/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js @@ -1,7 +1,6 @@ import { Readable, PassThrough, Writable } from 'stream' import { deliverFulfilmentFiles } from '../deliver-fulfilment-files.js' import { createS3WriteStream, readS3PartFiles } from '../../transport/s3.js' -import { createFtpWriteStream } from '../../transport/ftp.js' import { FULFILMENT_FILE_STATUS_OPTIONSET, getOptionSetEntry } from '../staging-common.js' import { FulfilmentRequestFile, executeQuery, persist } from '@defra-fish/dynamics-lib' import openpgp from 'openpgp' @@ -10,7 +9,6 @@ import streamHelper from '../streamHelper.js' import merge2 from 'merge2' jest.mock('../../transport/s3.js') -jest.mock('../../transport/ftp.js') jest.mock('openpgp', () => ({ readKey: jest.fn(() => ({})), encrypt: jest.fn(({ message: readableStream }) => readableStream), @@ -46,20 +44,10 @@ describe('deliverFulfilmentFiles', () => { executeQuery.mockResolvedValue([{ entity: mockFulfilmentRequestFile2 }, { entity: mockFulfilmentRequestFile1 }]) // Streams for file1 - const { - s3DataStreamFile: s3DataStreamFile1, - ftpDataStreamFile: ftpDataStreamFile1, - s3HashStreamFile: s3HashStreamFile1, - ftpHashStreamFile: ftpHashStreamFile1 - } = createMockFileStreams() + const { s3DataStreamFile: s3DataStreamFile1, s3HashStreamFile: s3HashStreamFile1 } = createMockFileStreams() // Streams for file2 - const { - s3DataStreamFile: s3DataStreamFile2, - ftpDataStreamFile: ftpDataStreamFile2, - s3HashStreamFile: s3HashStreamFile2, - ftpHashStreamFile: ftpHashStreamFile2 - } = createMockFileStreams() + const { s3DataStreamFile: s3DataStreamFile2, s3HashStreamFile: s3HashStreamFile2 } = createMockFileStreams() // Run the delivery await expect(deliverFulfilmentFiles()).resolves.toBeUndefined() @@ -69,22 +57,14 @@ describe('deliverFulfilmentFiles', () => { // File 1 expectations expect(createS3WriteStream).toHaveBeenNthCalledWith(1, 'EAFF202006180001.json') expect(createS3WriteStream).toHaveBeenNthCalledWith(3, 'EAFF202006180001.json.sha256') - expect(createFtpWriteStream).toHaveBeenNthCalledWith(1, 'EAFF202006180001.json') - expect(createFtpWriteStream).toHaveBeenNthCalledWith(3, 'EAFF202006180001.json.sha256') expect(JSON.parse(s3DataStreamFile1.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] }) - expect(JSON.parse(ftpDataStreamFile1.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] }) expect(s3HashStreamFile1.dataProcessed).toEqual(fileShaHash) // validated - expect(ftpHashStreamFile1.dataProcessed).toEqual(fileShaHash) // validated // File 2 expectations expect(createS3WriteStream).toHaveBeenNthCalledWith(4, 'EAFF202006180002.json') expect(createS3WriteStream).toHaveBeenNthCalledWith(6, 'EAFF202006180002.json.sha256') - expect(createFtpWriteStream).toHaveBeenNthCalledWith(4, 'EAFF202006180002.json') - expect(createFtpWriteStream).toHaveBeenNthCalledWith(6, 'EAFF202006180002.json.sha256') expect(JSON.parse(s3DataStreamFile2.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] }) - expect(JSON.parse(ftpDataStreamFile2.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] }) expect(s3HashStreamFile2.dataProcessed).toEqual(fileShaHash) // validated - expect(ftpHashStreamFile2.dataProcessed).toEqual(fileShaHash) // validated // Persist to dynamics for file 1 expect(persist).toHaveBeenNthCalledWith(1, [ @@ -197,10 +177,7 @@ describe('deliverFulfilmentFiles', () => { const s3 = createTestableStream() streamHelper.pipelinePromise.mockResolvedValue() openpgp.encrypt.mockResolvedValue(s2) - merge2 - .mockReturnValueOnce(s1) - .mockReturnValueOnce(s2) - .mockReturnValueOnce(s3) + merge2.mockReturnValueOnce(s1).mockReturnValueOnce(s2).mockReturnValueOnce(s3) await mockExecuteQuery() createMockFileStreams() @@ -227,27 +204,18 @@ const createMockFulfilmentRequestFile = async (fileName, date) => const createMockFileStreams = () => { const s3DataStreamFile = createTestableStream() - const ftpDataStreamFile = createTestableStream() createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3DataStreamFile, managedUpload: Promise.resolve() }) - createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpDataStreamFile, managedUpload: Promise.resolve() }) const s3EncryptedDataStreamFile = createTestableStream() - const ftpEncryptedDataStreamFile = createTestableStream() createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3EncryptedDataStreamFile, managedUpload: Promise.resolve() }) - createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpEncryptedDataStreamFile, managedUpload: Promise.resolve() }) const s3HashStreamFile = createTestableStream() - const ftpHashStreamFile = createTestableStream() createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3HashStreamFile, managedUpload: Promise.resolve() }) - createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpHashStreamFile, managedUpload: Promise.resolve() }) return { s3DataStreamFile, - ftpDataStreamFile, s3EncryptedDataStreamFile, - ftpEncryptedDataStreamFile, - s3HashStreamFile, - ftpHashStreamFile + s3HashStreamFile } } diff --git a/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js b/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js index 09c24ceb80..25036678dc 100644 --- a/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js +++ b/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js @@ -4,7 +4,6 @@ import merge2 from 'merge2' import moment from 'moment' import { executeQuery, persist, findFulfilmentFiles } from '@defra-fish/dynamics-lib' import { createS3WriteStream, readS3PartFiles } from '../transport/s3.js' -import { createFtpWriteStream } from '../transport/ftp.js' import { FULFILMENT_FILE_STATUS_OPTIONSET, getOptionSetEntry } from './staging-common.js' import db from 'debug' import openpgp from 'openpgp' @@ -69,11 +68,6 @@ const createEncryptedDataReadStream = async file => { */ const deliver = async (targetFileName, readableStream, ...transforms) => { const { s3WriteStream: s3DataStream, managedUpload: s3DataManagedUpload } = createS3WriteStream(targetFileName) - const { ftpWriteStream: ftpDataStream, managedUpload: ftpDataManagedUpload } = createFtpWriteStream(targetFileName) - await Promise.all([ - streamHelper.pipelinePromise([readableStream, ...transforms, s3DataStream, ftpDataStream]), - s3DataManagedUpload, - ftpDataManagedUpload - ]) + await Promise.all([streamHelper.pipelinePromise([readableStream, ...transforms, s3DataStream]), s3DataManagedUpload]) } diff --git a/packages/fulfilment-job/src/transport/__tests__/ftp.spec.js b/packages/fulfilment-job/src/transport/__tests__/ftp.spec.js deleted file mode 100644 index bf6e868ac4..0000000000 --- a/packages/fulfilment-job/src/transport/__tests__/ftp.spec.js +++ /dev/null @@ -1,63 +0,0 @@ -import { createFtpWriteStream } from '../ftp.js' -import { mockedFtpMethods } from 'ssh2-sftp-client' - -jest.mock('stream') -jest.mock('../../config.js', () => ({ - ftp: { - host: 'testhost', - port: 2222, - path: 'testpath/', - username: 'testusername', - privateKey: 'testprivatekey' - } -})) - -describe('ftp', () => { - beforeEach(() => { - jest.clearAllMocks() - }) - - describe('createFtpWriteStream', () => { - it('creates a stream to write to the configured FTP server', async () => { - const { ftpWriteStream, managedUpload } = createFtpWriteStream('testfile.json') - ftpWriteStream.write('Some data') - ftpWriteStream.end() - await managedUpload - expect(mockedFtpMethods.connect).toHaveBeenCalledWith( - expect.objectContaining({ - host: 'testhost', - port: 2222, - username: 'testusername', - privateKey: 'testprivatekey' - }) - ) - expect(mockedFtpMethods.put).toHaveBeenCalledWith(ftpWriteStream, 'testpath/testfile.json', { - flags: 'w', - encoding: 'UTF-8', - autoClose: false - }) - expect(mockedFtpMethods.end).toHaveBeenCalled() - }) - - it('rejects the managed upload promise if an FTP upload error occurs', async () => { - const testError = new Error('Test error') - mockedFtpMethods.put.mockImplementationOnce(() => Promise.reject(testError)) - const { ftpWriteStream, managedUpload } = createFtpWriteStream('testfile.json') - await expect(managedUpload).rejects.toThrow('Test error') - expect(mockedFtpMethods.connect).toHaveBeenCalledWith( - expect.objectContaining({ - host: 'testhost', - port: 2222, - username: 'testusername', - privateKey: 'testprivatekey' - }) - ) - expect(mockedFtpMethods.put).toHaveBeenCalledWith(ftpWriteStream, 'testpath/testfile.json', { - flags: 'w', - encoding: 'UTF-8', - autoClose: false - }) - expect(mockedFtpMethods.end).toHaveBeenCalled() - }) - }) -}) diff --git a/packages/fulfilment-job/src/transport/ftp.js b/packages/fulfilment-job/src/transport/ftp.js deleted file mode 100644 index 0ce3f41c9c..0000000000 --- a/packages/fulfilment-job/src/transport/ftp.js +++ /dev/null @@ -1,28 +0,0 @@ -import FtpClient from 'ssh2-sftp-client' -import Path from 'path' -import { PassThrough } from 'stream' -import config from '../config.js' -import db from 'debug' -const debug = db('fulfilment:transport') - -/** - * Create a stream to write to the configured FTP server - * - * @param {string} filename The name of the file to be written to the remote server - * @returns {{ftpWriteStream: module:stream.internal.PassThrough, managedUpload: Promise<*>}} - */ -export const createFtpWriteStream = filename => { - const sftp = new FtpClient() - const passThrough = new PassThrough() - const remoteFilePath = Path.join(config.ftp.path, filename) - return { - ftpWriteStream: passThrough, - managedUpload: sftp - .connect(config.ftp) - .then(() => sftp.put(passThrough, remoteFilePath, { flags: 'w', encoding: 'UTF-8', autoClose: false })) - .then(() => - debug('File successfully uploaded to fulfilment provider at sftp://%s:%s%s', config.ftp.host, config.ftp.port, remoteFilePath) - ) - .finally(() => sftp.end()) - } -} diff --git a/packages/gafl-webapp-service/src/handlers/__tests__/agreed-handler-recurring-payments.spec.js b/packages/gafl-webapp-service/src/handlers/__tests__/agreed-handler-recurring-payments.spec.js index 4355907d1e..582f28a17c 100644 --- a/packages/gafl-webapp-service/src/handlers/__tests__/agreed-handler-recurring-payments.spec.js +++ b/packages/gafl-webapp-service/src/handlers/__tests__/agreed-handler-recurring-payments.spec.js @@ -1,8 +1,8 @@ import { salesApi } from '@defra-fish/connectors-lib' import { COMPLETION_STATUS, RECURRING_PAYMENT } from '../../constants.js' import agreedHandler from '../agreed-handler.js' -import { prepareRecurringPayment } from '../../processors/payment.js' -import { sendRecurringPayment } from '../../services/payment/govuk-pay-service.js' +import { preparePayment, prepareRecurringPaymentAgreement } from '../../processors/payment.js' +import { sendPayment, sendRecurringPayment, getPaymentStatus } from '../../services/payment/govuk-pay-service.js' import { prepareApiTransactionPayload } from '../../processors/api-transaction.js' import { v4 as uuidv4 } from 'uuid' import db from 'debug' @@ -10,7 +10,13 @@ import db from 'debug' jest.mock('@defra-fish/connectors-lib') jest.mock('../../processors/payment.js') jest.mock('../../services/payment/govuk-pay-service.js', () => ({ - sendPayment: jest.fn(), + sendPayment: jest.fn(() => ({ + payment_id: 'payment-id-1', + _links: { + next_url: { href: 'next-url' }, + self: { href: 'self-url' } + } + })), getPaymentStatus: jest.fn(), sendRecurringPayment: jest.fn(() => ({ agreementId: 'agr-eem-ent-id1' })) })) @@ -32,12 +38,12 @@ describe('The agreed handler', () => { }) beforeEach(jest.clearAllMocks) - const getMockRequest = (overrides = {}) => ({ + const getMockRequest = ({ overrides = {}, transactionSet = () => {} } = {}) => ({ cache: () => ({ helpers: { transaction: { get: async () => ({ cost: 0 }), - set: async () => {} + set: transactionSet }, status: { get: async () => ({ @@ -54,6 +60,7 @@ describe('The agreed handler', () => { }) const getRequestToolkit = () => ({ + redirect: jest.fn(), redirectWithLanguageCode: jest.fn() }) @@ -61,18 +68,20 @@ describe('The agreed handler', () => { it('sends the request and transaction to prepare the recurring payment', async () => { const transaction = { cost: 0 } const mockRequest = getMockRequest({ - transaction: { - get: async () => transaction, - set: () => {} + overrides: { + transaction: { + get: async () => transaction, + set: () => {} + } } }) await agreedHandler(mockRequest, getRequestToolkit()) - expect(prepareRecurringPayment).toHaveBeenCalledWith(mockRequest, transaction) + expect(prepareRecurringPaymentAgreement).toHaveBeenCalledWith(mockRequest, transaction) }) it('adds a v4 guid to the transaction as an id', async () => { let transactionPayload = null - prepareRecurringPayment.mockImplementationOnce((_p1, tp) => { + prepareRecurringPaymentAgreement.mockImplementationOnce((_p1, tp) => { transactionPayload = { ...tp } }) const v4guid = Symbol('v4guid') @@ -90,46 +99,73 @@ describe('The agreed handler', () => { expect(transactionPayload.id).toBe(v4guid) }) - it("doesn't overwrite transaction id if one is already set", async () => { - const setTransaction = jest.fn() - const transactionId = 'abc-123-def-456' - uuidv4.mockReturnValue('def-789-ghi-012') - salesApi.finaliseTransaction.mockReturnValueOnce({ - permissions: [] + it('sends a recurring payment agreement creation request to Gov.UK Pay', async () => { + const preparedPayment = Symbol('preparedPayment') + prepareRecurringPaymentAgreement.mockResolvedValueOnce(preparedPayment) + await agreedHandler(getMockRequest(), getRequestToolkit()) + expect(sendRecurringPayment).toHaveBeenCalledWith(preparedPayment) + }) + + describe('when there is a cost and recurringAgreement status is set to true', () => { + beforeEach(() => { + salesApi.createTransaction.mockResolvedValueOnce({ + id: 'transaction-id-1', + cost: 100 + }) + }) + + it('calls preparePayment', async () => { + const transaction = { id: Symbol('transaction') } + const request = getMockRequest({ + overrides: { + transaction: { + get: async () => transaction, + set: () => {} + } + } + }) + const toolkit = getRequestToolkit() + + await agreedHandler(request, toolkit) + expect(preparePayment).toHaveBeenCalledWith(request, transaction) + }) + + it('calls sendPayment with recurring as true', async () => { + const preparedPayment = Symbol('preparedPayment') + preparePayment.mockReturnValueOnce(preparedPayment) + + await agreedHandler(getMockRequest(), getRequestToolkit()) + expect(sendPayment).toHaveBeenCalledWith(preparedPayment, true) }) - const mockRequest = { - cache: () => ({ - helpers: { + + it('calls getPaymentStatus with recurring as true', async () => { + const id = Symbol('paymentId') + const transaction = { id: '123', payment: { payment_id: id } } + const request = getMockRequest({ + overrides: { + transaction: { + get: async () => transaction, + set: () => {} + }, status: { get: async () => ({ [COMPLETION_STATUS.agreed]: true, - [COMPLETION_STATUS.posted]: true, - [COMPLETION_STATUS.finalised]: false, - [RECURRING_PAYMENT]: false + [COMPLETION_STATUS.posted]: false, + [COMPLETION_STATUS.finalised]: true, + [RECURRING_PAYMENT]: true, + [COMPLETION_STATUS.paymentCreated]: true }), set: () => {} - }, - transaction: { - get: async () => ({ cost: 0, id: transactionId }), - set: setTransaction } } }) - } + const toolkit = getRequestToolkit() - await agreedHandler(mockRequest, getRequestToolkit()) + getPaymentStatus.mockReturnValueOnce({ state: { finished: true, status: 'success' } }) - expect(salesApi.finaliseTransaction).toHaveBeenCalledWith( - transactionId, - undefined // prepareApiFinalisationPayload has no mocked return value - ) - }) - - it('sends a recurring payment creation request to Gov.UK Pay', async () => { - const preparedPayment = Symbol('preparedPayment') - prepareRecurringPayment.mockResolvedValueOnce(preparedPayment) - await agreedHandler(getMockRequest(), getRequestToolkit()) - expect(sendRecurringPayment).toHaveBeenCalledWith(preparedPayment) + await agreedHandler(request, toolkit) + expect(getPaymentStatus).toHaveBeenCalledWith(id, true) + }) }) // this doesn't really belong here, but until the other agreed handler tests are refactored to @@ -140,7 +176,7 @@ describe('The agreed handler', () => { await agreedHandler(getMockRequest(), getRequestToolkit()) - expect(prepareApiTransactionPayload).toHaveBeenCalledWith(expect.any(Object), v4guid) + expect(prepareApiTransactionPayload).toHaveBeenCalledWith(expect.any(Object), v4guid, undefined) }) it.each(['zxy-098-wvu-765', '467482f1-099d-403d-b6b3-8db7e70d19e3'])( @@ -158,5 +194,23 @@ describe('The agreed handler', () => { expect(debugMock).toHaveBeenCalledWith(`Created agreement with id ${agreement_id}`) } ) + + it.each(['zxy-098-wvu-765', '467482f1-099d-403d-b6b3-8db7e70d19e3'])( + "assigns agreement id '%s' to the transaction when recurring payment agreement created", + async agreementId => { + const mockTransactionCacheSet = jest.fn() + sendRecurringPayment.mockResolvedValueOnce({ + agreement_id: agreementId + }) + + await agreedHandler(getMockRequest({ transactionSet: mockTransactionCacheSet }), getRequestToolkit()) + + expect(mockTransactionCacheSet).toHaveBeenCalledWith( + expect.objectContaining({ + agreementId + }) + ) + } + ) }) }) diff --git a/packages/gafl-webapp-service/src/handlers/agreed-handler.js b/packages/gafl-webapp-service/src/handlers/agreed-handler.js index 9acb830d80..01b3c748f9 100644 --- a/packages/gafl-webapp-service/src/handlers/agreed-handler.js +++ b/packages/gafl-webapp-service/src/handlers/agreed-handler.js @@ -14,7 +14,7 @@ import db from 'debug' import { salesApi } from '@defra-fish/connectors-lib' import { prepareApiTransactionPayload, prepareApiFinalisationPayload } from '../processors/api-transaction.js' import { sendPayment, getPaymentStatus, sendRecurringPayment } from '../services/payment/govuk-pay-service.js' -import { preparePayment, prepareRecurringPayment } from '../processors/payment.js' +import { preparePayment, prepareRecurringPaymentAgreement } from '../processors/payment.js' import { COMPLETION_STATUS, RECURRING_PAYMENT } from '../constants.js' import { ORDER_COMPLETE, PAYMENT_CANCELLED, PAYMENT_FAILED } from '../uri.js' import { PAYMENT_JOURNAL_STATUS_CODES, GOVUK_PAY_ERROR_STATUS_CODES } from '@defra-fish/business-rules-lib' @@ -29,7 +29,7 @@ const debug = db('webapp:agreed-handler') * @returns {Promise<*>} */ const sendToSalesApi = async (request, transaction, status) => { - const apiTransactionPayload = await prepareApiTransactionPayload(request, transaction.id) + const apiTransactionPayload = await prepareApiTransactionPayload(request, transaction.id, transaction.agreementId) let response try { response = await salesApi.createTransaction(apiTransactionPayload) @@ -63,7 +63,7 @@ const createRecurringPayment = async (request, transaction, status) => { /* * Prepare the payment payload */ - const preparedPayment = await prepareRecurringPayment(request, transaction) + const preparedPayment = await prepareRecurringPaymentAgreement(request, transaction) /* * Send the prepared payment to the GOV.UK pay API using the connector @@ -72,7 +72,11 @@ const createRecurringPayment = async (request, transaction, status) => { debug(`Created agreement with id ${paymentResponse.agreement_id}`) status[COMPLETION_STATUS.recurringAgreement] = true + + transaction.agreementId = paymentResponse.agreement_id + await request.cache().helpers.status.set(status) + await request.cache().helpers.transaction.set(transaction) } /** @@ -88,6 +92,8 @@ const createRecurringPayment = async (request, transaction, status) => { * @returns {Promise} */ const createPayment = async (request, transaction, status) => { + const recurring = status && status[COMPLETION_STATUS.recurringAgreement] === true + /* * Prepare the payment payload */ @@ -96,7 +102,7 @@ const createPayment = async (request, transaction, status) => { /* * Send the prepared payment to the GOV.UK pay API using the connector */ - const paymentResponse = await sendPayment(preparedPayment) + const paymentResponse = await sendPayment(preparedPayment, recurring) /* * Used by the payment mop up job, create the payment journal entry which is removed when the user completes the journey @@ -143,10 +149,12 @@ const createPayment = async (request, transaction, status) => { * @returns {Promise} */ const processPayment = async (request, transaction, status) => { + const recurring = status && status[COMPLETION_STATUS.recurringAgreement] === true + /* * Get the payment status */ - const { state } = await getPaymentStatus(transaction.payment.payment_id) + const { state } = await getPaymentStatus(transaction.payment.payment_id, recurring) if (!state.finished) { throw Boom.forbidden('Attempt to access the agreed handler during payment journey') diff --git a/packages/gafl-webapp-service/src/locales/cy.json b/packages/gafl-webapp-service/src/locales/cy.json index b20be27875..fa2942b23b 100644 --- a/packages/gafl-webapp-service/src/locales/cy.json +++ b/packages/gafl-webapp-service/src/locales/cy.json @@ -466,11 +466,11 @@ "licence_type_payment_edge_case": "Mae’n rhaid i chi gwblhau eich taliad cyn 11:30pm ar 31 Mawrth 2024 i dalu’r pris a ddangosir", "licence_type_radio_salmon_hint": "Mae'n cynnwys brithyllod a physgod bras (hyd at 3 gwialen)", "licence_type_radio_salmon": "Eogiaid a brithyllod y môr", - "licence_type_radio_salmon_payment_summary": "eogiaid a brithyllod y môr", + "licence_type_radio_salmon_payment_summary": "Eogiaid a brithyllod y môr", "licence_type_radio_trout_three_rod": "Brithyllod a physgod bras (hyd at 3 gwialen)", - "licence_type_radio_trout_three_rod_payment_summary": "brithyllod a physgod bras (hyd at 3 gwialen)", + "licence_type_radio_trout_three_rod_payment_summary": "Brithyllod a physgod bras (hyd at 3 gwialen)", "licence_type_radio_trout_two_rod": "Brithyllod a physgod bras (hyd at 2 wialen)", - "licence_type_radio_trout_two_rod_payment_summary": "brithyllod a physgod bras (hyd at 2 wialen)", + "licence_type_radio_trout_two_rod_payment_summary": "Brithyllod a physgod bras (hyd at 2 wialen)", "licence_type_rules": "rheolau pysgota â gwialen (yn agor ar dudalen newydd)", "licence_type_salmon_acr_note_1": "Yn ôl y gyfraith, mae'n rhaid i chi roi gwybod am ", "licence_type_salmon_acr_note_2": " ffurflen daliadau (yn agor ar dudalen newydd)", diff --git a/packages/gafl-webapp-service/src/locales/en.json b/packages/gafl-webapp-service/src/locales/en.json index f73a836afa..d5fc5b9f92 100644 --- a/packages/gafl-webapp-service/src/locales/en.json +++ b/packages/gafl-webapp-service/src/locales/en.json @@ -466,11 +466,11 @@ "licence_type_payment_edge_case": "You must complete payment before 11:30pm on 31 March 2024 to get the price shown", "licence_type_radio_salmon_hint": "Includes trout and coarse (up to 3 rods)", "licence_type_radio_salmon": "Salmon and sea trout", - "licence_type_radio_salmon_payment_summary": "salmon and sea trout", + "licence_type_radio_salmon_payment_summary": "Salmon and sea trout", "licence_type_radio_trout_three_rod": "Trout and coarse (up to 3 rods)", - "licence_type_radio_trout_three_rod_payment_summary": "trout and coarse (up to 3 rods)", + "licence_type_radio_trout_three_rod_payment_summary": "Trout and coarse (up to 3 rods)", "licence_type_radio_trout_two_rod": "Trout and coarse (up to 2 rods)", - "licence_type_radio_trout_two_rod_payment_summary": "trout and coarse (up to 2 rods)", + "licence_type_radio_trout_two_rod_payment_summary": "Trout and coarse (up to 2 rods)", "licence_type_rules": "rod fishing rules (opens in new tab)", "licence_type_salmon_acr_note_1": "Licence holders must by law ", "licence_type_salmon_acr_note_2": " report a catch return (opens in new tab)", diff --git a/packages/gafl-webapp-service/src/pages/summary/licence-summary/__tests__/__snapshots__/route.spec.js.snap b/packages/gafl-webapp-service/src/pages/summary/licence-summary/__tests__/__snapshots__/route.spec.js.snap index 233578b8c1..6aa626139f 100644 --- a/packages/gafl-webapp-service/src/pages/summary/licence-summary/__tests__/__snapshots__/route.spec.js.snap +++ b/packages/gafl-webapp-service/src/pages/summary/licence-summary/__tests__/__snapshots__/route.spec.js.snap @@ -246,6 +246,127 @@ Array [ ] `; +exports[`licence-summary > route licence summary rows creates licence summary name rows for 1 year new three rod licence 1`] = ` +Array [ + Object { + "actions": Object { + "items": Array [ + Object { + "attributes": Object { + "id": "change-name", + }, + "href": "/buy/name", + "text": "contact_summary_change", + "visuallyHiddenText": "licence_summary_name", + }, + ], + }, + "key": Object { + "text": "licence_summary_name", + }, + "value": Object { + "html": "Brenin Pysgotwr", + }, + }, + Object { + "actions": Object { + "items": Array [ + Object { + "attributes": Object { + "id": "change-birth-date", + }, + "href": "/buy/date-of-birth", + "text": "contact_summary_change", + "visuallyHiddenText": "licence_summary_dob", + }, + ], + }, + "key": Object { + "text": "licence_summary_dob", + }, + "value": Object { + "html": "1st January 1946", + }, + }, + Object { + "actions": Object { + "items": Array [ + Object { + "attributes": Object { + "id": "change-licence-type", + }, + "href": "/buy/licence-type", + "text": "contact_summary_change", + "visuallyHiddenText": "licence_summary_type", + }, + ], + }, + "key": Object { + "text": "licence_summary_type", + }, + "value": Object { + "html": "Special Canal Licence, Shopping Trollies and Old Wellies", + }, + }, + Object { + "key": Object { + "text": "licence_summary_length", + }, + "value": Object { + "html": "licence_type_12m", + }, + }, + Object { + "actions": Object { + "items": Array [ + Object { + "attributes": Object { + "id": "change-licence-to-start", + }, + "href": "/buy/start-kind", + "text": "contact_summary_change", + "visuallyHiddenText": "licence_summary_start_date", + }, + ], + }, + "key": Object { + "text": "licence_summary_start_date", + }, + "value": Object { + "html": "30licence_summary_minutes_after_payment", + }, + }, + Object { + "actions": Object { + "items": Array [ + Object { + "attributes": Object { + "id": "change-benefit-check", + }, + "href": "/buy/disability-concession", + "text": "contact_summary_change", + "visuallyHiddenText": "licence_summary_ni_num", + }, + ], + }, + "key": Object { + "text": "licence_summary_ni_num", + }, + "value": Object { + "html": "AB 12 34 56 A", + }, + }, + Object { + "key": Object { + "text": "damage", + }, + "value": Object { + "html": "#6", + }, + }, +] +`; + exports[`licence-summary > route licence summary rows creates licence summary name rows for 1 year renewal 1`] = ` Array [ Object { diff --git a/packages/gafl-webapp-service/src/pages/summary/licence-summary/__tests__/route.spec.js b/packages/gafl-webapp-service/src/pages/summary/licence-summary/__tests__/route.spec.js index 3015c8565b..0637e3ad11 100644 --- a/packages/gafl-webapp-service/src/pages/summary/licence-summary/__tests__/route.spec.js +++ b/packages/gafl-webapp-service/src/pages/summary/licence-summary/__tests__/route.spec.js @@ -116,7 +116,7 @@ const getMockPermission = (licenseeOverrides = {}) => ({ licenceToStart: 'after-payment', licenceStartDate: '2022-11-10', licenceType: 'Trout and coarse', - numberOfRods: '3', + numberOfRods: '2', permit: { cost: 6 } }) @@ -372,16 +372,17 @@ describe('licence-summary > route', () => { describe('licence summary rows', () => { it.each` - desc | currentPermission - ${'1 year renewal'} | ${getMockPermission()} - ${'1 year new licence'} | ${getMockNewPermission()} - ${'1 year senior renewal'} | ${getMockSeniorPermission()} - ${'8 day licence'} | ${{ ...getMockNewPermission(), licenceLength: '8D' }} - ${'1 day licence'} | ${{ ...getMockNewPermission(), licenceLength: '1D' }} - ${'Junior licence'} | ${getMockJuniorPermission()} - ${'Blue badge concession'} | ${getMockBlueBadgePermission()} - ${'Continuing permission'} | ${getMockContinuingPermission()} - ${'Another date permission'} | ${{ ...getMockPermission(), licenceToStart: 'another-date' }} + desc | currentPermission + ${'1 year renewal'} | ${getMockPermission()} + ${'1 year new licence'} | ${getMockNewPermission()} + ${'1 year senior renewal'} | ${getMockSeniorPermission()} + ${'8 day licence'} | ${{ ...getMockNewPermission(), licenceLength: '8D' }} + ${'1 day licence'} | ${{ ...getMockNewPermission(), licenceLength: '1D' }} + ${'Junior licence'} | ${getMockJuniorPermission()} + ${'Blue badge concession'} | ${getMockBlueBadgePermission()} + ${'Continuing permission'} | ${getMockContinuingPermission()} + ${'Another date permission'} | ${{ ...getMockPermission(), licenceToStart: 'another-date' }} + ${'1 year new three rod licence '} | ${{ ...getMockNewPermission(), numberOfRods: '3' }} `('creates licence summary name rows for $desc', async ({ currentPermission }) => { const mockRequest = getMockRequest({ currentPermission }) const data = await getData(mockRequest) diff --git a/packages/gafl-webapp-service/src/pages/summary/licence-summary/route.js b/packages/gafl-webapp-service/src/pages/summary/licence-summary/route.js index b91b3b4ec2..b43e637f6b 100644 --- a/packages/gafl-webapp-service/src/pages/summary/licence-summary/route.js +++ b/packages/gafl-webapp-service/src/pages/summary/licence-summary/route.js @@ -113,12 +113,13 @@ class RowGenerator { } generateLicenceLengthRow () { - return this.generateStandardRow( - 'licence_summary_length', - this.labels[`licence_type_${this.permission.licenceLength.toLowerCase()}`], - LICENCE_LENGTH.uri, - 'change-licence-length' - ) + const args = ['licence_summary_length', this.labels[`licence_type_${this.permission.licenceLength.toLowerCase()}`]] + + if (this.permission.numberOfRods !== '3') { + args.push(LICENCE_LENGTH.uri, 'change-licence-length') + } + + return this.generateStandardRow(...args) } } diff --git a/packages/gafl-webapp-service/src/processors/__tests__/api-transaction.spec.js b/packages/gafl-webapp-service/src/processors/__tests__/api-transaction.spec.js index 82377b91ee..8382922427 100644 --- a/packages/gafl-webapp-service/src/processors/__tests__/api-transaction.spec.js +++ b/packages/gafl-webapp-service/src/processors/__tests__/api-transaction.spec.js @@ -63,7 +63,7 @@ describe('prepareApiTransactionPayload', () => { }) }) - it('adds transaction id to payload', async () => { + it('adds transactionId to payload', async () => { const transactionId = Symbol('transactionId') const payload = await prepareApiTransactionPayload(getMockRequest(), transactionId) @@ -71,6 +71,14 @@ describe('prepareApiTransactionPayload', () => { expect(payload.transactionId).toBe(transactionId) }) + it('adds agreementId to payload', async () => { + const agreementId = Symbol('agreementId') + + const payload = await prepareApiTransactionPayload(getMockRequest(), 'transaction_id', agreementId) + + expect(payload.agreementId).toBe(agreementId) + }) + const getMockRequest = (overrides = {}, state = {}) => ({ cache: () => ({ helpers: { diff --git a/packages/gafl-webapp-service/src/processors/__tests__/payment.spec.js b/packages/gafl-webapp-service/src/processors/__tests__/payment.spec.js index 9c17dfdbfe..f96d6f4af7 100644 --- a/packages/gafl-webapp-service/src/processors/__tests__/payment.spec.js +++ b/packages/gafl-webapp-service/src/processors/__tests__/payment.spec.js @@ -1,4 +1,4 @@ -import { preparePayment, prepareRecurringPayment } from '../payment.js' +import { preparePayment, prepareRecurringPaymentAgreement } from '../payment.js' import { licenceTypeAndLengthDisplay } from '../licence-type-display.js' import { addLanguageCodeToUri } from '../uri-helper.js' import { AGREED } from '../../uri.js' @@ -20,9 +20,16 @@ const createRequest = (opts = {}, catalog = {}) => ({ server: { info: { protocol: opts.protocol || '' } } }) -const createTransaction = ({ isLicenceForYou = true, additionalPermissions = [], cost = 12, licenseeOverrides = {} } = {}) => ({ +const createTransaction = ({ + isLicenceForYou = true, + additionalPermissions = [], + cost = 12, + licenseeOverrides = {}, + agreementId +} = {}) => ({ id: 'transaction-id', cost, + agreementId, permissions: [ { id: 'permission-id', @@ -49,7 +56,7 @@ describe('preparePayment', () => { it.each(['http', 'https'])('uses SSL when "x-forwarded-proto" header is present, proto "%s"', proto => { addLanguageCodeToUri.mockReturnValue(proto + '://localhost:1234/buy/agreed') const request = createRequest({ headers: { 'x-forwarded-proto': proto } }) - const result = preparePayment(request, createTransaction()) + const result = preparePayment(request, createTransaction(), false) expect(result.return_url).toBe(`${proto}://localhost:1234/buy/agreed`) }) @@ -211,12 +218,30 @@ describe('preparePayment', () => { expect(result.email).toBe(undefined) }) }) + + describe('if agreementId is not present', () => { + it('does not include set_up_agreement', () => { + const result = preparePayment(createRequest(), createTransaction()) + expect(result.set_up_agreement).toBe(undefined) + }) + }) + + describe('if agreementId is present', () => { + it('set_up_agreement is set to agreementId', () => { + const agreementId = 'foo' + const recurringPaymentTransaction = createTransaction({ agreementId }) + + const result = preparePayment(createRequest(), recurringPaymentTransaction) + + expect(result.set_up_agreement).toBe(agreementId) + }) + }) }) -describe('prepareRecurringPayment', () => { +describe('prepareRecurringPaymentAgreement', () => { it('reference equals transaction.id', async () => { const transaction = createTransaction() - const result = await prepareRecurringPayment(createRequest(), transaction) + const result = await prepareRecurringPaymentAgreement(createRequest(), transaction) expect(result.reference).toBe(transaction.id) }) @@ -227,7 +252,7 @@ describe('prepareRecurringPayment', () => { const request = createRequest({}, mockCatalog) const transaction = createTransaction() - const result = await prepareRecurringPayment(request, transaction) + const result = await prepareRecurringPaymentAgreement(request, transaction) expect(result.description).toBe(mockCatalog.recurring_payment_description) }) @@ -235,7 +260,7 @@ describe('prepareRecurringPayment', () => { const transaction = createTransaction() const request = createRequest() - const result = await prepareRecurringPayment(request, transaction) - expect(debug).toHaveBeenCalledWith('Creating prepared recurring payment %o', result) + const result = await prepareRecurringPaymentAgreement(request, transaction) + expect(debug).toHaveBeenCalledWith('Creating prepared recurring payment agreement %o', result) }) }) diff --git a/packages/gafl-webapp-service/src/processors/api-transaction.js b/packages/gafl-webapp-service/src/processors/api-transaction.js index 07c199b435..41d5fb23a9 100644 --- a/packages/gafl-webapp-service/src/processors/api-transaction.js +++ b/packages/gafl-webapp-service/src/processors/api-transaction.js @@ -7,7 +7,7 @@ import { countries } from './refdata-helper.js' import { salesApi } from '@defra-fish/connectors-lib' import { licenceToStart } from '../pages/licence-details/licence-to-start/update-transaction.js' -export const prepareApiTransactionPayload = async (request, transactionId) => { +export const prepareApiTransactionPayload = async (request, transactionId, agreementId) => { const transactionCache = await request.cache().helpers.transaction.get() const concessions = await salesApi.concessions.getAll() const countryList = await countries.getAll() @@ -63,7 +63,8 @@ export const prepareApiTransactionPayload = async (request, transactionId) => { request.state && request.state[process.env.OIDC_SESSION_COOKIE_NAME] ? request.state[process.env.OIDC_SESSION_COOKIE_NAME].oid : undefined, - transactionId + transactionId, + agreementId } } diff --git a/packages/gafl-webapp-service/src/processors/payment.js b/packages/gafl-webapp-service/src/processors/payment.js index fc75fdfd1b..320152a3a4 100644 --- a/packages/gafl-webapp-service/src/processors/payment.js +++ b/packages/gafl-webapp-service/src/processors/payment.js @@ -50,17 +50,21 @@ export const preparePayment = (request, transaction) => { } } + if (transaction.agreementId) { + result.set_up_agreement = transaction.agreementId + } + debug('Creating prepared payment %o', result) return result } -export const prepareRecurringPayment = async (request, transaction) => { +export const prepareRecurringPaymentAgreement = async (request, transaction) => { debug('Preparing recurring payment %s', JSON.stringify(transaction, undefined, '\t')) - // The recurring card payment for your rod fishing licence + // The recurring card payment agreement for your rod fishing licence const result = { reference: transaction.id, description: request.i18n.getCatalog().recurring_payment_description } - debug('Creating prepared recurring payment %o', result) + debug('Creating prepared recurring payment agreement %o', result) return result } diff --git a/packages/gafl-webapp-service/src/services/payment/__test__/govuk-pay-service.spec.js b/packages/gafl-webapp-service/src/services/payment/__test__/govuk-pay-service.spec.js index 2eeb743b29..ac995d1b2a 100644 --- a/packages/gafl-webapp-service/src/services/payment/__test__/govuk-pay-service.spec.js +++ b/packages/gafl-webapp-service/src/services/payment/__test__/govuk-pay-service.spec.js @@ -2,7 +2,7 @@ import mockTransaction from './data/mock-transaction.js' import { preparePayment } from '../../../processors/payment.js' import { AGREED } from '../../../uri.js' import { addLanguageCodeToUri } from '../../../processors/uri-helper.js' -import { sendRecurringPayment } from '../govuk-pay-service.js' +import { sendPayment, sendRecurringPayment, getPaymentStatus } from '../govuk-pay-service.js' import { govUkPayApi } from '@defra-fish/connectors-lib' import db from 'debug' const { value: debug } = db.mock.results[db.mock.calls.findIndex(c => c[0] === 'webapp:govuk-pay-service')] @@ -158,6 +158,142 @@ describe('The govuk-pay-service', () => { console.log(preparedPayment) }) + describe('sendPayment', () => { + const preparedPayment = { + id: '1234', + user_identifier: 'test-user' + } + + beforeEach(() => { + jest.clearAllMocks() + }) + + it.each([ + [true, true], + [false, false], + [false, undefined] + ])('should call the govUkPayApi with recurring as %s if the argument is %s', async (expected, value) => { + const mockResponse = { + ok: true, + json: jest.fn().mockResolvedValue({ success: true, paymentId: 'abc123' }) + } + govUkPayApi.createPayment.mockResolvedValue(mockResponse) + const unique = Symbol('payload') + const payload = { unique } + await sendPayment(payload, value) + expect(govUkPayApi.createPayment).toHaveBeenCalledWith(payload, expected) + }) + + it('should send provided payload data to Gov.UK Pay', async () => { + const mockResponse = { + ok: true, + json: jest.fn().mockResolvedValue({ success: true, paymentId: 'abc123' }) + } + govUkPayApi.createPayment.mockResolvedValue(mockResponse) + const unique = Symbol('payload') + const payload = { + reference: 'd81f1a2b-6508-468f-8342-b6770f60f7cd', + description: 'Fishing permission', + user_identifier: '1218c1c5-38e4-4bf3-81ea-9cbce3994d30', + unique + } + await sendPayment(payload) + expect(govUkPayApi.createPayment).toHaveBeenCalledWith(payload, false) + }) + + it('should return response body when payment creation is successful', async () => { + const mockResponse = { + ok: true, + json: jest.fn().mockResolvedValue({ success: true, paymentId: 'abc123' }) + } + govUkPayApi.createPayment.mockResolvedValue(mockResponse) + + const result = await sendPayment(preparedPayment) + + expect(result).toEqual({ success: true, paymentId: 'abc123' }) + }) + + it('should log debug message when response.ok is true', async () => { + const mockResponse = { + ok: true, + json: jest.fn().mockResolvedValue({ success: true, paymentId: 'abc123' }) + } + govUkPayApi.createPayment.mockResolvedValue(mockResponse) + + await sendPayment(preparedPayment) + + expect(debug).toHaveBeenCalledWith('Successful payment creation response: %o', { success: true, paymentId: 'abc123' }) + }) + + it('should log error message when response.ok is false', async () => { + const mockResponse = { + ok: false, + status: 500, + json: jest.fn().mockResolvedValue({ message: 'Server error' }) + } + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}) + govUkPayApi.createPayment.mockResolvedValue(mockResponse) + + try { + await sendPayment(preparedPayment) + } catch (error) { + expect(consoleErrorSpy).toHaveBeenCalledWith('Failure creating payment in the GOV.UK API service', { + transactionId: preparedPayment.id, + method: 'POST', + payload: preparedPayment, + status: mockResponse.status, + response: { message: 'Server error' } + }) + } + }) + + it('should throw error when API call fails with network issue', async () => { + const mockError = new Error('Network error') + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(jest.fn()) + govUkPayApi.createPayment.mockRejectedValue(mockError) + + try { + await sendPayment(preparedPayment) + } catch (error) { + expect(consoleErrorSpy).toHaveBeenCalledWith( + `Error creating payment in the GOV.UK API service - tid: ${preparedPayment.id}`, + mockError + ) + } + }) + + it('should throw error for when rate limit is breached', async () => { + const mockResponse = { + ok: false, + status: 429, + json: jest.fn().mockResolvedValue({ message: 'Rate limit exceeded' }) + } + const consoleErrorSpy = jest.spyOn(console, 'info').mockImplementation(jest.fn()) + govUkPayApi.createPayment.mockResolvedValue(mockResponse) + + try { + await sendPayment(preparedPayment) + } catch (error) { + expect(consoleErrorSpy).toHaveBeenCalledWith(`GOV.UK Pay API rate limit breach - tid: ${preparedPayment.id}`) + } + }) + + it('should throw error for unexpected response status', async () => { + const mockResponse = { + ok: false, + status: 500, + json: jest.fn().mockResolvedValue({ message: 'Server error' }) + } + govUkPayApi.createPayment.mockResolvedValue(mockResponse) + + try { + await sendPayment(preparedPayment) + } catch (error) { + expect(error.message).toBe('Unexpected response from GOV.UK pay API') + } + }) + }) + describe('sendRecurringPayment', () => { const preparedPayment = { id: '1234', @@ -277,4 +413,120 @@ describe('The govuk-pay-service', () => { } }) }) + + describe('getPaymentStatus', () => { + const paymentId = '1234' + + beforeEach(() => { + jest.clearAllMocks() + }) + + it.each([ + [true, true], + [false, false], + [false, undefined] + ])('should call the govUkPayApi with recurring as %s if the argument is %s', async (expected, value) => { + const mockResponse = { ok: true, status: 200, json: () => {} } + govUkPayApi.fetchPaymentStatus.mockResolvedValue(mockResponse) + await getPaymentStatus(paymentId, value) + expect(govUkPayApi.fetchPaymentStatus).toHaveBeenCalledWith(paymentId, expected) + }) + + it('should send provided paymentId to Gov.UK Pay', async () => { + const mockResponse = { ok: true, status: 200, json: () => {} } + govUkPayApi.fetchPaymentStatus.mockResolvedValue(mockResponse) + await getPaymentStatus(paymentId) + expect(govUkPayApi.fetchPaymentStatus).toHaveBeenCalledWith(paymentId, false) + }) + + it('should return response body when payment status check is successful', async () => { + const resBody = Symbol('body') + const mockResponse = { ok: true, status: 200, json: jest.fn().mockResolvedValue(resBody) } + govUkPayApi.fetchPaymentStatus.mockResolvedValue(mockResponse) + + const result = await getPaymentStatus(paymentId) + + expect(result).toEqual(resBody) + }) + + it('should log debug message when response.ok is true', async () => { + const resBody = Symbol('body') + const mockResponse = { ok: true, status: 200, json: jest.fn().mockResolvedValue(resBody) } + govUkPayApi.fetchPaymentStatus.mockResolvedValue(mockResponse) + + await getPaymentStatus(paymentId) + + expect(debug).toHaveBeenCalledWith('Payment status response: %o', resBody) + }) + + it('should log error message when response.ok is false', async () => { + const mockResponse = { + ok: false, + status: 500, + json: jest.fn().mockResolvedValue({ message: 'Server error' }) + } + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}) + govUkPayApi.fetchPaymentStatus.mockResolvedValue(mockResponse) + + try { + await getPaymentStatus(paymentId) + } catch (error) { + expect(consoleErrorSpy).toHaveBeenCalledWith( + `Error retrieving the payment status from the GOV.UK API service - tid: ${paymentId}`, + { + method: 'GET', + paymentId: paymentId, + status: mockResponse.status, + response: { message: 'Server error' } + } + ) + } + }) + + it('should throw error when API call fails with network issue', async () => { + const mockError = new Error('Network error') + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(jest.fn()) + govUkPayApi.fetchPaymentStatus.mockRejectedValue(mockError) + + try { + await getPaymentStatus(paymentId) + } catch (error) { + expect(consoleErrorSpy).toHaveBeenCalledWith( + `Error retrieving the payment status from the GOV.UK API service - paymentId: ${paymentId}`, + mockError + ) + } + }) + + it('should throw error for when rate limit is breached', async () => { + const mockResponse = { + ok: false, + status: 429, + json: jest.fn().mockResolvedValue({ message: 'Rate limit exceeded' }) + } + const consoleErrorSpy = jest.spyOn(console, 'info').mockImplementation(jest.fn()) + govUkPayApi.fetchPaymentStatus.mockResolvedValue(mockResponse) + + try { + await getPaymentStatus(paymentId) + } catch (error) { + expect(consoleErrorSpy).toHaveBeenCalledWith(`GOV.UK Pay API rate limit breach - paymentId: ${paymentId}`) + } + }) + + it('should throw error for unexpected response status', async () => { + const mockResponse = { + ok: false, + status: 500, + json: jest.fn().mockResolvedValue({ message: 'Server error' }) + } + govUkPayApi.fetchPaymentStatus.mockResolvedValue(mockResponse) + + try { + await getPaymentStatus(paymentId) + } catch (error) { + expect(error.message).toBe('Unexpected response from GOV.UK pay API') + } + }) + }) }) diff --git a/packages/gafl-webapp-service/src/services/payment/govuk-pay-service.js b/packages/gafl-webapp-service/src/services/payment/govuk-pay-service.js index c0fabc3c73..80bedb845f 100644 --- a/packages/gafl-webapp-service/src/services/payment/govuk-pay-service.js +++ b/packages/gafl-webapp-service/src/services/payment/govuk-pay-service.js @@ -46,10 +46,10 @@ const getTransactionErrorMessage = async (transactionId, payload, response) => ( * @param preparedPayment - the prepared payload for the payment. See in processors/payment.js * @returns {Promise<*>} */ -export const sendPayment = async preparedPayment => { +export const sendPayment = async (preparedPayment, recurring = false) => { let response try { - response = await govUkPayApi.createPayment(preparedPayment) + response = await govUkPayApi.createPayment(preparedPayment, recurring) } catch (err) { /* * Potentially errors caught here (unreachable, timeouts) may be retried - set origin on the error to indicate @@ -78,11 +78,11 @@ export const sendPayment = async preparedPayment => { * @param paymentId - the paymentId * @returns {Promise} */ -export const getPaymentStatus = async paymentId => { +export const getPaymentStatus = async (paymentId, recurring = false) => { debug(`Get payment status for paymentId: ${paymentId}`) let response try { - response = await govUkPayApi.fetchPaymentStatus(paymentId) + response = await govUkPayApi.fetchPaymentStatus(paymentId, recurring) } catch (err) { /* * Errors caught here (unreachable, timeouts) may be retried - set origin on the error to indicate diff --git a/packages/pocl-job/package-lock.json b/packages/pocl-job/package-lock.json index 69437ad1fc..c0d57c036c 100644 --- a/packages/pocl-job/package-lock.json +++ b/packages/pocl-job/package-lock.json @@ -17,8 +17,7 @@ "md5-file": "^5.0.0", "moment": "^2.29.1", "moment-timezone": "^0.5.34", - "sax-stream": "^1.3.0", - "ssh2-sftp-client": "^6.0.1" + "sax-stream": "^1.3.0" }, "engines": { "node": ">=18.17" @@ -51,20 +50,10 @@ "node": ">=10" } }, - "node_modules/@dabh/diagnostics": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz", - "integrity": "sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==", - "dependencies": { - "colorspace": "1.1.x", - "enabled": "2.0.x", - "kuler": "^2.0.0" - } - }, "node_modules/@defra-fish/business-rules-lib": { - "version": "1.50.0-rc.7", - "resolved": "https://registry.npmjs.org/@defra-fish/business-rules-lib/-/business-rules-lib-1.50.0-rc.7.tgz", - "integrity": "sha512-Zc0wkDOx5H4rADDl1TYtVK3HIFl8uB5udcL0VmlXrCGFG3SOZEgeGElR4I0Y4rEz3Y4F/eP/uYSzscf1HGIsCg==", + "version": "1.50.0-rc.8", + "resolved": "https://registry.npmjs.org/@defra-fish/business-rules-lib/-/business-rules-lib-1.50.0-rc.8.tgz", + "integrity": "sha512-Xo+lI+8DON5vahQKQe8ZX1fLpmE2TQm7eThYaFeUxBAEwbhhP7u7l3Ck49YR9hzWsAUWMLWGKgUrWAhCdId/3g==", "dependencies": { "joi": "^17.6.0", "moment": "^2.29.1", @@ -75,9 +64,9 @@ } }, "node_modules/@defra-fish/connectors-lib": { - "version": "1.50.0-rc.7", - "resolved": "https://registry.npmjs.org/@defra-fish/connectors-lib/-/connectors-lib-1.50.0-rc.7.tgz", - "integrity": "sha512-DV8J4CnxRuYZTfbX8JCkGhm1gQ4hgScIxdRuoy/X5THMIPaLgDSxBkOH90UT/2D61U8A5QGVfgKjPuKSisfGrA==", + "version": "1.50.0-rc.8", + "resolved": "https://registry.npmjs.org/@defra-fish/connectors-lib/-/connectors-lib-1.50.0-rc.8.tgz", + "integrity": "sha512-aXPMRNfXBF6xytV+TD7C3fbeEmMb0m2VPsaDiq4uN4ShqqTg2q3OuAo6YUZypKOkX2tH6/Hj4Qpb3UMerNJ6hg==", "dependencies": { "@airbrake/node": "^2.1.7", "aws-sdk": "^2.1074.0", @@ -155,19 +144,6 @@ "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" }, - "node_modules/asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "dependencies": { - "safer-buffer": "~2.1.0" - } - }, - "node_modules/async": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz", - "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==" - }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -240,14 +216,6 @@ } ] }, - "node_modules/bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "dependencies": { - "tweetnacl": "^0.14.3" - } - }, "node_modules/bintrees": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", @@ -268,11 +236,6 @@ "isarray": "^1.0.0" } }, - "node_modules/buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" - }, "node_modules/call-bind": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", @@ -299,54 +262,6 @@ "node": ">=0.10.0" } }, - "node_modules/color": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", - "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==", - "dependencies": { - "color-convert": "^1.9.1", - "color-string": "^1.5.2" - } - }, - "node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" - }, - "node_modules/color-string": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.4.tgz", - "integrity": "sha512-57yF5yt8Xa3czSEW1jfQDE79Idk0+AkN/4KWad6tbdxUmAs3MvjxlWSWD4deYytcRfoZ9nhKyFl1kj5tBvidbw==", - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, - "node_modules/colors": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", - "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/colorspace": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz", - "integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==", - "dependencies": { - "color": "3.0.x", - "text-hex": "1.0.x" - } - }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -366,25 +281,6 @@ "node": ">= 10" } }, - "node_modules/concat-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", - "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "engines": [ - "node >= 6.0" - ], - "dependencies": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.0.2", - "typedarray": "^0.0.6" - } - }, - "node_modules/core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, "node_modules/cross-fetch": { "version": "3.1.8", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", @@ -441,16 +337,6 @@ "node": ">=0.10" } }, - "node_modules/enabled": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", - "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" - }, - "node_modules/err-code": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", - "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==" - }, "node_modules/error-stack-parser": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.1.4.tgz", @@ -486,16 +372,6 @@ "node": ">=0.4.x" } }, - "node_modules/fast-safe-stringify": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", - "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" - }, - "node_modules/fecha": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.0.tgz", - "integrity": "sha512-aN3pcx/DSmtyoovUudctc8+6Hl4T+hI9GBBHLjA76jdZl7+b1sgh5g4k+u/GL3dTy1/pnYzKp69FpJ0OicE3Wg==" - }, "node_modules/filesize": { "version": "6.4.0", "resolved": "https://registry.npmjs.org/filesize/-/filesize-6.4.0.tgz", @@ -504,11 +380,6 @@ "node": ">= 0.4.0" } }, - "node_modules/fn.name": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", - "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" - }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -675,11 +546,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -705,14 +571,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", - "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==", - "engines": { - "node": ">=8" - } - }, "node_modules/is-typed-array": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", @@ -752,11 +610,6 @@ "@sideway/pinpoint": "^2.0.0" } }, - "node_modules/kuler": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", - "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" - }, "node_modules/lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", @@ -772,18 +625,6 @@ "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" }, - "node_modules/logform": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz", - "integrity": "sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg==", - "dependencies": { - "colors": "^1.2.1", - "fast-safe-stringify": "^2.0.4", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "triple-beam": "^1.3.0" - } - }, "node_modules/md5-file": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/md5-file/-/md5-file-5.0.0.tgz", @@ -857,14 +698,6 @@ } } }, - "node_modules/one-time": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", - "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", - "dependencies": { - "fn.name": "1.x.x" - } - }, "node_modules/p-map": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", @@ -881,28 +714,11 @@ "node": ">= 0.4" } }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, "node_modules/promise-polyfill": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/promise-polyfill/-/promise-polyfill-8.3.0.tgz", "integrity": "sha512-H5oELycFml5yto/atYqmjyigJoAo3+OXwolYiH7OfQuYlAqhxNvTfiNMbV9hsC6Yp83yE5r2KTVmtrG6R9i6Pg==" }, - "node_modules/promise-retry": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", - "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", - "dependencies": { - "err-code": "^2.0.2", - "retry": "^0.12.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -917,19 +733,6 @@ "node": ">=0.4.x" } }, - "node_modules/readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/redis-commands": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", @@ -965,38 +768,6 @@ "node": ">=8.0.0" } }, - "node_modules/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=", - "engines": { - "node": ">= 4" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, "node_modules/sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", @@ -1040,57 +811,6 @@ "node": ">= 0.4" } }, - "node_modules/simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=", - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, - "node_modules/ssh2": { - "version": "0.8.9", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-0.8.9.tgz", - "integrity": "sha512-GmoNPxWDMkVpMFa9LVVzQZHF6EW3WKmBwL+4/GeILf2hFmix5Isxm7Amamo8o7bHiU0tC+wXsGcUXOxp8ChPaw==", - "dependencies": { - "ssh2-streams": "~0.4.10" - }, - "engines": { - "node": ">=5.2.0" - } - }, - "node_modules/ssh2-sftp-client": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-6.0.1.tgz", - "integrity": "sha512-Glut2SmK/XpNOBiEuzqlKZGKkIyha2XMbuWVXR2hFUJkNsbyl/wmlZSeUEPxKFp/dC9UEvUKzanKydgLmNdfkw==", - "dependencies": { - "concat-stream": "^2.0.0", - "promise-retry": "^2.0.1", - "ssh2": "^0.8.9", - "winston": "^3.3.3" - } - }, - "node_modules/ssh2-streams": { - "version": "0.4.10", - "resolved": "https://registry.npmjs.org/ssh2-streams/-/ssh2-streams-0.4.10.tgz", - "integrity": "sha512-8pnlMjvnIZJvmTzUIIA5nT4jr2ZWNNVHwyXfMGdRJbug9TpI3kd99ffglgfSWqujVv/0gxwMsDn9j9RVst8yhQ==", - "dependencies": { - "asn1": "~0.2.0", - "bcrypt-pbkdf": "^1.0.2", - "streamsearch": "~0.1.2" - }, - "engines": { - "node": ">=5.2.0" - } - }, - "node_modules/stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=", - "engines": { - "node": "*" - } - }, "node_modules/stackframe": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.3.4.tgz", @@ -1101,22 +821,6 @@ "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" }, - "node_modules/streamsearch": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", - "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, "node_modules/tdigest": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", @@ -1125,31 +829,11 @@ "bintrees": "1.0.2" } }, - "node_modules/text-hex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", - "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" - }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, - "node_modules/triple-beam": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", - "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" - }, - "node_modules/tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" - }, - "node_modules/typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" - }, "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", @@ -1176,11 +860,6 @@ "which-typed-array": "^1.1.2" } }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, "node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -1221,64 +900,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/winston": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/winston/-/winston-3.3.3.tgz", - "integrity": "sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw==", - "dependencies": { - "@dabh/diagnostics": "^2.0.2", - "async": "^3.1.0", - "is-stream": "^2.0.0", - "logform": "^2.2.0", - "one-time": "^1.0.0", - "readable-stream": "^3.4.0", - "stack-trace": "0.0.x", - "triple-beam": "^1.3.0", - "winston-transport": "^4.4.0" - }, - "engines": { - "node": ">= 6.4.0" - } - }, - "node_modules/winston-transport": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.0.tgz", - "integrity": "sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw==", - "dependencies": { - "readable-stream": "^2.3.7", - "triple-beam": "^1.2.0" - }, - "engines": { - "node": ">= 6.4.0" - } - }, - "node_modules/winston-transport/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/winston-transport/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/winston-transport/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, "node_modules/xml2js": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", diff --git a/packages/pocl-job/package.json b/packages/pocl-job/package.json index 751262432c..65ee5e509d 100644 --- a/packages/pocl-job/package.json +++ b/packages/pocl-job/package.json @@ -43,7 +43,6 @@ "md5-file": "^5.0.0", "moment": "^2.29.1", "moment-timezone": "^0.5.34", - "sax-stream": "^1.3.0", - "ssh2-sftp-client": "^6.0.1" + "sax-stream": "^1.3.0" } } diff --git a/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js b/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js deleted file mode 100644 index f24599f0b1..0000000000 --- a/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js +++ /dev/null @@ -1,11 +0,0 @@ -const ssh2sftpClient = jest.genMockFromModule('ssh2-sftp-client') - -export const mockedFtpMethods = { - connect: jest.fn(), - list: jest.fn(), - fastGet: jest.fn(), - delete: jest.fn(), - end: jest.fn() -} -ssh2sftpClient.mockImplementation(() => mockedFtpMethods) -export default ssh2sftpClient diff --git a/packages/pocl-job/src/__tests__/config.spec.js b/packages/pocl-job/src/__tests__/config.spec.js index f0c1c6d8eb..5fd427d0f6 100644 --- a/packages/pocl-job/src/__tests__/config.spec.js +++ b/packages/pocl-job/src/__tests__/config.spec.js @@ -11,11 +11,6 @@ describe('config', () => { process.env.POCL_RECORD_STAGING_TABLE = 'test-record-staging-table' process.env.POCL_STAGING_TTL = 1234 - process.env.POCL_FTP_HOST = 'test-host' - process.env.POCL_FTP_PORT = 2222 - process.env.POCL_FTP_PATH = '/remote/share' - process.env.POCL_FTP_USERNAME = 'test-user' - process.env.POCL_FTP_KEY_SECRET_ID = 'test-secret-id' process.env.POCL_S3_BUCKET = 'test-bucket' await config.initialise() }) @@ -38,32 +33,6 @@ describe('config', () => { }) }) - describe('ftp', () => { - it('provides properties relating the use of SFTP', async () => { - expect(config.ftp).toEqual( - expect.objectContaining({ - host: 'test-host', - port: '2222', - path: '/remote/share', - username: 'test-user', - privateKey: 'test-ssh-key', - algorithms: { cipher: expect.any(Array), kex: expect.any(Array) }, - // Wait up to 60 seconds for the SSH handshake - readyTimeout: expect.any(Number), - // Retry 5 times over a minute - retries: expect.any(Number), - retry_minTimeout: expect.any(Number), - debug: expect.any(Function) - }) - ) - }) - it('defaults the sftp port to 22 if the environment variable is not configured', async () => { - delete process.env.POCL_FTP_PORT - await config.initialise() - expect(config.ftp.port).toEqual('22') - }) - }) - describe('s3', () => { it('provides properties relating the use of Amazon S3', async () => { expect(config.s3.bucket).toEqual('test-bucket') diff --git a/packages/pocl-job/src/__tests__/pocl-processor.spec.js b/packages/pocl-job/src/__tests__/pocl-processor.spec.js index f3a5e4696a..ade97101d7 100644 --- a/packages/pocl-job/src/__tests__/pocl-processor.spec.js +++ b/packages/pocl-job/src/__tests__/pocl-processor.spec.js @@ -41,7 +41,7 @@ jest.mock('../config.js', () => ({ bucket: 'testbucket' } })) -jest.mock('../transport/ftp-to-s3.js') +jest.mock('../transport/storeS3MetaData.js') jest.mock('../transport/s3-to-local.js') jest.mock('../io/db.js') jest.mock('../io/s3.js') diff --git a/packages/pocl-job/src/config.js b/packages/pocl-job/src/config.js index 0430b4a457..d08132335b 100644 --- a/packages/pocl-job/src/config.js +++ b/packages/pocl-job/src/config.js @@ -1,49 +1,5 @@ -import { AWS } from '@defra-fish/connectors-lib' -import db from 'debug' -const { secretsManager } = AWS() - -/** - * Key exchange algorithms for public key authentication - in descending order of priority - * @type {string[]} - */ -export const SFTP_KEY_EXCHANGE_ALGORITHMS = [ - 'curve25519-sha256@libssh.org', - 'curve25519-sha256', - 'ecdh-sha2-nistp521', - 'ecdh-sha2-nistp384', - 'ecdh-sha2-nistp256', - 'diffie-hellman-group-exchange-sha256', - 'diffie-hellman-group14-sha256', - 'diffie-hellman-group16-sha512', - 'diffie-hellman-group18-sha512', - 'diffie-hellman-group14-sha1', - 'diffie-hellman-group-exchange-sha1', - 'diffie-hellman-group1-sha1' -] -/** - * Ciphers for SFTP support - in descending order of priority - * @type {string[]} - */ -export const SFTP_CIPHERS = [ - // http://tools.ietf.org/html/rfc4344#section-4 - 'aes256-ctr', - 'aes192-ctr', - 'aes128-ctr', - 'aes256-gcm', - 'aes256-gcm@openssh.com', - 'aes128-gcm', - 'aes128-gcm@openssh.com', - 'aes256-cbc', - 'aes192-cbc', - 'aes128-cbc', - 'blowfish-cbc', - '3des-cbc', - 'cast128-cbc' -] - class Config { _db - _ftp _s3 async initialise () { @@ -53,21 +9,6 @@ class Config { stagingTtlDelta: Number.parseInt(process.env.POCL_STAGING_TTL || 60 * 60 * 168) } - this.ftp = { - host: process.env.POCL_FTP_HOST, - port: process.env.POCL_FTP_PORT || '22', - path: process.env.POCL_FTP_PATH, - username: process.env.POCL_FTP_USERNAME, - privateKey: (await secretsManager.getSecretValue({ SecretId: process.env.POCL_FTP_KEY_SECRET_ID }).promise()).SecretString, - algorithms: { cipher: SFTP_CIPHERS, kex: SFTP_KEY_EXCHANGE_ALGORITHMS }, - // Wait up to 60 seconds for the SSH handshake - readyTimeout: 60000, - // Retry 5 times over a minute - retries: 5, - retry_minTimeout: 12000, - debug: db('pocl:ftp') - } - this.s3 = { bucket: process.env.POCL_S3_BUCKET } @@ -85,18 +26,6 @@ class Config { this._db = cfg } - /** - * FTP configuration settings - * @type {object} - */ - get ftp () { - return this._ftp - } - - set ftp (cfg) { - this._ftp = cfg - } - /** * S3 configuration settings * @type {object} diff --git a/packages/pocl-job/src/io/__tests__/s3.spec.js b/packages/pocl-job/src/io/__tests__/s3.spec.js index 4a1a2da226..26c7fe9be1 100644 --- a/packages/pocl-job/src/io/__tests__/s3.spec.js +++ b/packages/pocl-job/src/io/__tests__/s3.spec.js @@ -5,7 +5,6 @@ import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../.. import { salesApi } from '@defra-fish/connectors-lib' import fs from 'fs' import AwsMock from 'aws-sdk' -import { mockedFtpMethods } from 'ssh2-sftp-client' jest.mock('fs') jest.mock('md5-file') @@ -159,7 +158,6 @@ describe('s3 operations', () => { }) it('skips file processing if a file has already been marked as processed in Dynamics', async () => { - mockedFtpMethods.list.mockResolvedValue([{ name: 'test-already-processed.xml' }]) fs.createReadStream.mockReturnValueOnce('teststream') fs.statSync.mockReturnValueOnce({ size: 1024 }) salesApi.getTransactionFile.mockResolvedValueOnce({ status: { description: 'Processed' } }) diff --git a/packages/pocl-job/src/io/s3.js b/packages/pocl-job/src/io/s3.js index c24d3ba3d0..66a97cdbfb 100644 --- a/packages/pocl-job/src/io/s3.js +++ b/packages/pocl-job/src/io/s3.js @@ -2,7 +2,7 @@ import moment from 'moment' import filesize from 'filesize' import config from '../config.js' import { DYNAMICS_IMPORT_STAGE } from '../staging/constants.js' -import { storeS3Metadata } from '../transport/ftp-to-s3.js' +import { storeS3Metadata } from '../transport/storeS3MetaData.js' import { AWS, salesApi } from '@defra-fish/connectors-lib' const { s3 } = AWS() diff --git a/packages/pocl-job/src/transport/__tests__/ftp-to-s3.spec.js b/packages/pocl-job/src/transport/__tests__/ftp-to-s3.spec.js deleted file mode 100644 index 7b5259da27..0000000000 --- a/packages/pocl-job/src/transport/__tests__/ftp-to-s3.spec.js +++ /dev/null @@ -1,145 +0,0 @@ -import { ftpToS3 } from '../ftp-to-s3.js' -import moment from 'moment' -import { updateFileStagingTable } from '../../io/db.js' -import { getTempDir } from '../../io/file.js' -import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../../staging/constants.js' -import { salesApi } from '@defra-fish/connectors-lib' -import fs from 'fs' -import md5File from 'md5-file' -import AwsMock from 'aws-sdk' -import { mockedFtpMethods } from 'ssh2-sftp-client' - -jest.mock('fs') -jest.mock('md5-file') -jest.mock('../../io/db.js') -jest.mock('../../io/file.js') - -jest.mock('@defra-fish/connectors-lib', () => { - const actual = jest.requireActual('@defra-fish/connectors-lib') - return { - AWS: actual.AWS, - salesApi: { - ...Object.keys(actual.salesApi).reduce((acc, k) => ({ ...acc, [k]: jest.fn(async () => {}) }), {}) - } - } -}) - -jest.mock('../../config.js', () => ({ - ftp: { - path: '/ftpservershare/' - }, - s3: { - bucket: 'testbucket' - } -})) - -describe('ftp-to-s3', () => { - beforeAll(() => { - getTempDir.mockReturnValue('/local/tmp') - md5File.mockResolvedValue('example-md5') - }) - beforeEach(() => { - jest.clearAllMocks() - AwsMock.__resetAll() - }) - - it('retrieves files from SFTP and stores in S3', async () => { - mockedFtpMethods.list.mockResolvedValue([{ name: 'test1.xml' }, { name: 'test2.xml' }]) - fs.createReadStream.mockReturnValueOnce('test1stream') - fs.createReadStream.mockReturnValueOnce('test2stream') - fs.statSync.mockReturnValueOnce({ size: 1024 }) - fs.statSync.mockReturnValueOnce({ size: 2048 }) - await ftpToS3() - - const localPath1 = '/local/tmp/test1.xml' - const localPath2 = '/local/tmp/test2.xml' - - const s3Key1 = `${moment().format('YYYY-MM-DD')}/test1.xml` - const s3Key2 = `${moment().format('YYYY-MM-DD')}/test2.xml` - - expect(mockedFtpMethods.fastGet).toHaveBeenNthCalledWith(1, '/ftpservershare/test1.xml', localPath1, {}) - expect(mockedFtpMethods.fastGet).toHaveBeenNthCalledWith(2, '/ftpservershare/test2.xml', localPath2, {}) - expect(AwsMock.S3.mockedMethods.putObject).toHaveBeenNthCalledWith(1, { - Bucket: 'testbucket', - Key: s3Key1, - Body: 'test1stream' - }) - expect(AwsMock.S3.mockedMethods.putObject).toHaveBeenNthCalledWith(2, { - Bucket: 'testbucket', - Key: s3Key2, - Body: 'test2stream' - }) - expect(updateFileStagingTable).toHaveBeenNthCalledWith(1, { - filename: 'test1.xml', - md5: 'example-md5', - fileSize: '1 KB', - stage: FILE_STAGE.Pending, - s3Key: s3Key1 - }) - expect(updateFileStagingTable).toHaveBeenNthCalledWith(2, { - filename: 'test2.xml', - md5: 'example-md5', - fileSize: '2 KB', - stage: FILE_STAGE.Pending, - s3Key: s3Key2 - }) - expect(salesApi.upsertTransactionFile).toHaveBeenNthCalledWith(1, 'test1.xml', { - status: DYNAMICS_IMPORT_STAGE.Pending, - dataSource: POST_OFFICE_DATASOURCE, - fileSize: '1 KB', - receiptTimestamp: expect.any(String), - salesDate: expect.any(String), - notes: 'Retrieved from the remote server and awaiting processing' - }) - expect(salesApi.upsertTransactionFile).toHaveBeenNthCalledWith(2, 'test2.xml', { - status: DYNAMICS_IMPORT_STAGE.Pending, - dataSource: POST_OFFICE_DATASOURCE, - fileSize: '2 KB', - receiptTimestamp: expect.any(String), - salesDate: expect.any(String), - notes: 'Retrieved from the remote server and awaiting processing' - }) - expect(fs.unlinkSync).toHaveBeenNthCalledWith(1, localPath1) - expect(fs.unlinkSync).toHaveBeenNthCalledWith(2, localPath2) - expect(mockedFtpMethods.end).toHaveBeenCalledTimes(1) - }) - - it('moves the file to s3 but skips file processing if a file has already been marked as processed in Dynamics', async () => { - mockedFtpMethods.list.mockResolvedValue([{ name: 'test-already-processed.xml' }]) - fs.createReadStream.mockReturnValueOnce('teststream') - fs.statSync.mockReturnValueOnce({ size: 1024 }) - salesApi.getTransactionFile.mockResolvedValueOnce({ status: { description: 'Processed' } }) - const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(jest.fn()) - await ftpToS3() - const localPath = '/local/tmp/test-already-processed.xml' - const s3Key = `${moment().format('YYYY-MM-DD')}/test-already-processed.xml` - expect(mockedFtpMethods.fastGet).toHaveBeenCalledWith('/ftpservershare/test-already-processed.xml', localPath, {}) - expect(AwsMock.S3.mockedMethods.putObject).toHaveBeenCalledWith({ - Bucket: 'testbucket', - Key: s3Key, - Body: 'teststream' - }) - expect(updateFileStagingTable).not.toHaveBeenCalled() - expect(salesApi.upsertTransactionFile).not.toHaveBeenCalled() - expect(consoleErrorSpy).toHaveBeenCalled() - expect(fs.unlinkSync).toHaveBeenCalledWith(localPath) - expect(mockedFtpMethods.end).toHaveBeenCalledTimes(1) - }) - - it('logs and propogates errors back up the stack', async () => { - const testError = new Error('Test error') - mockedFtpMethods.list.mockRejectedValue(testError) - const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}) - await expect(ftpToS3).rejects.toThrow(testError) - expect(consoleErrorSpy).toHaveBeenCalled() - }) - - it('ignores non-xml files', async () => { - mockedFtpMethods.list.mockResolvedValue([{ name: 'test1.pdf' }, { name: 'test2.md' }]) - await ftpToS3() - expect(mockedFtpMethods.fastGet).not.toHaveBeenCalled() - expect(AwsMock.S3.mockedMethods.putObject).not.toHaveBeenCalled() - expect(fs.unlinkSync).not.toHaveBeenCalled() - expect(mockedFtpMethods.end).toHaveBeenCalledTimes(1) - }) -}) diff --git a/packages/pocl-job/src/transport/__tests__/storeS3MetaData.spec.js b/packages/pocl-job/src/transport/__tests__/storeS3MetaData.spec.js new file mode 100644 index 0000000000..57d4434d65 --- /dev/null +++ b/packages/pocl-job/src/transport/__tests__/storeS3MetaData.spec.js @@ -0,0 +1,64 @@ +import moment from 'moment' +import { salesApi } from '@defra-fish/connectors-lib' +import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../../staging/constants.js' +import { updateFileStagingTable } from '../../io/db.js' +import { storeS3Metadata } from '../storeS3MetaData.js' + +jest.mock('../../io/db.js', () => ({ + updateFileStagingTable: jest.fn() +})) +jest.mock('@defra-fish/connectors-lib', () => ({ + salesApi: { + upsertTransactionFile: jest.fn() + } +})) + +describe('storeS3Metadata', () => { + const md5 = 'mockMd5Hash' + const fileSize = 12345 + const filename = 'testfile' + const s3Key = 'mock/s3/key/testfile' + const receiptMoment = new Date('2024-10-17T00:00:00Z') + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('console log should output "Storing metadata for s3Key"', async () => { + const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(jest.fn()) + await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) + expect(consoleLogSpy).toHaveBeenCalledWith(`Storing metadata for ${s3Key}`) + }) + + it('should call updateFileStagingTable with correct arguments', async () => { + await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) + expect(updateFileStagingTable).toHaveBeenCalledWith({ + filename, + md5, + fileSize, + s3Key, + stage: FILE_STAGE.Pending + }) + }) + + it('should call salesApi.upsertTransactionFile with correct arguments', async () => { + const expectedSalesDate = moment(receiptMoment).subtract(1, 'days').toISOString() + const expectedReceiptTimestamp = receiptMoment.toISOString() + + await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) + expect(salesApi.upsertTransactionFile).toHaveBeenCalledWith(filename, { + status: DYNAMICS_IMPORT_STAGE.Pending, + dataSource: POST_OFFICE_DATASOURCE, + fileSize: fileSize, + salesDate: expectedSalesDate, + receiptTimestamp: expectedReceiptTimestamp, + notes: 'Retrieved from the remote server and awaiting processing' + }) + }) + + test('should log "Stored metadata for s3Key"', async () => { + const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(jest.fn()) + await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) + expect(consoleLogSpy).toHaveBeenCalledWith(`Stored metadata for ${s3Key}`) + }) +}) diff --git a/packages/pocl-job/src/transport/ftp-to-s3.js b/packages/pocl-job/src/transport/ftp-to-s3.js deleted file mode 100644 index b3c9376448..0000000000 --- a/packages/pocl-job/src/transport/ftp-to-s3.js +++ /dev/null @@ -1,91 +0,0 @@ -import FtpClient from 'ssh2-sftp-client' -import moment from 'moment' -import Path from 'path' -import fs from 'fs' -import db from 'debug' -import md5File from 'md5-file' -import filesize from 'filesize' -import config from '../config.js' -import { getTempDir } from '../io/file.js' -import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../staging/constants.js' -import { AWS, salesApi } from '@defra-fish/connectors-lib' -import { updateFileStagingTable } from '../io/db.js' -const { s3 } = AWS() - -const debug = db('pocl:transport') -const sftp = new FtpClient() - -export async function ftpToS3 () { - try { - debug('Connecting to SFTP endpoint at sftp://%s:%s%s', config.ftp.host, config.ftp.port, config.ftp.path) - await sftp.connect(config.ftp) - const fileList = await sftp.list(config.ftp.path) - debug('Discovered the following files on the SFTP server: %o', fileList) - const xmlFiles = fileList.filter(f => Path.extname(f.name).toLowerCase() === '.xml') - - if (!xmlFiles.length) { - debug('No XML files were waiting to be processed on the SFTP server.') - } else { - await retrieveAllFiles(xmlFiles) - } - } catch (e) { - console.error('Error migrating files from the SFTP endpoint', e) - throw e - } finally { - debug('Closing SFTP connection.') - await sftp.end() - } -} - -export async function storeS3Metadata (md5, fileSize, filename, s3Key, receiptMoment) { - console.log(`Storing metadata for ${s3Key}`) - await updateFileStagingTable({ filename, md5, fileSize, s3Key, stage: FILE_STAGE.Pending }) - - await salesApi.upsertTransactionFile(filename, { - status: DYNAMICS_IMPORT_STAGE.Pending, - dataSource: POST_OFFICE_DATASOURCE, - fileSize: fileSize, - salesDate: moment(receiptMoment).subtract(1, 'days').toISOString(), - receiptTimestamp: receiptMoment.toISOString(), - notes: 'Retrieved from the remote server and awaiting processing' - }) - - console.log(`Stored metadata for ${s3Key}`) -} - -const retrieveAllFiles = async xmlFiles => { - const tempDir = getTempDir('ftp') - - for (const fileEntry of xmlFiles) { - const filename = fileEntry.name - const remoteFilePath = Path.join(config.ftp.path, filename) - const localFilePath = Path.resolve(tempDir, filename) - - // Retrieve from FTP server to local temporary directory - debug('Transferring %s to %s', remoteFilePath, localFilePath) - await sftp.fastGet(remoteFilePath, localFilePath, {}) - - // Transfer to S3 - const receiptMoment = moment() - const s3Key = Path.join(receiptMoment.format('YYYY-MM-DD'), filename) - debug('Transferring file to S3 bucket %s with key %s', config.s3.bucket, s3Key) - await s3.putObject({ Bucket: config.s3.bucket, Key: s3Key, Body: fs.createReadStream(localFilePath) }).promise() - - const dynamicsRecord = await salesApi.getTransactionFile(filename) - if (dynamicsRecord && DYNAMICS_IMPORT_STAGE.isAlreadyProcessed(dynamicsRecord.status.description)) { - console.error( - 'Retrieved file %s from SFTP and stored in S3, however an entry already exists in Dynamics with this filename. Skipping import.', - filename - ) - } else { - const md5 = await md5File(localFilePath) - const fileSize = filesize(fs.statSync(localFilePath).size) - await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) - } - - // Remove from FTP server and local tmp - debug('Removing remote file %s', remoteFilePath) - await sftp.delete(remoteFilePath) - fs.unlinkSync(localFilePath) - } -} diff --git a/packages/pocl-job/src/transport/storeS3MetaData.js b/packages/pocl-job/src/transport/storeS3MetaData.js new file mode 100644 index 0000000000..31eb6a4810 --- /dev/null +++ b/packages/pocl-job/src/transport/storeS3MetaData.js @@ -0,0 +1,20 @@ +import moment from 'moment' +import { salesApi } from '@defra-fish/connectors-lib' +import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../staging/constants.js' +import { updateFileStagingTable } from '../io/db.js' + +export async function storeS3Metadata (md5, fileSize, filename, s3Key, receiptMoment) { + console.log(`Storing metadata for ${s3Key}`) + await updateFileStagingTable({ filename, md5, fileSize, s3Key, stage: FILE_STAGE.Pending }) + + await salesApi.upsertTransactionFile(filename, { + status: DYNAMICS_IMPORT_STAGE.Pending, + dataSource: POST_OFFICE_DATASOURCE, + fileSize: fileSize, + salesDate: moment(receiptMoment).subtract(1, 'days').toISOString(), + receiptTimestamp: receiptMoment.toISOString(), + notes: 'Retrieved from the remote server and awaiting processing' + }) + + console.log(`Stored metadata for ${s3Key}`) +} diff --git a/packages/sales-api-service/src/schema/__tests__/transaction.schema.spec.js b/packages/sales-api-service/src/schema/__tests__/transaction.schema.spec.js index efa9eeb99a..ace8302b15 100644 --- a/packages/sales-api-service/src/schema/__tests__/transaction.schema.spec.js +++ b/packages/sales-api-service/src/schema/__tests__/transaction.schema.spec.js @@ -71,7 +71,7 @@ describe('createTransactionSchema', () => { await expect(createTransactionSchema.validateAsync(mockPayload)).rejects.toThrow('"permissions[0].isLicenceForYou" must be a boolean') }) - it('validates successfully when a uuid v4 transaction id is supplied', async () => { + it('validates successfully when a uuid v4 transactionId is supplied', async () => { const mockPayload = mockTransactionPayload() mockPayload.transactionId = '25fa0126-55da-4309-9bce-9957990d141e' await expect(createTransactionSchema.validateAsync(mockPayload)).resolves.not.toThrow() @@ -90,11 +90,34 @@ describe('createTransactionSchema', () => { ['uuid6 string', 'a3bb189e-8bf9-3888-9912-ace4e6543002'], ['uuid7 string', '01927705-ffac-77b5-89af-c97451b1bbe2'], ['numeric', 4567] - ])('fails validation when provided with a %s', async (_d, transactionId) => { + ])('fails validation when provided with a %s for transactionId', async (_d, transactionId) => { const mockPayload = mockTransactionPayload() mockPayload.transactionId = transactionId await expect(createTransactionSchema.validateAsync(mockPayload)).rejects.toThrow() }) + + it('validates successfully when an agreementId is supplied', async () => { + const mockPayload = mockTransactionPayload() + mockPayload.agreementId = 't3jl08v2nqqmujrnhs09pmhtjx' + await expect(createTransactionSchema.validateAsync(mockPayload)).resolves.not.toThrow() + }) + + it('validates successfully when agreementId is omitted', async () => { + const mockPayload = mockTransactionPayload() + await expect(createTransactionSchema.validateAsync(mockPayload)).resolves.not.toThrow() + }) + + it.each([ + ['too short string', 'foo'], + ['too long string', 'foobarbazfoobarbazfoobarbaz'], + ['string containing invalid characters', '!3j@08v2nqqmujrnhs09_mhtjx'], + ['null', null], + ['numeric', 4567] + ])('fails validation when provided with a %s for agreementId', async (_d, agreementId) => { + const mockPayload = mockTransactionPayload() + mockPayload.agreementId = agreementId + await expect(createTransactionSchema.validateAsync(mockPayload)).rejects.toThrow() + }) }) describe('createTransactionResponseSchema', () => { diff --git a/packages/sales-api-service/src/schema/transaction.schema.js b/packages/sales-api-service/src/schema/transaction.schema.js index b953239df9..44f9c61fc3 100644 --- a/packages/sales-api-service/src/schema/transaction.schema.js +++ b/packages/sales-api-service/src/schema/transaction.schema.js @@ -7,6 +7,8 @@ import { MAX_PERMISSIONS_PER_TRANSACTION, POCL_TRANSACTION_SOURCES } from '@defr import { v4 as uuidv4 } from 'uuid' +const AGREEMENT_ID_LENGTH = 26 + /** * Maximum number of items that can be created in a batch - limited by DynamoDB max batch size * @type {number} @@ -33,7 +35,8 @@ const createTransactionRequestSchemaContent = { }), createdBy: Joi.string().optional(), journalId: Joi.string().optional(), - transactionId: Joi.string().guid({ version: 'uuidv4' }) + transactionId: Joi.string().guid({ version: 'uuidv4' }).optional(), + agreementId: Joi.string().alphanum().length(AGREEMENT_ID_LENGTH).optional() } /**