Remove ftp functionality

packages/fulfilment-job/package.json

@@ -42,7 +42,6 @@
     "merge2": "^1.4.1",
     "moment": "^2.29.1",
     "openpgp": "^5.0.0-1",
-    "pluralize": "^8.0.0",
-    "ssh2-sftp-client": "^6.0.1"
+    "pluralize": "^8.0.0"
   }
 }

packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js

@@ -1,7 +1,6 @@
 import { Readable, PassThrough, Writable } from 'stream'
 import { deliverFulfilmentFiles } from '../deliver-fulfilment-files.js'
 import { createS3WriteStream, readS3PartFiles } from '../../transport/s3.js'
-import { createFtpWriteStream } from '../../transport/ftp.js'
 import { FULFILMENT_FILE_STATUS_OPTIONSET, getOptionSetEntry } from '../staging-common.js'
 import { FulfilmentRequestFile, executeQuery, persist } from '@defra-fish/dynamics-lib'
 import openpgp from 'openpgp'
@@ -10,7 +9,6 @@ import streamHelper from '../streamHelper.js'
 import merge2 from 'merge2'
 
 jest.mock('../../transport/s3.js')
-jest.mock('../../transport/ftp.js')
 jest.mock('openpgp', () => ({
   readKey: jest.fn(() => ({})),
   encrypt: jest.fn(({ message: readableStream }) => readableStream),
@@ -46,20 +44,10 @@ describe('deliverFulfilmentFiles', () => {
     executeQuery.mockResolvedValue([{ entity: mockFulfilmentRequestFile2 }, { entity: mockFulfilmentRequestFile1 }])
 
     // Streams for file1
-    const {
-      s3DataStreamFile: s3DataStreamFile1,
-      ftpDataStreamFile: ftpDataStreamFile1,
-      s3HashStreamFile: s3HashStreamFile1,
-      ftpHashStreamFile: ftpHashStreamFile1
-    } = createMockFileStreams()
+    const { s3DataStreamFile: s3DataStreamFile1, s3HashStreamFile: s3HashStreamFile1 } = createMockFileStreams()
 
     // Streams for file2
-    const {
-      s3DataStreamFile: s3DataStreamFile2,
-      ftpDataStreamFile: ftpDataStreamFile2,
-      s3HashStreamFile: s3HashStreamFile2,
-      ftpHashStreamFile: ftpHashStreamFile2
-    } = createMockFileStreams()
+    const { s3DataStreamFile: s3DataStreamFile2, s3HashStreamFile: s3HashStreamFile2 } = createMockFileStreams()
 
     // Run the delivery
     await expect(deliverFulfilmentFiles()).resolves.toBeUndefined()
@@ -69,22 +57,14 @@ describe('deliverFulfilmentFiles', () => {
     // File 1 expectations
     expect(createS3WriteStream).toHaveBeenNthCalledWith(1, 'EAFF202006180001.json')
     expect(createS3WriteStream).toHaveBeenNthCalledWith(3, 'EAFF202006180001.json.sha256')
-    expect(createFtpWriteStream).toHaveBeenNthCalledWith(1, 'EAFF202006180001.json')
-    expect(createFtpWriteStream).toHaveBeenNthCalledWith(3, 'EAFF202006180001.json.sha256')
     expect(JSON.parse(s3DataStreamFile1.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] })
-    expect(JSON.parse(ftpDataStreamFile1.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] })
     expect(s3HashStreamFile1.dataProcessed).toEqual(fileShaHash) // validated
-    expect(ftpHashStreamFile1.dataProcessed).toEqual(fileShaHash) // validated
 
     // File 2 expectations
     expect(createS3WriteStream).toHaveBeenNthCalledWith(4, 'EAFF202006180002.json')
     expect(createS3WriteStream).toHaveBeenNthCalledWith(6, 'EAFF202006180002.json.sha256')
-    expect(createFtpWriteStream).toHaveBeenNthCalledWith(4, 'EAFF202006180002.json')
-    expect(createFtpWriteStream).toHaveBeenNthCalledWith(6, 'EAFF202006180002.json.sha256')
     expect(JSON.parse(s3DataStreamFile2.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] })
-    expect(JSON.parse(ftpDataStreamFile2.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] })
     expect(s3HashStreamFile2.dataProcessed).toEqual(fileShaHash) // validated
-    expect(ftpHashStreamFile2.dataProcessed).toEqual(fileShaHash) // validated
 
     // Persist to dynamics for file 1
     expect(persist).toHaveBeenNthCalledWith(1, [
@@ -197,10 +177,7 @@ describe('deliverFulfilmentFiles', () => {
     const s3 = createTestableStream()
     streamHelper.pipelinePromise.mockResolvedValue()
     openpgp.encrypt.mockResolvedValue(s2)
-    merge2
-      .mockReturnValueOnce(s1)
-      .mockReturnValueOnce(s2)
-      .mockReturnValueOnce(s3)
+    merge2.mockReturnValueOnce(s1).mockReturnValueOnce(s2).mockReturnValueOnce(s3)
     await mockExecuteQuery()
     createMockFileStreams()
 
@@ -227,27 +204,18 @@ const createMockFulfilmentRequestFile = async (fileName, date) =>
 
 const createMockFileStreams = () => {
   const s3DataStreamFile = createTestableStream()
-  const ftpDataStreamFile = createTestableStream()
   createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3DataStreamFile, managedUpload: Promise.resolve() })
-  createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpDataStreamFile, managedUpload: Promise.resolve() })
 
   const s3EncryptedDataStreamFile = createTestableStream()
-  const ftpEncryptedDataStreamFile = createTestableStream()
   createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3EncryptedDataStreamFile, managedUpload: Promise.resolve() })
-  createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpEncryptedDataStreamFile, managedUpload: Promise.resolve() })
 
   const s3HashStreamFile = createTestableStream()
-  const ftpHashStreamFile = createTestableStream()
   createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3HashStreamFile, managedUpload: Promise.resolve() })
-  createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpHashStreamFile, managedUpload: Promise.resolve() })
 
   return {
     s3DataStreamFile,
-    ftpDataStreamFile,
     s3EncryptedDataStreamFile,
-    ftpEncryptedDataStreamFile,
-    s3HashStreamFile,
-    ftpHashStreamFile
+    s3HashStreamFile
   }
 }
 

packages/fulfilment-job/src/staging/deliver-fulfilment-files.js

@@ -4,7 +4,6 @@ import merge2 from 'merge2'
 import moment from 'moment'
 import { executeQuery, persist, findFulfilmentFiles } from '@defra-fish/dynamics-lib'
 import { createS3WriteStream, readS3PartFiles } from '../transport/s3.js'
-import { createFtpWriteStream } from '../transport/ftp.js'
 import { FULFILMENT_FILE_STATUS_OPTIONSET, getOptionSetEntry } from './staging-common.js'
 import db from 'debug'
 import openpgp from 'openpgp'
@@ -69,11 +68,6 @@ const createEncryptedDataReadStream = async file => {
  */
 const deliver = async (targetFileName, readableStream, ...transforms) => {
   const { s3WriteStream: s3DataStream, managedUpload: s3DataManagedUpload } = createS3WriteStream(targetFileName)
-  const { ftpWriteStream: ftpDataStream, managedUpload: ftpDataManagedUpload } = createFtpWriteStream(targetFileName)
 
-  await Promise.all([
-    streamHelper.pipelinePromise([readableStream, ...transforms, s3DataStream, ftpDataStream]),
-    s3DataManagedUpload,
-    ftpDataManagedUpload
-  ])
+  await Promise.all([streamHelper.pipelinePromise([readableStream, ...transforms, s3DataStream]), s3DataManagedUpload])
 }