From 633460235c0cc584dece33f84d5996ef9fe408cb Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Wed, 2 Oct 2024 11:05:26 +0100 Subject: [PATCH 01/19] Remove ftp functionality https://eaflood.atlassian.net/browse/IWTF-4278 Remove ftp functionality as package we use in fulfilment and pocl jobs (ssh2-sftp-client) has critical vulnerability From c0ecae46ab4e8eeb1ef8bc7e108c3768f4ca9ba3 Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Wed, 9 Oct 2024 14:32:37 +0100 Subject: [PATCH 02/19] remove ssh2 sftp client --- packages/fulfilment-job/package-lock.json | 385 +---------------- packages/fulfilment-job/package.json | 3 +- .../src/__mocks__/ssh2-sftp-client.js | 9 - .../deliver-fulfilment-files.spec.js | 29 +- .../src/staging/deliver-fulfilment-files.js | 7 +- .../src/transport/__tests__/ftp.spec.js | 63 --- packages/fulfilment-job/src/transport/ftp.js | 28 -- packages/pocl-job/package-lock.json | 395 +----------------- packages/pocl-job/package.json | 3 +- .../src/__mocks__/ssh2-sftp-client.js | 11 - packages/pocl-job/src/io/__tests__/s3.spec.js | 2 - packages/pocl-job/src/io/s3.js | 12 +- .../src/transport/__tests__/ftp-to-s3.spec.js | 145 ------- packages/pocl-job/src/transport/ftp-to-s3.js | 91 ---- 14 files changed, 24 insertions(+), 1159 deletions(-) delete mode 100644 packages/fulfilment-job/src/__mocks__/ssh2-sftp-client.js delete mode 100644 packages/fulfilment-job/src/transport/__tests__/ftp.spec.js delete mode 100644 packages/fulfilment-job/src/transport/ftp.js delete mode 100644 packages/pocl-job/src/__mocks__/ssh2-sftp-client.js delete mode 100644 packages/pocl-job/src/transport/__tests__/ftp-to-s3.spec.js delete mode 100644 packages/pocl-job/src/transport/ftp-to-s3.js diff --git a/packages/fulfilment-job/package-lock.json b/packages/fulfilment-job/package-lock.json index 86946bc643..c23811f2d6 100644 --- a/packages/fulfilment-job/package-lock.json +++ b/packages/fulfilment-job/package-lock.json @@ -16,8 +16,7 @@ "merge2": "^1.4.1", "moment": "^2.29.1", "openpgp": "^5.0.0-1", - "pluralize": "^8.0.0", - "ssh2-sftp-client": "^6.0.1" + "pluralize": "^8.0.0" }, "engines": { "node": ">=18.17" @@ -50,20 +49,10 @@ "node": ">=10" } }, - "node_modules/@dabh/diagnostics": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz", - "integrity": "sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==", - "dependencies": { - "colorspace": "1.1.x", - "enabled": "2.0.x", - "kuler": "^2.0.0" - } - }, "node_modules/@defra-fish/connectors-lib": { - "version": "1.49.0-rc.10", - "resolved": "https://registry.npmjs.org/@defra-fish/connectors-lib/-/connectors-lib-1.49.0-rc.10.tgz", - "integrity": "sha512-/6yr5fIfpZ1jBS5HybOXX4mXGyj5gSNPkrWipaL7rLKFR1qrWyjLKIr6ZWwAMhWHP6TwOP7AdUSgamME9tJrGA==", + "version": "1.50.0-rc.8", + "resolved": "https://registry.npmjs.org/@defra-fish/connectors-lib/-/connectors-lib-1.50.0-rc.8.tgz", + "integrity": "sha512-aXPMRNfXBF6xytV+TD7C3fbeEmMb0m2VPsaDiq4uN4ShqqTg2q3OuAo6YUZypKOkX2tH6/Hj4Qpb3UMerNJ6hg==", "dependencies": { "@airbrake/node": "^2.1.7", "aws-sdk": "^2.1074.0", @@ -77,9 +66,9 @@ } }, "node_modules/@defra-fish/dynamics-lib": { - "version": "1.49.0-rc.10", - "resolved": "https://registry.npmjs.org/@defra-fish/dynamics-lib/-/dynamics-lib-1.49.0-rc.10.tgz", - "integrity": "sha512-1hl3nFRAnaIdc8doOV3hXAMPOQC4s88nrcYSbmoHlPJGtq5h1o6roWAf+JQhDi9/UQgyToD1/hSNLn1SpxZOUg==", + "version": "1.50.0-rc.8", + "resolved": "https://registry.npmjs.org/@defra-fish/dynamics-lib/-/dynamics-lib-1.50.0-rc.8.tgz", + "integrity": "sha512-K0d15rfayclGfqcoIrrv9XaHRTXqnvhPu+1cnRmoWHgaDBeKr/YrGB/H2Y0S5Mq4tk78E1QdYIupJY9oXAWuxA==", "dependencies": { "cache-manager": "^3.6.0", "cache-manager-ioredis": "^2.1.0", @@ -202,14 +191,6 @@ "node": ">= 6.0.0" } }, - "node_modules/asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "dependencies": { - "safer-buffer": "~2.1.0" - } - }, "node_modules/asn1.js": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", @@ -293,14 +274,6 @@ } ] }, - "node_modules/bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "dependencies": { - "tweetnacl": "^0.14.3" - } - }, "node_modules/bintrees": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", @@ -326,11 +299,6 @@ "isarray": "^1.0.0" } }, - "node_modules/buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" - }, "node_modules/cache-manager": { "version": "3.6.3", "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-3.6.3.tgz", @@ -378,54 +346,6 @@ "node": ">=0.10.0" } }, - "node_modules/color": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", - "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==", - "dependencies": { - "color-convert": "^1.9.1", - "color-string": "^1.5.2" - } - }, - "node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" - }, - "node_modules/color-string": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.4.tgz", - "integrity": "sha512-57yF5yt8Xa3czSEW1jfQDE79Idk0+AkN/4KWad6tbdxUmAs3MvjxlWSWD4deYytcRfoZ9nhKyFl1kj5tBvidbw==", - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, - "node_modules/colors": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", - "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/colorspace": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz", - "integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==", - "dependencies": { - "color": "3.0.x", - "text-hex": "1.0.x" - } - }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -445,25 +365,6 @@ "node": ">= 10" } }, - "node_modules/concat-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", - "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "engines": [ - "node >= 6.0" - ], - "dependencies": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.0.2", - "typedarray": "^0.0.6" - } - }, - "node_modules/core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, "node_modules/cross-fetch": { "version": "3.1.8", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", @@ -529,16 +430,6 @@ "https-proxy-agent": "^5.0.0" } }, - "node_modules/enabled": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", - "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" - }, - "node_modules/err-code": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", - "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==" - }, "node_modules/error-stack-parser": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.1.4.tgz", @@ -574,21 +465,6 @@ "node": ">=0.4.x" } }, - "node_modules/fast-safe-stringify": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", - "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" - }, - "node_modules/fecha": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.0.tgz", - "integrity": "sha512-aN3pcx/DSmtyoovUudctc8+6Hl4T+hI9GBBHLjA76jdZl7+b1sgh5g4k+u/GL3dTy1/pnYzKp69FpJ0OicE3Wg==" - }, - "node_modules/fn.name": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", - "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" - }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -780,11 +656,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -810,14 +681,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", - "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==", - "engines": { - "node": ">=8" - } - }, "node_modules/is-typed-array": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", @@ -857,11 +720,6 @@ "@sideway/pinpoint": "^2.0.0" } }, - "node_modules/kuler": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", - "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" - }, "node_modules/lodash.clonedeep": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", @@ -882,18 +740,6 @@ "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" }, - "node_modules/logform": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz", - "integrity": "sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg==", - "dependencies": { - "colors": "^1.2.1", - "fast-safe-stringify": "^2.0.4", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "triple-beam": "^1.3.0" - } - }, "node_modules/lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -969,14 +815,6 @@ } } }, - "node_modules/one-time": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", - "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", - "dependencies": { - "fn.name": "1.x.x" - } - }, "node_modules/openpgp": { "version": "5.0.0-1", "resolved": "https://registry.npmjs.org/openpgp/-/openpgp-5.0.0-1.tgz", @@ -1012,28 +850,11 @@ "node": ">= 0.4" } }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, "node_modules/promise-polyfill": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/promise-polyfill/-/promise-polyfill-8.3.0.tgz", "integrity": "sha512-H5oELycFml5yto/atYqmjyigJoAo3+OXwolYiH7OfQuYlAqhxNvTfiNMbV9hsC6Yp83yE5r2KTVmtrG6R9i6Pg==" }, - "node_modules/promise-retry": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", - "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", - "dependencies": { - "err-code": "^2.0.2", - "retry": "^0.12.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -1048,19 +869,6 @@ "node": ">=0.4.x" } }, - "node_modules/readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/redis-commands": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", @@ -1096,33 +904,6 @@ "node": ">=8.0.0" } }, - "node_modules/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=", - "engines": { - "node": ">= 4" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -1160,57 +941,6 @@ "joi": "^17.3.0" } }, - "node_modules/simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=", - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, - "node_modules/ssh2": { - "version": "0.8.9", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-0.8.9.tgz", - "integrity": "sha512-GmoNPxWDMkVpMFa9LVVzQZHF6EW3WKmBwL+4/GeILf2hFmix5Isxm7Amamo8o7bHiU0tC+wXsGcUXOxp8ChPaw==", - "dependencies": { - "ssh2-streams": "~0.4.10" - }, - "engines": { - "node": ">=5.2.0" - } - }, - "node_modules/ssh2-sftp-client": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-6.0.1.tgz", - "integrity": "sha512-Glut2SmK/XpNOBiEuzqlKZGKkIyha2XMbuWVXR2hFUJkNsbyl/wmlZSeUEPxKFp/dC9UEvUKzanKydgLmNdfkw==", - "dependencies": { - "concat-stream": "^2.0.0", - "promise-retry": "^2.0.1", - "ssh2": "^0.8.9", - "winston": "^3.3.3" - } - }, - "node_modules/ssh2-streams": { - "version": "0.4.10", - "resolved": "https://registry.npmjs.org/ssh2-streams/-/ssh2-streams-0.4.10.tgz", - "integrity": "sha512-8pnlMjvnIZJvmTzUIIA5nT4jr2ZWNNVHwyXfMGdRJbug9TpI3kd99ffglgfSWqujVv/0gxwMsDn9j9RVst8yhQ==", - "dependencies": { - "asn1": "~0.2.0", - "bcrypt-pbkdf": "^1.0.2", - "streamsearch": "~0.1.2" - }, - "engines": { - "node": ">=5.2.0" - } - }, - "node_modules/stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=", - "engines": { - "node": "*" - } - }, "node_modules/stackframe": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.3.4.tgz", @@ -1221,22 +951,6 @@ "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" }, - "node_modules/streamsearch": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", - "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, "node_modules/tdigest": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", @@ -1245,31 +959,11 @@ "bintrees": "1.0.2" } }, - "node_modules/text-hex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", - "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" - }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, - "node_modules/triple-beam": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", - "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" - }, - "node_modules/tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" - }, - "node_modules/typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" - }, "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", @@ -1296,11 +990,6 @@ "which-typed-array": "^1.1.2" } }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, "node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -1341,64 +1030,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/winston": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/winston/-/winston-3.3.3.tgz", - "integrity": "sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw==", - "dependencies": { - "@dabh/diagnostics": "^2.0.2", - "async": "^3.1.0", - "is-stream": "^2.0.0", - "logform": "^2.2.0", - "one-time": "^1.0.0", - "readable-stream": "^3.4.0", - "stack-trace": "0.0.x", - "triple-beam": "^1.3.0", - "winston-transport": "^4.4.0" - }, - "engines": { - "node": ">= 6.4.0" - } - }, - "node_modules/winston-transport": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.0.tgz", - "integrity": "sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw==", - "dependencies": { - "readable-stream": "^2.3.7", - "triple-beam": "^1.2.0" - }, - "engines": { - "node": ">= 6.4.0" - } - }, - "node_modules/winston-transport/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/winston-transport/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/winston-transport/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, "node_modules/xml2js": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", @@ -1425,4 +1056,4 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" } } -} \ No newline at end of file +} diff --git a/packages/fulfilment-job/package.json b/packages/fulfilment-job/package.json index 8fd0528c0b..0c1d4de678 100644 --- a/packages/fulfilment-job/package.json +++ b/packages/fulfilment-job/package.json @@ -42,7 +42,6 @@ "merge2": "^1.4.1", "moment": "^2.29.1", "openpgp": "^5.0.0-1", - "pluralize": "^8.0.0", - "ssh2-sftp-client": "^6.0.1" + "pluralize": "^8.0.0" } } diff --git a/packages/fulfilment-job/src/__mocks__/ssh2-sftp-client.js b/packages/fulfilment-job/src/__mocks__/ssh2-sftp-client.js deleted file mode 100644 index 124d4777bb..0000000000 --- a/packages/fulfilment-job/src/__mocks__/ssh2-sftp-client.js +++ /dev/null @@ -1,9 +0,0 @@ -const ssh2sftpClient = jest.genMockFromModule('ssh2-sftp-client') - -export const mockedFtpMethods = { - connect: jest.fn(async () => {}), - put: jest.fn(async () => {}), - end: jest.fn() -} -ssh2sftpClient.mockImplementation(() => mockedFtpMethods) -export default ssh2sftpClient diff --git a/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js b/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js index 83e2145335..a6d6896c11 100644 --- a/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js +++ b/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js @@ -1,7 +1,6 @@ import { Readable, PassThrough, Writable } from 'stream' import { deliverFulfilmentFiles } from '../deliver-fulfilment-files.js' import { createS3WriteStream, readS3PartFiles } from '../../transport/s3.js' -import { createFtpWriteStream } from '../../transport/ftp.js' import { FULFILMENT_FILE_STATUS_OPTIONSET, getOptionSetEntry } from '../staging-common.js' import { FulfilmentRequestFile, executeQuery, persist } from '@defra-fish/dynamics-lib' import openpgp from 'openpgp' @@ -10,7 +9,6 @@ import streamHelper from '../streamHelper.js' import merge2 from 'merge2' jest.mock('../../transport/s3.js') -jest.mock('../../transport/ftp.js') jest.mock('openpgp', () => ({ readKey: jest.fn(() => ({})), encrypt: jest.fn(({ message: readableStream }) => readableStream), @@ -48,17 +46,13 @@ describe('deliverFulfilmentFiles', () => { // Streams for file1 const { s3DataStreamFile: s3DataStreamFile1, - ftpDataStreamFile: ftpDataStreamFile1, - s3HashStreamFile: s3HashStreamFile1, - ftpHashStreamFile: ftpHashStreamFile1 + s3HashStreamFile: s3HashStreamFile1 } = createMockFileStreams() // Streams for file2 const { s3DataStreamFile: s3DataStreamFile2, - ftpDataStreamFile: ftpDataStreamFile2, - s3HashStreamFile: s3HashStreamFile2, - ftpHashStreamFile: ftpHashStreamFile2 + s3HashStreamFile: s3HashStreamFile2 } = createMockFileStreams() // Run the delivery @@ -69,22 +63,14 @@ describe('deliverFulfilmentFiles', () => { // File 1 expectations expect(createS3WriteStream).toHaveBeenNthCalledWith(1, 'EAFF202006180001.json') expect(createS3WriteStream).toHaveBeenNthCalledWith(3, 'EAFF202006180001.json.sha256') - expect(createFtpWriteStream).toHaveBeenNthCalledWith(1, 'EAFF202006180001.json') - expect(createFtpWriteStream).toHaveBeenNthCalledWith(3, 'EAFF202006180001.json.sha256') expect(JSON.parse(s3DataStreamFile1.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] }) - expect(JSON.parse(ftpDataStreamFile1.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] }) expect(s3HashStreamFile1.dataProcessed).toEqual(fileShaHash) // validated - expect(ftpHashStreamFile1.dataProcessed).toEqual(fileShaHash) // validated // File 2 expectations expect(createS3WriteStream).toHaveBeenNthCalledWith(4, 'EAFF202006180002.json') expect(createS3WriteStream).toHaveBeenNthCalledWith(6, 'EAFF202006180002.json.sha256') - expect(createFtpWriteStream).toHaveBeenNthCalledWith(4, 'EAFF202006180002.json') - expect(createFtpWriteStream).toHaveBeenNthCalledWith(6, 'EAFF202006180002.json.sha256') expect(JSON.parse(s3DataStreamFile2.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] }) - expect(JSON.parse(ftpDataStreamFile2.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] }) expect(s3HashStreamFile2.dataProcessed).toEqual(fileShaHash) // validated - expect(ftpHashStreamFile2.dataProcessed).toEqual(fileShaHash) // validated // Persist to dynamics for file 1 expect(persist).toHaveBeenNthCalledWith(1, [ @@ -227,27 +213,18 @@ const createMockFulfilmentRequestFile = async (fileName, date) => const createMockFileStreams = () => { const s3DataStreamFile = createTestableStream() - const ftpDataStreamFile = createTestableStream() createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3DataStreamFile, managedUpload: Promise.resolve() }) - createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpDataStreamFile, managedUpload: Promise.resolve() }) const s3EncryptedDataStreamFile = createTestableStream() - const ftpEncryptedDataStreamFile = createTestableStream() createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3EncryptedDataStreamFile, managedUpload: Promise.resolve() }) - createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpEncryptedDataStreamFile, managedUpload: Promise.resolve() }) const s3HashStreamFile = createTestableStream() - const ftpHashStreamFile = createTestableStream() createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3HashStreamFile, managedUpload: Promise.resolve() }) - createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpHashStreamFile, managedUpload: Promise.resolve() }) return { s3DataStreamFile, - ftpDataStreamFile, s3EncryptedDataStreamFile, - ftpEncryptedDataStreamFile, - s3HashStreamFile, - ftpHashStreamFile + s3HashStreamFile } } diff --git a/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js b/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js index 09c24ceb80..d14e19ea27 100644 --- a/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js +++ b/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js @@ -4,7 +4,6 @@ import merge2 from 'merge2' import moment from 'moment' import { executeQuery, persist, findFulfilmentFiles } from '@defra-fish/dynamics-lib' import { createS3WriteStream, readS3PartFiles } from '../transport/s3.js' -import { createFtpWriteStream } from '../transport/ftp.js' import { FULFILMENT_FILE_STATUS_OPTIONSET, getOptionSetEntry } from './staging-common.js' import db from 'debug' import openpgp from 'openpgp' @@ -69,11 +68,9 @@ const createEncryptedDataReadStream = async file => { */ const deliver = async (targetFileName, readableStream, ...transforms) => { const { s3WriteStream: s3DataStream, managedUpload: s3DataManagedUpload } = createS3WriteStream(targetFileName) - const { ftpWriteStream: ftpDataStream, managedUpload: ftpDataManagedUpload } = createFtpWriteStream(targetFileName) await Promise.all([ - streamHelper.pipelinePromise([readableStream, ...transforms, s3DataStream, ftpDataStream]), - s3DataManagedUpload, - ftpDataManagedUpload + streamHelper.pipelinePromise([readableStream, ...transforms, s3DataStream]), + s3DataManagedUpload ]) } diff --git a/packages/fulfilment-job/src/transport/__tests__/ftp.spec.js b/packages/fulfilment-job/src/transport/__tests__/ftp.spec.js deleted file mode 100644 index bf6e868ac4..0000000000 --- a/packages/fulfilment-job/src/transport/__tests__/ftp.spec.js +++ /dev/null @@ -1,63 +0,0 @@ -import { createFtpWriteStream } from '../ftp.js' -import { mockedFtpMethods } from 'ssh2-sftp-client' - -jest.mock('stream') -jest.mock('../../config.js', () => ({ - ftp: { - host: 'testhost', - port: 2222, - path: 'testpath/', - username: 'testusername', - privateKey: 'testprivatekey' - } -})) - -describe('ftp', () => { - beforeEach(() => { - jest.clearAllMocks() - }) - - describe('createFtpWriteStream', () => { - it('creates a stream to write to the configured FTP server', async () => { - const { ftpWriteStream, managedUpload } = createFtpWriteStream('testfile.json') - ftpWriteStream.write('Some data') - ftpWriteStream.end() - await managedUpload - expect(mockedFtpMethods.connect).toHaveBeenCalledWith( - expect.objectContaining({ - host: 'testhost', - port: 2222, - username: 'testusername', - privateKey: 'testprivatekey' - }) - ) - expect(mockedFtpMethods.put).toHaveBeenCalledWith(ftpWriteStream, 'testpath/testfile.json', { - flags: 'w', - encoding: 'UTF-8', - autoClose: false - }) - expect(mockedFtpMethods.end).toHaveBeenCalled() - }) - - it('rejects the managed upload promise if an FTP upload error occurs', async () => { - const testError = new Error('Test error') - mockedFtpMethods.put.mockImplementationOnce(() => Promise.reject(testError)) - const { ftpWriteStream, managedUpload } = createFtpWriteStream('testfile.json') - await expect(managedUpload).rejects.toThrow('Test error') - expect(mockedFtpMethods.connect).toHaveBeenCalledWith( - expect.objectContaining({ - host: 'testhost', - port: 2222, - username: 'testusername', - privateKey: 'testprivatekey' - }) - ) - expect(mockedFtpMethods.put).toHaveBeenCalledWith(ftpWriteStream, 'testpath/testfile.json', { - flags: 'w', - encoding: 'UTF-8', - autoClose: false - }) - expect(mockedFtpMethods.end).toHaveBeenCalled() - }) - }) -}) diff --git a/packages/fulfilment-job/src/transport/ftp.js b/packages/fulfilment-job/src/transport/ftp.js deleted file mode 100644 index 0ce3f41c9c..0000000000 --- a/packages/fulfilment-job/src/transport/ftp.js +++ /dev/null @@ -1,28 +0,0 @@ -import FtpClient from 'ssh2-sftp-client' -import Path from 'path' -import { PassThrough } from 'stream' -import config from '../config.js' -import db from 'debug' -const debug = db('fulfilment:transport') - -/** - * Create a stream to write to the configured FTP server - * - * @param {string} filename The name of the file to be written to the remote server - * @returns {{ftpWriteStream: module:stream.internal.PassThrough, managedUpload: Promise<*>}} - */ -export const createFtpWriteStream = filename => { - const sftp = new FtpClient() - const passThrough = new PassThrough() - const remoteFilePath = Path.join(config.ftp.path, filename) - return { - ftpWriteStream: passThrough, - managedUpload: sftp - .connect(config.ftp) - .then(() => sftp.put(passThrough, remoteFilePath, { flags: 'w', encoding: 'UTF-8', autoClose: false })) - .then(() => - debug('File successfully uploaded to fulfilment provider at sftp://%s:%s%s', config.ftp.host, config.ftp.port, remoteFilePath) - ) - .finally(() => sftp.end()) - } -} diff --git a/packages/pocl-job/package-lock.json b/packages/pocl-job/package-lock.json index 0c2833da12..3471e125e2 100644 --- a/packages/pocl-job/package-lock.json +++ b/packages/pocl-job/package-lock.json @@ -17,8 +17,7 @@ "md5-file": "^5.0.0", "moment": "^2.29.1", "moment-timezone": "^0.5.34", - "sax-stream": "^1.3.0", - "ssh2-sftp-client": "^6.0.1" + "sax-stream": "^1.3.0" }, "engines": { "node": ">=18.17" @@ -51,20 +50,10 @@ "node": ">=10" } }, - "node_modules/@dabh/diagnostics": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz", - "integrity": "sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==", - "dependencies": { - "colorspace": "1.1.x", - "enabled": "2.0.x", - "kuler": "^2.0.0" - } - }, "node_modules/@defra-fish/business-rules-lib": { - "version": "1.49.0-rc.10", - "resolved": "https://registry.npmjs.org/@defra-fish/business-rules-lib/-/business-rules-lib-1.49.0-rc.10.tgz", - "integrity": "sha512-N/FaFy2oY57Op9FewIc94vWgfF5bplgD9kPDJ/+VRRrOi3n5tl2Mm2WT93h26k6Fou4kUGH1g8wyWkwek13lrg==", + "version": "1.50.0-rc.8", + "resolved": "https://registry.npmjs.org/@defra-fish/business-rules-lib/-/business-rules-lib-1.50.0-rc.8.tgz", + "integrity": "sha512-Xo+lI+8DON5vahQKQe8ZX1fLpmE2TQm7eThYaFeUxBAEwbhhP7u7l3Ck49YR9hzWsAUWMLWGKgUrWAhCdId/3g==", "dependencies": { "joi": "^17.6.0", "moment": "^2.29.1", @@ -75,9 +64,9 @@ } }, "node_modules/@defra-fish/connectors-lib": { - "version": "1.49.0-rc.10", - "resolved": "https://registry.npmjs.org/@defra-fish/connectors-lib/-/connectors-lib-1.49.0-rc.10.tgz", - "integrity": "sha512-/6yr5fIfpZ1jBS5HybOXX4mXGyj5gSNPkrWipaL7rLKFR1qrWyjLKIr6ZWwAMhWHP6TwOP7AdUSgamME9tJrGA==", + "version": "1.50.0-rc.8", + "resolved": "https://registry.npmjs.org/@defra-fish/connectors-lib/-/connectors-lib-1.50.0-rc.8.tgz", + "integrity": "sha512-aXPMRNfXBF6xytV+TD7C3fbeEmMb0m2VPsaDiq4uN4ShqqTg2q3OuAo6YUZypKOkX2tH6/Hj4Qpb3UMerNJ6hg==", "dependencies": { "@airbrake/node": "^2.1.7", "aws-sdk": "^2.1074.0", @@ -155,19 +144,6 @@ "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" }, - "node_modules/asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "dependencies": { - "safer-buffer": "~2.1.0" - } - }, - "node_modules/async": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz", - "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==" - }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -240,14 +216,6 @@ } ] }, - "node_modules/bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "dependencies": { - "tweetnacl": "^0.14.3" - } - }, "node_modules/bintrees": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", @@ -268,11 +236,6 @@ "isarray": "^1.0.0" } }, - "node_modules/buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" - }, "node_modules/call-bind": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", @@ -299,54 +262,6 @@ "node": ">=0.10.0" } }, - "node_modules/color": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", - "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==", - "dependencies": { - "color-convert": "^1.9.1", - "color-string": "^1.5.2" - } - }, - "node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" - }, - "node_modules/color-string": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.4.tgz", - "integrity": "sha512-57yF5yt8Xa3czSEW1jfQDE79Idk0+AkN/4KWad6tbdxUmAs3MvjxlWSWD4deYytcRfoZ9nhKyFl1kj5tBvidbw==", - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, - "node_modules/colors": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", - "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/colorspace": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz", - "integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==", - "dependencies": { - "color": "3.0.x", - "text-hex": "1.0.x" - } - }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -366,25 +281,6 @@ "node": ">= 10" } }, - "node_modules/concat-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", - "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "engines": [ - "node >= 6.0" - ], - "dependencies": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.0.2", - "typedarray": "^0.0.6" - } - }, - "node_modules/core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, "node_modules/cross-fetch": { "version": "3.1.8", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", @@ -441,16 +337,6 @@ "node": ">=0.10" } }, - "node_modules/enabled": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", - "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" - }, - "node_modules/err-code": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", - "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==" - }, "node_modules/error-stack-parser": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.1.4.tgz", @@ -486,16 +372,6 @@ "node": ">=0.4.x" } }, - "node_modules/fast-safe-stringify": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", - "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" - }, - "node_modules/fecha": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.0.tgz", - "integrity": "sha512-aN3pcx/DSmtyoovUudctc8+6Hl4T+hI9GBBHLjA76jdZl7+b1sgh5g4k+u/GL3dTy1/pnYzKp69FpJ0OicE3Wg==" - }, "node_modules/filesize": { "version": "6.4.0", "resolved": "https://registry.npmjs.org/filesize/-/filesize-6.4.0.tgz", @@ -504,11 +380,6 @@ "node": ">= 0.4.0" } }, - "node_modules/fn.name": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", - "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" - }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -675,11 +546,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -705,14 +571,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", - "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==", - "engines": { - "node": ">=8" - } - }, "node_modules/is-typed-array": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", @@ -752,11 +610,6 @@ "@sideway/pinpoint": "^2.0.0" } }, - "node_modules/kuler": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", - "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" - }, "node_modules/lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", @@ -772,18 +625,6 @@ "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" }, - "node_modules/logform": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz", - "integrity": "sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg==", - "dependencies": { - "colors": "^1.2.1", - "fast-safe-stringify": "^2.0.4", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "triple-beam": "^1.3.0" - } - }, "node_modules/md5-file": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/md5-file/-/md5-file-5.0.0.tgz", @@ -857,14 +698,6 @@ } } }, - "node_modules/one-time": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", - "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", - "dependencies": { - "fn.name": "1.x.x" - } - }, "node_modules/p-map": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", @@ -881,28 +714,11 @@ "node": ">= 0.4" } }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, "node_modules/promise-polyfill": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/promise-polyfill/-/promise-polyfill-8.3.0.tgz", "integrity": "sha512-H5oELycFml5yto/atYqmjyigJoAo3+OXwolYiH7OfQuYlAqhxNvTfiNMbV9hsC6Yp83yE5r2KTVmtrG6R9i6Pg==" }, - "node_modules/promise-retry": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", - "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", - "dependencies": { - "err-code": "^2.0.2", - "retry": "^0.12.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -917,19 +733,6 @@ "node": ">=0.4.x" } }, - "node_modules/readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/redis-commands": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", @@ -965,38 +768,6 @@ "node": ">=8.0.0" } }, - "node_modules/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=", - "engines": { - "node": ">= 4" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, "node_modules/sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", @@ -1040,57 +811,6 @@ "node": ">= 0.4" } }, - "node_modules/simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=", - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, - "node_modules/ssh2": { - "version": "0.8.9", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-0.8.9.tgz", - "integrity": "sha512-GmoNPxWDMkVpMFa9LVVzQZHF6EW3WKmBwL+4/GeILf2hFmix5Isxm7Amamo8o7bHiU0tC+wXsGcUXOxp8ChPaw==", - "dependencies": { - "ssh2-streams": "~0.4.10" - }, - "engines": { - "node": ">=5.2.0" - } - }, - "node_modules/ssh2-sftp-client": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-6.0.1.tgz", - "integrity": "sha512-Glut2SmK/XpNOBiEuzqlKZGKkIyha2XMbuWVXR2hFUJkNsbyl/wmlZSeUEPxKFp/dC9UEvUKzanKydgLmNdfkw==", - "dependencies": { - "concat-stream": "^2.0.0", - "promise-retry": "^2.0.1", - "ssh2": "^0.8.9", - "winston": "^3.3.3" - } - }, - "node_modules/ssh2-streams": { - "version": "0.4.10", - "resolved": "https://registry.npmjs.org/ssh2-streams/-/ssh2-streams-0.4.10.tgz", - "integrity": "sha512-8pnlMjvnIZJvmTzUIIA5nT4jr2ZWNNVHwyXfMGdRJbug9TpI3kd99ffglgfSWqujVv/0gxwMsDn9j9RVst8yhQ==", - "dependencies": { - "asn1": "~0.2.0", - "bcrypt-pbkdf": "^1.0.2", - "streamsearch": "~0.1.2" - }, - "engines": { - "node": ">=5.2.0" - } - }, - "node_modules/stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=", - "engines": { - "node": "*" - } - }, "node_modules/stackframe": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.3.4.tgz", @@ -1101,22 +821,6 @@ "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" }, - "node_modules/streamsearch": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", - "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, "node_modules/tdigest": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", @@ -1125,31 +829,11 @@ "bintrees": "1.0.2" } }, - "node_modules/text-hex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", - "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" - }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, - "node_modules/triple-beam": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", - "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" - }, - "node_modules/tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" - }, - "node_modules/typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" - }, "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", @@ -1176,11 +860,6 @@ "which-typed-array": "^1.1.2" } }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, "node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -1221,64 +900,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/winston": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/winston/-/winston-3.3.3.tgz", - "integrity": "sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw==", - "dependencies": { - "@dabh/diagnostics": "^2.0.2", - "async": "^3.1.0", - "is-stream": "^2.0.0", - "logform": "^2.2.0", - "one-time": "^1.0.0", - "readable-stream": "^3.4.0", - "stack-trace": "0.0.x", - "triple-beam": "^1.3.0", - "winston-transport": "^4.4.0" - }, - "engines": { - "node": ">= 6.4.0" - } - }, - "node_modules/winston-transport": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.0.tgz", - "integrity": "sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw==", - "dependencies": { - "readable-stream": "^2.3.7", - "triple-beam": "^1.2.0" - }, - "engines": { - "node": ">= 6.4.0" - } - }, - "node_modules/winston-transport/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/winston-transport/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/winston-transport/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, "node_modules/xml2js": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", @@ -1300,4 +921,4 @@ } } } -} \ No newline at end of file +} diff --git a/packages/pocl-job/package.json b/packages/pocl-job/package.json index 151c8bef6a..4b4c1c3341 100644 --- a/packages/pocl-job/package.json +++ b/packages/pocl-job/package.json @@ -43,7 +43,6 @@ "md5-file": "^5.0.0", "moment": "^2.29.1", "moment-timezone": "^0.5.34", - "sax-stream": "^1.3.0", - "ssh2-sftp-client": "^6.0.1" + "sax-stream": "^1.3.0" } } diff --git a/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js b/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js deleted file mode 100644 index f24599f0b1..0000000000 --- a/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js +++ /dev/null @@ -1,11 +0,0 @@ -const ssh2sftpClient = jest.genMockFromModule('ssh2-sftp-client') - -export const mockedFtpMethods = { - connect: jest.fn(), - list: jest.fn(), - fastGet: jest.fn(), - delete: jest.fn(), - end: jest.fn() -} -ssh2sftpClient.mockImplementation(() => mockedFtpMethods) -export default ssh2sftpClient diff --git a/packages/pocl-job/src/io/__tests__/s3.spec.js b/packages/pocl-job/src/io/__tests__/s3.spec.js index 4a1a2da226..26c7fe9be1 100644 --- a/packages/pocl-job/src/io/__tests__/s3.spec.js +++ b/packages/pocl-job/src/io/__tests__/s3.spec.js @@ -5,7 +5,6 @@ import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../.. import { salesApi } from '@defra-fish/connectors-lib' import fs from 'fs' import AwsMock from 'aws-sdk' -import { mockedFtpMethods } from 'ssh2-sftp-client' jest.mock('fs') jest.mock('md5-file') @@ -159,7 +158,6 @@ describe('s3 operations', () => { }) it('skips file processing if a file has already been marked as processed in Dynamics', async () => { - mockedFtpMethods.list.mockResolvedValue([{ name: 'test-already-processed.xml' }]) fs.createReadStream.mockReturnValueOnce('teststream') fs.statSync.mockReturnValueOnce({ size: 1024 }) salesApi.getTransactionFile.mockResolvedValueOnce({ status: { description: 'Processed' } }) diff --git a/packages/pocl-job/src/io/s3.js b/packages/pocl-job/src/io/s3.js index c24d3ba3d0..cb94087563 100644 --- a/packages/pocl-job/src/io/s3.js +++ b/packages/pocl-job/src/io/s3.js @@ -1,9 +1,6 @@ import moment from 'moment' -import filesize from 'filesize' import config from '../config.js' -import { DYNAMICS_IMPORT_STAGE } from '../staging/constants.js' -import { storeS3Metadata } from '../transport/ftp-to-s3.js' -import { AWS, salesApi } from '@defra-fish/connectors-lib' +import { AWS } from '@defra-fish/connectors-lib' const { s3 } = AWS() const listObjectsV2 = async function (params) { @@ -47,13 +44,6 @@ export const refreshS3Metadata = async () => { const filename = file.Key.split('/').pop() console.log(`Processing ${filename}`) - - const dynamicsRecord = await salesApi.getTransactionFile(filename) - if (!dynamicsRecord || !DYNAMICS_IMPORT_STAGE.isAlreadyProcessed(dynamicsRecord.status.description)) { - await storeS3Metadata(file.ETag, filesize(file.Size), filename, file.Key, moment(new Date(file.LastModified))) - } else { - console.log(`${file.Key} is already processed, skipping`) - } } console.log('Processed S3 files') diff --git a/packages/pocl-job/src/transport/__tests__/ftp-to-s3.spec.js b/packages/pocl-job/src/transport/__tests__/ftp-to-s3.spec.js deleted file mode 100644 index 7b5259da27..0000000000 --- a/packages/pocl-job/src/transport/__tests__/ftp-to-s3.spec.js +++ /dev/null @@ -1,145 +0,0 @@ -import { ftpToS3 } from '../ftp-to-s3.js' -import moment from 'moment' -import { updateFileStagingTable } from '../../io/db.js' -import { getTempDir } from '../../io/file.js' -import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../../staging/constants.js' -import { salesApi } from '@defra-fish/connectors-lib' -import fs from 'fs' -import md5File from 'md5-file' -import AwsMock from 'aws-sdk' -import { mockedFtpMethods } from 'ssh2-sftp-client' - -jest.mock('fs') -jest.mock('md5-file') -jest.mock('../../io/db.js') -jest.mock('../../io/file.js') - -jest.mock('@defra-fish/connectors-lib', () => { - const actual = jest.requireActual('@defra-fish/connectors-lib') - return { - AWS: actual.AWS, - salesApi: { - ...Object.keys(actual.salesApi).reduce((acc, k) => ({ ...acc, [k]: jest.fn(async () => {}) }), {}) - } - } -}) - -jest.mock('../../config.js', () => ({ - ftp: { - path: '/ftpservershare/' - }, - s3: { - bucket: 'testbucket' - } -})) - -describe('ftp-to-s3', () => { - beforeAll(() => { - getTempDir.mockReturnValue('/local/tmp') - md5File.mockResolvedValue('example-md5') - }) - beforeEach(() => { - jest.clearAllMocks() - AwsMock.__resetAll() - }) - - it('retrieves files from SFTP and stores in S3', async () => { - mockedFtpMethods.list.mockResolvedValue([{ name: 'test1.xml' }, { name: 'test2.xml' }]) - fs.createReadStream.mockReturnValueOnce('test1stream') - fs.createReadStream.mockReturnValueOnce('test2stream') - fs.statSync.mockReturnValueOnce({ size: 1024 }) - fs.statSync.mockReturnValueOnce({ size: 2048 }) - await ftpToS3() - - const localPath1 = '/local/tmp/test1.xml' - const localPath2 = '/local/tmp/test2.xml' - - const s3Key1 = `${moment().format('YYYY-MM-DD')}/test1.xml` - const s3Key2 = `${moment().format('YYYY-MM-DD')}/test2.xml` - - expect(mockedFtpMethods.fastGet).toHaveBeenNthCalledWith(1, '/ftpservershare/test1.xml', localPath1, {}) - expect(mockedFtpMethods.fastGet).toHaveBeenNthCalledWith(2, '/ftpservershare/test2.xml', localPath2, {}) - expect(AwsMock.S3.mockedMethods.putObject).toHaveBeenNthCalledWith(1, { - Bucket: 'testbucket', - Key: s3Key1, - Body: 'test1stream' - }) - expect(AwsMock.S3.mockedMethods.putObject).toHaveBeenNthCalledWith(2, { - Bucket: 'testbucket', - Key: s3Key2, - Body: 'test2stream' - }) - expect(updateFileStagingTable).toHaveBeenNthCalledWith(1, { - filename: 'test1.xml', - md5: 'example-md5', - fileSize: '1 KB', - stage: FILE_STAGE.Pending, - s3Key: s3Key1 - }) - expect(updateFileStagingTable).toHaveBeenNthCalledWith(2, { - filename: 'test2.xml', - md5: 'example-md5', - fileSize: '2 KB', - stage: FILE_STAGE.Pending, - s3Key: s3Key2 - }) - expect(salesApi.upsertTransactionFile).toHaveBeenNthCalledWith(1, 'test1.xml', { - status: DYNAMICS_IMPORT_STAGE.Pending, - dataSource: POST_OFFICE_DATASOURCE, - fileSize: '1 KB', - receiptTimestamp: expect.any(String), - salesDate: expect.any(String), - notes: 'Retrieved from the remote server and awaiting processing' - }) - expect(salesApi.upsertTransactionFile).toHaveBeenNthCalledWith(2, 'test2.xml', { - status: DYNAMICS_IMPORT_STAGE.Pending, - dataSource: POST_OFFICE_DATASOURCE, - fileSize: '2 KB', - receiptTimestamp: expect.any(String), - salesDate: expect.any(String), - notes: 'Retrieved from the remote server and awaiting processing' - }) - expect(fs.unlinkSync).toHaveBeenNthCalledWith(1, localPath1) - expect(fs.unlinkSync).toHaveBeenNthCalledWith(2, localPath2) - expect(mockedFtpMethods.end).toHaveBeenCalledTimes(1) - }) - - it('moves the file to s3 but skips file processing if a file has already been marked as processed in Dynamics', async () => { - mockedFtpMethods.list.mockResolvedValue([{ name: 'test-already-processed.xml' }]) - fs.createReadStream.mockReturnValueOnce('teststream') - fs.statSync.mockReturnValueOnce({ size: 1024 }) - salesApi.getTransactionFile.mockResolvedValueOnce({ status: { description: 'Processed' } }) - const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(jest.fn()) - await ftpToS3() - const localPath = '/local/tmp/test-already-processed.xml' - const s3Key = `${moment().format('YYYY-MM-DD')}/test-already-processed.xml` - expect(mockedFtpMethods.fastGet).toHaveBeenCalledWith('/ftpservershare/test-already-processed.xml', localPath, {}) - expect(AwsMock.S3.mockedMethods.putObject).toHaveBeenCalledWith({ - Bucket: 'testbucket', - Key: s3Key, - Body: 'teststream' - }) - expect(updateFileStagingTable).not.toHaveBeenCalled() - expect(salesApi.upsertTransactionFile).not.toHaveBeenCalled() - expect(consoleErrorSpy).toHaveBeenCalled() - expect(fs.unlinkSync).toHaveBeenCalledWith(localPath) - expect(mockedFtpMethods.end).toHaveBeenCalledTimes(1) - }) - - it('logs and propogates errors back up the stack', async () => { - const testError = new Error('Test error') - mockedFtpMethods.list.mockRejectedValue(testError) - const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}) - await expect(ftpToS3).rejects.toThrow(testError) - expect(consoleErrorSpy).toHaveBeenCalled() - }) - - it('ignores non-xml files', async () => { - mockedFtpMethods.list.mockResolvedValue([{ name: 'test1.pdf' }, { name: 'test2.md' }]) - await ftpToS3() - expect(mockedFtpMethods.fastGet).not.toHaveBeenCalled() - expect(AwsMock.S3.mockedMethods.putObject).not.toHaveBeenCalled() - expect(fs.unlinkSync).not.toHaveBeenCalled() - expect(mockedFtpMethods.end).toHaveBeenCalledTimes(1) - }) -}) diff --git a/packages/pocl-job/src/transport/ftp-to-s3.js b/packages/pocl-job/src/transport/ftp-to-s3.js deleted file mode 100644 index b3c9376448..0000000000 --- a/packages/pocl-job/src/transport/ftp-to-s3.js +++ /dev/null @@ -1,91 +0,0 @@ -import FtpClient from 'ssh2-sftp-client' -import moment from 'moment' -import Path from 'path' -import fs from 'fs' -import db from 'debug' -import md5File from 'md5-file' -import filesize from 'filesize' -import config from '../config.js' -import { getTempDir } from '../io/file.js' -import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../staging/constants.js' -import { AWS, salesApi } from '@defra-fish/connectors-lib' -import { updateFileStagingTable } from '../io/db.js' -const { s3 } = AWS() - -const debug = db('pocl:transport') -const sftp = new FtpClient() - -export async function ftpToS3 () { - try { - debug('Connecting to SFTP endpoint at sftp://%s:%s%s', config.ftp.host, config.ftp.port, config.ftp.path) - await sftp.connect(config.ftp) - const fileList = await sftp.list(config.ftp.path) - debug('Discovered the following files on the SFTP server: %o', fileList) - const xmlFiles = fileList.filter(f => Path.extname(f.name).toLowerCase() === '.xml') - - if (!xmlFiles.length) { - debug('No XML files were waiting to be processed on the SFTP server.') - } else { - await retrieveAllFiles(xmlFiles) - } - } catch (e) { - console.error('Error migrating files from the SFTP endpoint', e) - throw e - } finally { - debug('Closing SFTP connection.') - await sftp.end() - } -} - -export async function storeS3Metadata (md5, fileSize, filename, s3Key, receiptMoment) { - console.log(`Storing metadata for ${s3Key}`) - await updateFileStagingTable({ filename, md5, fileSize, s3Key, stage: FILE_STAGE.Pending }) - - await salesApi.upsertTransactionFile(filename, { - status: DYNAMICS_IMPORT_STAGE.Pending, - dataSource: POST_OFFICE_DATASOURCE, - fileSize: fileSize, - salesDate: moment(receiptMoment).subtract(1, 'days').toISOString(), - receiptTimestamp: receiptMoment.toISOString(), - notes: 'Retrieved from the remote server and awaiting processing' - }) - - console.log(`Stored metadata for ${s3Key}`) -} - -const retrieveAllFiles = async xmlFiles => { - const tempDir = getTempDir('ftp') - - for (const fileEntry of xmlFiles) { - const filename = fileEntry.name - const remoteFilePath = Path.join(config.ftp.path, filename) - const localFilePath = Path.resolve(tempDir, filename) - - // Retrieve from FTP server to local temporary directory - debug('Transferring %s to %s', remoteFilePath, localFilePath) - await sftp.fastGet(remoteFilePath, localFilePath, {}) - - // Transfer to S3 - const receiptMoment = moment() - const s3Key = Path.join(receiptMoment.format('YYYY-MM-DD'), filename) - debug('Transferring file to S3 bucket %s with key %s', config.s3.bucket, s3Key) - await s3.putObject({ Bucket: config.s3.bucket, Key: s3Key, Body: fs.createReadStream(localFilePath) }).promise() - - const dynamicsRecord = await salesApi.getTransactionFile(filename) - if (dynamicsRecord && DYNAMICS_IMPORT_STAGE.isAlreadyProcessed(dynamicsRecord.status.description)) { - console.error( - 'Retrieved file %s from SFTP and stored in S3, however an entry already exists in Dynamics with this filename. Skipping import.', - filename - ) - } else { - const md5 = await md5File(localFilePath) - const fileSize = filesize(fs.statSync(localFilePath).size) - await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) - } - - // Remove from FTP server and local tmp - debug('Removing remote file %s', remoteFilePath) - await sftp.delete(remoteFilePath) - fs.unlinkSync(localFilePath) - } -} From fa464a943af0c5f0fac601cbdb77b9e797f61417 Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Wed, 9 Oct 2024 14:37:15 +0100 Subject: [PATCH 03/19] remove ssh2 sftp client and fix lint --- .../__tests__/deliver-fulfilment-files.spec.js | 15 +++------------ .../src/staging/deliver-fulfilment-files.js | 5 +---- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js b/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js index a6d6896c11..ca157ec115 100644 --- a/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js +++ b/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js @@ -44,16 +44,10 @@ describe('deliverFulfilmentFiles', () => { executeQuery.mockResolvedValue([{ entity: mockFulfilmentRequestFile2 }, { entity: mockFulfilmentRequestFile1 }]) // Streams for file1 - const { - s3DataStreamFile: s3DataStreamFile1, - s3HashStreamFile: s3HashStreamFile1 - } = createMockFileStreams() + const { s3DataStreamFile: s3DataStreamFile1, s3HashStreamFile: s3HashStreamFile1 } = createMockFileStreams() // Streams for file2 - const { - s3DataStreamFile: s3DataStreamFile2, - s3HashStreamFile: s3HashStreamFile2 - } = createMockFileStreams() + const { s3DataStreamFile: s3DataStreamFile2, s3HashStreamFile: s3HashStreamFile2 } = createMockFileStreams() // Run the delivery await expect(deliverFulfilmentFiles()).resolves.toBeUndefined() @@ -183,10 +177,7 @@ describe('deliverFulfilmentFiles', () => { const s3 = createTestableStream() streamHelper.pipelinePromise.mockResolvedValue() openpgp.encrypt.mockResolvedValue(s2) - merge2 - .mockReturnValueOnce(s1) - .mockReturnValueOnce(s2) - .mockReturnValueOnce(s3) + merge2.mockReturnValueOnce(s1).mockReturnValueOnce(s2).mockReturnValueOnce(s3) await mockExecuteQuery() createMockFileStreams() diff --git a/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js b/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js index d14e19ea27..25036678dc 100644 --- a/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js +++ b/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js @@ -69,8 +69,5 @@ const createEncryptedDataReadStream = async file => { const deliver = async (targetFileName, readableStream, ...transforms) => { const { s3WriteStream: s3DataStream, managedUpload: s3DataManagedUpload } = createS3WriteStream(targetFileName) - await Promise.all([ - streamHelper.pipelinePromise([readableStream, ...transforms, s3DataStream]), - s3DataManagedUpload - ]) + await Promise.all([streamHelper.pipelinePromise([readableStream, ...transforms, s3DataStream]), s3DataManagedUpload]) } From 13f8943df4c17db6c508ddcc6f76ab61fbdeb2ee Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Wed, 9 Oct 2024 15:57:18 +0100 Subject: [PATCH 04/19] remove reference --- packages/pocl-job/src/__tests__/pocl-processor.spec.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/pocl-job/src/__tests__/pocl-processor.spec.js b/packages/pocl-job/src/__tests__/pocl-processor.spec.js index f3a5e4696a..f358861b78 100644 --- a/packages/pocl-job/src/__tests__/pocl-processor.spec.js +++ b/packages/pocl-job/src/__tests__/pocl-processor.spec.js @@ -41,7 +41,6 @@ jest.mock('../config.js', () => ({ bucket: 'testbucket' } })) -jest.mock('../transport/ftp-to-s3.js') jest.mock('../transport/s3-to-local.js') jest.mock('../io/db.js') jest.mock('../io/s3.js') From 2bbacd774f36bf784439f13fed4815921c58493c Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Thu, 10 Oct 2024 08:51:14 +0100 Subject: [PATCH 05/19] fix tests --- packages/pocl-job/src/io/__tests__/s3.spec.js | 156 ++---------------- 1 file changed, 13 insertions(+), 143 deletions(-) diff --git a/packages/pocl-job/src/io/__tests__/s3.spec.js b/packages/pocl-job/src/io/__tests__/s3.spec.js index 26c7fe9be1..199e8b6b6c 100644 --- a/packages/pocl-job/src/io/__tests__/s3.spec.js +++ b/packages/pocl-job/src/io/__tests__/s3.spec.js @@ -1,15 +1,9 @@ import { refreshS3Metadata } from '../s3' import moment from 'moment' -import { updateFileStagingTable } from '../../io/db.js' -import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../../staging/constants.js' -import { salesApi } from '@defra-fish/connectors-lib' -import fs from 'fs' import AwsMock from 'aws-sdk' jest.mock('fs') jest.mock('md5-file') -jest.mock('../../io/db.js') -jest.mock('../../io/file.js') jest.mock('@defra-fish/connectors-lib', () => { const actual = jest.requireActual('@defra-fish/connectors-lib') @@ -22,13 +16,11 @@ jest.mock('@defra-fish/connectors-lib', () => { }) jest.mock('../../config.js', () => ({ - ftp: { - path: '/ftpservershare/' - }, s3: { bucket: 'testbucket' } })) + describe('s3 operations', () => { beforeEach(() => { jest.clearAllMocks() @@ -36,7 +28,7 @@ describe('s3 operations', () => { }) describe('refreshS3Metadata', () => { - it('gets a list of files from S3', async () => { + it('gets a list of files from S3 and logs file processing', async () => { const s3Key1 = `${moment().format('YYYY-MM-DD')}/test1.xml` const s3Key2 = `${moment().format('YYYY-MM-DD')}/test2.xml` @@ -44,21 +36,12 @@ describe('s3 operations', () => { promise: () => ({ IsTruncated: false, Contents: [ - { - Key: s3Key1, - LastModified: moment().toISOString(), - ETag: 'example-md5', - Size: 1024 - }, - { - Key: s3Key2, - LastModified: moment().toISOString(), - ETag: 'example-md5', - Size: 2048 - } + { Key: s3Key1, LastModified: moment().toISOString() }, + { Key: s3Key2, LastModified: moment().toISOString() } ] }) }) + const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(() => {}) await refreshS3Metadata() @@ -66,36 +49,9 @@ describe('s3 operations', () => { Bucket: 'testbucket', ContinuationToken: undefined }) - expect(updateFileStagingTable).toHaveBeenNthCalledWith(1, { - filename: 'test1.xml', - md5: 'example-md5', - fileSize: '1 KB', - stage: FILE_STAGE.Pending, - s3Key: s3Key1 - }) - expect(updateFileStagingTable).toHaveBeenNthCalledWith(2, { - filename: 'test2.xml', - md5: 'example-md5', - fileSize: '2 KB', - stage: FILE_STAGE.Pending, - s3Key: s3Key2 - }) - expect(salesApi.upsertTransactionFile).toHaveBeenNthCalledWith(1, 'test1.xml', { - status: DYNAMICS_IMPORT_STAGE.Pending, - dataSource: POST_OFFICE_DATASOURCE, - fileSize: '1 KB', - receiptTimestamp: expect.any(String), - salesDate: expect.any(String), - notes: 'Retrieved from the remote server and awaiting processing' - }) - expect(salesApi.upsertTransactionFile).toHaveBeenNthCalledWith(2, 'test2.xml', { - status: DYNAMICS_IMPORT_STAGE.Pending, - dataSource: POST_OFFICE_DATASOURCE, - fileSize: '2 KB', - receiptTimestamp: expect.any(String), - salesDate: expect.any(String), - notes: 'Retrieved from the remote server and awaiting processing' - }) + expect(consoleLogSpy).toHaveBeenCalledWith('Processing 2 S3 files') + expect(consoleLogSpy).toHaveBeenCalledWith('Processing test1.xml') + expect(consoleLogSpy).toHaveBeenCalledWith('Processing test2.xml') }) it('gets a truncated list of files from S3', async () => { @@ -106,12 +62,7 @@ describe('s3 operations', () => { promise: () => ({ IsTruncated: false, Contents: [ - { - Key: s3Key1, - LastModified: moment().toISOString(), - ETag: 'example-md5', - Size: 1024 - } + { Key: s3Key1, LastModified: moment().toISOString() } ] }) }) @@ -120,15 +71,11 @@ describe('s3 operations', () => { IsTruncated: true, NextContinuationToken: 'token', Contents: [ - { - Key: s3Key1, - LastModified: moment().toISOString(), - ETag: 'example-md5', - Size: 1024 - } + { Key: s3Key1, LastModified: moment().toISOString() } ] }) }) + const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(() => {}) await refreshS3Metadata() @@ -140,85 +87,8 @@ describe('s3 operations', () => { Bucket: 'testbucket', ContinuationToken: 'token' }) - expect(updateFileStagingTable).toHaveBeenNthCalledWith(1, { - filename: 'test1.xml', - md5: 'example-md5', - fileSize: '1 KB', - stage: FILE_STAGE.Pending, - s3Key: s3Key1 - }) - expect(salesApi.upsertTransactionFile).toHaveBeenNthCalledWith(1, 'test1.xml', { - status: DYNAMICS_IMPORT_STAGE.Pending, - dataSource: POST_OFFICE_DATASOURCE, - fileSize: '1 KB', - receiptTimestamp: expect.any(String), - salesDate: expect.any(String), - notes: 'Retrieved from the remote server and awaiting processing' - }) - }) - - it('skips file processing if a file has already been marked as processed in Dynamics', async () => { - fs.createReadStream.mockReturnValueOnce('teststream') - fs.statSync.mockReturnValueOnce({ size: 1024 }) - salesApi.getTransactionFile.mockResolvedValueOnce({ status: { description: 'Processed' } }) - const s3Key = `${moment().format('YYYY-MM-DD')}/test-already-processed.xml` - - AwsMock.S3.mockedMethods.listObjectsV2.mockReturnValueOnce({ - promise: () => ({ - IsTruncated: false, - Contents: [ - { - Key: s3Key, - LastModified: moment().toISOString(), - ETag: 'example-md5', - Size: 1024 - } - ] - }) - }) - - await refreshS3Metadata() - - expect(updateFileStagingTable).not.toHaveBeenCalled() - expect(salesApi.upsertTransactionFile).not.toHaveBeenCalled() - }) - - it('skips file processing if a file is older than one week', async () => { - const s3Key1 = `${moment().format('YYYY-MM-DD')}/test1.xml` - - AwsMock.S3.mockedMethods.listObjectsV2 - .mockReturnValue({ - promise: () => ({ - IsTruncated: false, - Contents: [ - { - Key: s3Key1, - LastModified: moment().subtract(1, 'days').toISOString(), - ETag: 'example-md5', - Size: 1024 - } - ] - }) - }) - .mockReturnValueOnce({ - promise: () => ({ - IsTruncated: true, - NextContinuationToken: 'token', - Contents: [ - { - Key: s3Key1, - LastModified: moment().subtract(1, 'days').toISOString(), - ETag: 'example-md5', - Size: 1024 - } - ] - }) - }) - - await refreshS3Metadata() - - expect(updateFileStagingTable).not.toHaveBeenCalled() - expect(salesApi.upsertTransactionFile).not.toHaveBeenCalled() + expect(consoleLogSpy).toHaveBeenCalledWith('Processing 1 S3 files') + expect(consoleLogSpy).toHaveBeenCalledWith('Processing test1.xml') }) it('logs any errors raised by calling s3.listObjectsV2', async () => { From 1642c2fd57c1795a44117bf76e8d18f988ce0026 Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Thu, 10 Oct 2024 08:52:29 +0100 Subject: [PATCH 06/19] fix tests and lint --- packages/pocl-job/src/io/__tests__/s3.spec.js | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/pocl-job/src/io/__tests__/s3.spec.js b/packages/pocl-job/src/io/__tests__/s3.spec.js index 199e8b6b6c..1285390740 100644 --- a/packages/pocl-job/src/io/__tests__/s3.spec.js +++ b/packages/pocl-job/src/io/__tests__/s3.spec.js @@ -61,18 +61,14 @@ describe('s3 operations', () => { .mockReturnValue({ promise: () => ({ IsTruncated: false, - Contents: [ - { Key: s3Key1, LastModified: moment().toISOString() } - ] + Contents: [{ Key: s3Key1, LastModified: moment().toISOString() }] }) }) .mockReturnValueOnce({ promise: () => ({ IsTruncated: true, NextContinuationToken: 'token', - Contents: [ - { Key: s3Key1, LastModified: moment().toISOString() } - ] + Contents: [{ Key: s3Key1, LastModified: moment().toISOString() }] }) }) const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(() => {}) From 1fc085c71ee11dd34433fb54d74e9aad0bdac5f8 Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Thu, 10 Oct 2024 09:03:21 +0100 Subject: [PATCH 07/19] update tests --- packages/pocl-job/src/io/__tests__/s3.spec.js | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/pocl-job/src/io/__tests__/s3.spec.js b/packages/pocl-job/src/io/__tests__/s3.spec.js index 1285390740..af65f919e6 100644 --- a/packages/pocl-job/src/io/__tests__/s3.spec.js +++ b/packages/pocl-job/src/io/__tests__/s3.spec.js @@ -57,20 +57,21 @@ describe('s3 operations', () => { it('gets a truncated list of files from S3', async () => { const s3Key1 = `${moment().format('YYYY-MM-DD')}/test1.xml` - AwsMock.S3.mockedMethods.listObjectsV2 - .mockReturnValue({ - promise: () => ({ - IsTruncated: false, - Contents: [{ Key: s3Key1, LastModified: moment().toISOString() }] - }) + AwsMock.S3.mockedMethods.listObjectsV2.mockReturnValueOnce({ + promise: () => ({ + IsTruncated: true, + NextContinuationToken: 'token', + Contents: [{ Key: s3Key1, LastModified: moment().toISOString() }] }) - .mockReturnValueOnce({ - promise: () => ({ - IsTruncated: true, - NextContinuationToken: 'token', - Contents: [{ Key: s3Key1, LastModified: moment().toISOString() }] - }) + }) + + AwsMock.S3.mockedMethods.listObjectsV2.mockReturnValueOnce({ + promise: () => ({ + IsTruncated: false, + Contents: [{ Key: s3Key1, LastModified: moment().toISOString() }] }) + }) + const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(() => {}) await refreshS3Metadata() @@ -85,6 +86,7 @@ describe('s3 operations', () => { }) expect(consoleLogSpy).toHaveBeenCalledWith('Processing 1 S3 files') expect(consoleLogSpy).toHaveBeenCalledWith('Processing test1.xml') + expect(consoleLogSpy).toHaveBeenCalledTimes(3) }) it('logs any errors raised by calling s3.listObjectsV2', async () => { From 79c749c1fef64d37174f02c15bcb5ef83a016eea Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Thu, 10 Oct 2024 09:13:52 +0100 Subject: [PATCH 08/19] refactor --- packages/pocl-job/src/io/__tests__/s3.spec.js | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/packages/pocl-job/src/io/__tests__/s3.spec.js b/packages/pocl-job/src/io/__tests__/s3.spec.js index af65f919e6..c7cb127e12 100644 --- a/packages/pocl-job/src/io/__tests__/s3.spec.js +++ b/packages/pocl-job/src/io/__tests__/s3.spec.js @@ -28,6 +28,10 @@ describe('s3 operations', () => { }) describe('refreshS3Metadata', () => { + afterEach(() => { + jest.clearAllMocks() + }) + it('gets a list of files from S3 and logs file processing', async () => { const s3Key1 = `${moment().format('YYYY-MM-DD')}/test1.xml` const s3Key2 = `${moment().format('YYYY-MM-DD')}/test2.xml` @@ -57,18 +61,26 @@ describe('s3 operations', () => { it('gets a truncated list of files from S3', async () => { const s3Key1 = `${moment().format('YYYY-MM-DD')}/test1.xml` - AwsMock.S3.mockedMethods.listObjectsV2.mockReturnValueOnce({ + AwsMock.S3.mockedMethods.listObjectsV2.mockReturnValue({ promise: () => ({ - IsTruncated: true, - NextContinuationToken: 'token', - Contents: [{ Key: s3Key1, LastModified: moment().toISOString() }] + IsTruncated: false, + Contents: [ + { + Key: s3Key1, + LastModified: moment().toISOString() + } + ] }) - }) - - AwsMock.S3.mockedMethods.listObjectsV2.mockReturnValueOnce({ + }).mockReturnValueOnce({ promise: () => ({ - IsTruncated: false, - Contents: [{ Key: s3Key1, LastModified: moment().toISOString() }] + IsTruncated: true, + NextContinuationToken: 'token', + Contents: [ + { + Key: s3Key1, + LastModified: moment().toISOString() + } + ] }) }) From a559e21d761480df0b48181bf13a29948c4972c1 Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Thu, 10 Oct 2024 09:15:31 +0100 Subject: [PATCH 09/19] refactor --- packages/pocl-job/src/io/__tests__/s3.spec.js | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/packages/pocl-job/src/io/__tests__/s3.spec.js b/packages/pocl-job/src/io/__tests__/s3.spec.js index c7cb127e12..d0cf870db4 100644 --- a/packages/pocl-job/src/io/__tests__/s3.spec.js +++ b/packages/pocl-job/src/io/__tests__/s3.spec.js @@ -61,28 +61,30 @@ describe('s3 operations', () => { it('gets a truncated list of files from S3', async () => { const s3Key1 = `${moment().format('YYYY-MM-DD')}/test1.xml` - AwsMock.S3.mockedMethods.listObjectsV2.mockReturnValue({ - promise: () => ({ - IsTruncated: false, - Contents: [ - { - Key: s3Key1, - LastModified: moment().toISOString() - } - ] + AwsMock.S3.mockedMethods.listObjectsV2 + .mockReturnValue({ + promise: () => ({ + IsTruncated: false, + Contents: [ + { + Key: s3Key1, + LastModified: moment().toISOString() + } + ] + }) }) - }).mockReturnValueOnce({ - promise: () => ({ - IsTruncated: true, - NextContinuationToken: 'token', - Contents: [ - { - Key: s3Key1, - LastModified: moment().toISOString() - } - ] + .mockReturnValueOnce({ + promise: () => ({ + IsTruncated: true, + NextContinuationToken: 'token', + Contents: [ + { + Key: s3Key1, + LastModified: moment().toISOString() + } + ] + }) }) - }) const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(() => {}) From a1eff19e1aceb35ac7db8f2d42fbaaacc746df9b Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Thu, 10 Oct 2024 14:09:32 +0100 Subject: [PATCH 10/19] update tests --- packages/pocl-job/src/io/__tests__/s3.spec.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/pocl-job/src/io/__tests__/s3.spec.js b/packages/pocl-job/src/io/__tests__/s3.spec.js index d0cf870db4..5b43bf3fc0 100644 --- a/packages/pocl-job/src/io/__tests__/s3.spec.js +++ b/packages/pocl-job/src/io/__tests__/s3.spec.js @@ -98,9 +98,10 @@ describe('s3 operations', () => { Bucket: 'testbucket', ContinuationToken: 'token' }) - expect(consoleLogSpy).toHaveBeenCalledWith('Processing 1 S3 files') + expect(consoleLogSpy).toHaveBeenCalledWith('Processing 2 S3 files') + expect(consoleLogSpy).toHaveBeenCalledWith('Processing test1.xml') expect(consoleLogSpy).toHaveBeenCalledWith('Processing test1.xml') - expect(consoleLogSpy).toHaveBeenCalledTimes(3) + expect(consoleLogSpy).toHaveBeenCalledWith('Processed S3 files') }) it('logs any errors raised by calling s3.listObjectsV2', async () => { From 77e61495eeb4b8c332d77c2efbfddeca3b28953b Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Mon, 14 Oct 2024 09:53:21 +0100 Subject: [PATCH 11/19] refactor tests --- packages/pocl-job/src/io/__tests__/s3.spec.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pocl-job/src/io/__tests__/s3.spec.js b/packages/pocl-job/src/io/__tests__/s3.spec.js index 5b43bf3fc0..1e55501fc3 100644 --- a/packages/pocl-job/src/io/__tests__/s3.spec.js +++ b/packages/pocl-job/src/io/__tests__/s3.spec.js @@ -56,13 +56,14 @@ describe('s3 operations', () => { expect(consoleLogSpy).toHaveBeenCalledWith('Processing 2 S3 files') expect(consoleLogSpy).toHaveBeenCalledWith('Processing test1.xml') expect(consoleLogSpy).toHaveBeenCalledWith('Processing test2.xml') + expect(consoleLogSpy).toHaveBeenCalledWith('Processed S3 files') }) it('gets a truncated list of files from S3', async () => { const s3Key1 = `${moment().format('YYYY-MM-DD')}/test1.xml` AwsMock.S3.mockedMethods.listObjectsV2 - .mockReturnValue({ + .mockReturnValueOnce({ promise: () => ({ IsTruncated: false, Contents: [ @@ -73,7 +74,7 @@ describe('s3 operations', () => { ] }) }) - .mockReturnValueOnce({ + .mockReturnValue({ promise: () => ({ IsTruncated: true, NextContinuationToken: 'token', @@ -98,8 +99,7 @@ describe('s3 operations', () => { Bucket: 'testbucket', ContinuationToken: 'token' }) - expect(consoleLogSpy).toHaveBeenCalledWith('Processing 2 S3 files') - expect(consoleLogSpy).toHaveBeenCalledWith('Processing test1.xml') + expect(consoleLogSpy).toHaveBeenCalledWith('Processing 1 S3 files') expect(consoleLogSpy).toHaveBeenCalledWith('Processing test1.xml') expect(consoleLogSpy).toHaveBeenCalledWith('Processed S3 files') }) From 700b6fcdee32b92772c591440f03208e924fa73a Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Mon, 14 Oct 2024 09:58:28 +0100 Subject: [PATCH 12/19] s3spec --- packages/pocl-job/src/io/__tests__/s3.spec.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/pocl-job/src/io/__tests__/s3.spec.js b/packages/pocl-job/src/io/__tests__/s3.spec.js index 1e55501fc3..04e89d5edf 100644 --- a/packages/pocl-job/src/io/__tests__/s3.spec.js +++ b/packages/pocl-job/src/io/__tests__/s3.spec.js @@ -91,10 +91,6 @@ describe('s3 operations', () => { await refreshS3Metadata() - expect(AwsMock.S3.mockedMethods.listObjectsV2).toHaveBeenNthCalledWith(1, { - Bucket: 'testbucket', - ContinuationToken: undefined - }) expect(AwsMock.S3.mockedMethods.listObjectsV2).toHaveBeenNthCalledWith(2, { Bucket: 'testbucket', ContinuationToken: 'token' From 4edef1dfbbdfedf27efdab3147fa01d47b9f0ae2 Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Mon, 14 Oct 2024 10:02:53 +0100 Subject: [PATCH 13/19] undefined token --- packages/pocl-job/src/io/__tests__/s3.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pocl-job/src/io/__tests__/s3.spec.js b/packages/pocl-job/src/io/__tests__/s3.spec.js index 04e89d5edf..19e371dd4c 100644 --- a/packages/pocl-job/src/io/__tests__/s3.spec.js +++ b/packages/pocl-job/src/io/__tests__/s3.spec.js @@ -91,9 +91,9 @@ describe('s3 operations', () => { await refreshS3Metadata() - expect(AwsMock.S3.mockedMethods.listObjectsV2).toHaveBeenNthCalledWith(2, { + expect(AwsMock.S3.mockedMethods.listObjectsV2).toHaveBeenNthCalledWith(1, { Bucket: 'testbucket', - ContinuationToken: 'token' + ContinuationToken: undefined }) expect(consoleLogSpy).toHaveBeenCalledWith('Processing 1 S3 files') expect(consoleLogSpy).toHaveBeenCalledWith('Processing test1.xml') From 6023b924b99f49b13eace709723893a599882d90 Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Mon, 21 Oct 2024 11:44:22 +0100 Subject: [PATCH 14/19] rename file and undo removal of stores3 --- packages/pocl-job/src/io/__tests__/s3.spec.js | 167 +++++++++++++++--- packages/pocl-job/src/io/s3.js | 12 +- .../__tests__/storeS3MetaData.spec.js | 64 +++++++ .../pocl-job/src/transport/storeS3MetaData.js | 21 +++ 4 files changed, 241 insertions(+), 23 deletions(-) create mode 100644 packages/pocl-job/src/transport/__tests__/storeS3MetaData.spec.js create mode 100644 packages/pocl-job/src/transport/storeS3MetaData.js diff --git a/packages/pocl-job/src/io/__tests__/s3.spec.js b/packages/pocl-job/src/io/__tests__/s3.spec.js index 19e371dd4c..4a1a2da226 100644 --- a/packages/pocl-job/src/io/__tests__/s3.spec.js +++ b/packages/pocl-job/src/io/__tests__/s3.spec.js @@ -1,9 +1,16 @@ import { refreshS3Metadata } from '../s3' import moment from 'moment' +import { updateFileStagingTable } from '../../io/db.js' +import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../../staging/constants.js' +import { salesApi } from '@defra-fish/connectors-lib' +import fs from 'fs' import AwsMock from 'aws-sdk' +import { mockedFtpMethods } from 'ssh2-sftp-client' jest.mock('fs') jest.mock('md5-file') +jest.mock('../../io/db.js') +jest.mock('../../io/file.js') jest.mock('@defra-fish/connectors-lib', () => { const actual = jest.requireActual('@defra-fish/connectors-lib') @@ -16,11 +23,13 @@ jest.mock('@defra-fish/connectors-lib', () => { }) jest.mock('../../config.js', () => ({ + ftp: { + path: '/ftpservershare/' + }, s3: { bucket: 'testbucket' } })) - describe('s3 operations', () => { beforeEach(() => { jest.clearAllMocks() @@ -28,11 +37,7 @@ describe('s3 operations', () => { }) describe('refreshS3Metadata', () => { - afterEach(() => { - jest.clearAllMocks() - }) - - it('gets a list of files from S3 and logs file processing', async () => { + it('gets a list of files from S3', async () => { const s3Key1 = `${moment().format('YYYY-MM-DD')}/test1.xml` const s3Key2 = `${moment().format('YYYY-MM-DD')}/test2.xml` @@ -40,12 +45,21 @@ describe('s3 operations', () => { promise: () => ({ IsTruncated: false, Contents: [ - { Key: s3Key1, LastModified: moment().toISOString() }, - { Key: s3Key2, LastModified: moment().toISOString() } + { + Key: s3Key1, + LastModified: moment().toISOString(), + ETag: 'example-md5', + Size: 1024 + }, + { + Key: s3Key2, + LastModified: moment().toISOString(), + ETag: 'example-md5', + Size: 2048 + } ] }) }) - const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(() => {}) await refreshS3Metadata() @@ -53,51 +67,160 @@ describe('s3 operations', () => { Bucket: 'testbucket', ContinuationToken: undefined }) - expect(consoleLogSpy).toHaveBeenCalledWith('Processing 2 S3 files') - expect(consoleLogSpy).toHaveBeenCalledWith('Processing test1.xml') - expect(consoleLogSpy).toHaveBeenCalledWith('Processing test2.xml') - expect(consoleLogSpy).toHaveBeenCalledWith('Processed S3 files') + expect(updateFileStagingTable).toHaveBeenNthCalledWith(1, { + filename: 'test1.xml', + md5: 'example-md5', + fileSize: '1 KB', + stage: FILE_STAGE.Pending, + s3Key: s3Key1 + }) + expect(updateFileStagingTable).toHaveBeenNthCalledWith(2, { + filename: 'test2.xml', + md5: 'example-md5', + fileSize: '2 KB', + stage: FILE_STAGE.Pending, + s3Key: s3Key2 + }) + expect(salesApi.upsertTransactionFile).toHaveBeenNthCalledWith(1, 'test1.xml', { + status: DYNAMICS_IMPORT_STAGE.Pending, + dataSource: POST_OFFICE_DATASOURCE, + fileSize: '1 KB', + receiptTimestamp: expect.any(String), + salesDate: expect.any(String), + notes: 'Retrieved from the remote server and awaiting processing' + }) + expect(salesApi.upsertTransactionFile).toHaveBeenNthCalledWith(2, 'test2.xml', { + status: DYNAMICS_IMPORT_STAGE.Pending, + dataSource: POST_OFFICE_DATASOURCE, + fileSize: '2 KB', + receiptTimestamp: expect.any(String), + salesDate: expect.any(String), + notes: 'Retrieved from the remote server and awaiting processing' + }) }) it('gets a truncated list of files from S3', async () => { const s3Key1 = `${moment().format('YYYY-MM-DD')}/test1.xml` AwsMock.S3.mockedMethods.listObjectsV2 - .mockReturnValueOnce({ + .mockReturnValue({ promise: () => ({ IsTruncated: false, Contents: [ { Key: s3Key1, - LastModified: moment().toISOString() + LastModified: moment().toISOString(), + ETag: 'example-md5', + Size: 1024 } ] }) }) - .mockReturnValue({ + .mockReturnValueOnce({ promise: () => ({ IsTruncated: true, NextContinuationToken: 'token', Contents: [ { Key: s3Key1, - LastModified: moment().toISOString() + LastModified: moment().toISOString(), + ETag: 'example-md5', + Size: 1024 } ] }) }) - const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(() => {}) - await refreshS3Metadata() expect(AwsMock.S3.mockedMethods.listObjectsV2).toHaveBeenNthCalledWith(1, { Bucket: 'testbucket', ContinuationToken: undefined }) - expect(consoleLogSpy).toHaveBeenCalledWith('Processing 1 S3 files') - expect(consoleLogSpy).toHaveBeenCalledWith('Processing test1.xml') - expect(consoleLogSpy).toHaveBeenCalledWith('Processed S3 files') + expect(AwsMock.S3.mockedMethods.listObjectsV2).toHaveBeenNthCalledWith(2, { + Bucket: 'testbucket', + ContinuationToken: 'token' + }) + expect(updateFileStagingTable).toHaveBeenNthCalledWith(1, { + filename: 'test1.xml', + md5: 'example-md5', + fileSize: '1 KB', + stage: FILE_STAGE.Pending, + s3Key: s3Key1 + }) + expect(salesApi.upsertTransactionFile).toHaveBeenNthCalledWith(1, 'test1.xml', { + status: DYNAMICS_IMPORT_STAGE.Pending, + dataSource: POST_OFFICE_DATASOURCE, + fileSize: '1 KB', + receiptTimestamp: expect.any(String), + salesDate: expect.any(String), + notes: 'Retrieved from the remote server and awaiting processing' + }) + }) + + it('skips file processing if a file has already been marked as processed in Dynamics', async () => { + mockedFtpMethods.list.mockResolvedValue([{ name: 'test-already-processed.xml' }]) + fs.createReadStream.mockReturnValueOnce('teststream') + fs.statSync.mockReturnValueOnce({ size: 1024 }) + salesApi.getTransactionFile.mockResolvedValueOnce({ status: { description: 'Processed' } }) + const s3Key = `${moment().format('YYYY-MM-DD')}/test-already-processed.xml` + + AwsMock.S3.mockedMethods.listObjectsV2.mockReturnValueOnce({ + promise: () => ({ + IsTruncated: false, + Contents: [ + { + Key: s3Key, + LastModified: moment().toISOString(), + ETag: 'example-md5', + Size: 1024 + } + ] + }) + }) + + await refreshS3Metadata() + + expect(updateFileStagingTable).not.toHaveBeenCalled() + expect(salesApi.upsertTransactionFile).not.toHaveBeenCalled() + }) + + it('skips file processing if a file is older than one week', async () => { + const s3Key1 = `${moment().format('YYYY-MM-DD')}/test1.xml` + + AwsMock.S3.mockedMethods.listObjectsV2 + .mockReturnValue({ + promise: () => ({ + IsTruncated: false, + Contents: [ + { + Key: s3Key1, + LastModified: moment().subtract(1, 'days').toISOString(), + ETag: 'example-md5', + Size: 1024 + } + ] + }) + }) + .mockReturnValueOnce({ + promise: () => ({ + IsTruncated: true, + NextContinuationToken: 'token', + Contents: [ + { + Key: s3Key1, + LastModified: moment().subtract(1, 'days').toISOString(), + ETag: 'example-md5', + Size: 1024 + } + ] + }) + }) + + await refreshS3Metadata() + + expect(updateFileStagingTable).not.toHaveBeenCalled() + expect(salesApi.upsertTransactionFile).not.toHaveBeenCalled() }) it('logs any errors raised by calling s3.listObjectsV2', async () => { diff --git a/packages/pocl-job/src/io/s3.js b/packages/pocl-job/src/io/s3.js index cb94087563..66a97cdbfb 100644 --- a/packages/pocl-job/src/io/s3.js +++ b/packages/pocl-job/src/io/s3.js @@ -1,6 +1,9 @@ import moment from 'moment' +import filesize from 'filesize' import config from '../config.js' -import { AWS } from '@defra-fish/connectors-lib' +import { DYNAMICS_IMPORT_STAGE } from '../staging/constants.js' +import { storeS3Metadata } from '../transport/storeS3MetaData.js' +import { AWS, salesApi } from '@defra-fish/connectors-lib' const { s3 } = AWS() const listObjectsV2 = async function (params) { @@ -44,6 +47,13 @@ export const refreshS3Metadata = async () => { const filename = file.Key.split('/').pop() console.log(`Processing ${filename}`) + + const dynamicsRecord = await salesApi.getTransactionFile(filename) + if (!dynamicsRecord || !DYNAMICS_IMPORT_STAGE.isAlreadyProcessed(dynamicsRecord.status.description)) { + await storeS3Metadata(file.ETag, filesize(file.Size), filename, file.Key, moment(new Date(file.LastModified))) + } else { + console.log(`${file.Key} is already processed, skipping`) + } } console.log('Processed S3 files') diff --git a/packages/pocl-job/src/transport/__tests__/storeS3MetaData.spec.js b/packages/pocl-job/src/transport/__tests__/storeS3MetaData.spec.js new file mode 100644 index 0000000000..57d4434d65 --- /dev/null +++ b/packages/pocl-job/src/transport/__tests__/storeS3MetaData.spec.js @@ -0,0 +1,64 @@ +import moment from 'moment' +import { salesApi } from '@defra-fish/connectors-lib' +import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../../staging/constants.js' +import { updateFileStagingTable } from '../../io/db.js' +import { storeS3Metadata } from '../storeS3MetaData.js' + +jest.mock('../../io/db.js', () => ({ + updateFileStagingTable: jest.fn() +})) +jest.mock('@defra-fish/connectors-lib', () => ({ + salesApi: { + upsertTransactionFile: jest.fn() + } +})) + +describe('storeS3Metadata', () => { + const md5 = 'mockMd5Hash' + const fileSize = 12345 + const filename = 'testfile' + const s3Key = 'mock/s3/key/testfile' + const receiptMoment = new Date('2024-10-17T00:00:00Z') + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('console log should output "Storing metadata for s3Key"', async () => { + const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(jest.fn()) + await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) + expect(consoleLogSpy).toHaveBeenCalledWith(`Storing metadata for ${s3Key}`) + }) + + it('should call updateFileStagingTable with correct arguments', async () => { + await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) + expect(updateFileStagingTable).toHaveBeenCalledWith({ + filename, + md5, + fileSize, + s3Key, + stage: FILE_STAGE.Pending + }) + }) + + it('should call salesApi.upsertTransactionFile with correct arguments', async () => { + const expectedSalesDate = moment(receiptMoment).subtract(1, 'days').toISOString() + const expectedReceiptTimestamp = receiptMoment.toISOString() + + await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) + expect(salesApi.upsertTransactionFile).toHaveBeenCalledWith(filename, { + status: DYNAMICS_IMPORT_STAGE.Pending, + dataSource: POST_OFFICE_DATASOURCE, + fileSize: fileSize, + salesDate: expectedSalesDate, + receiptTimestamp: expectedReceiptTimestamp, + notes: 'Retrieved from the remote server and awaiting processing' + }) + }) + + test('should log "Stored metadata for s3Key"', async () => { + const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(jest.fn()) + await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) + expect(consoleLogSpy).toHaveBeenCalledWith(`Stored metadata for ${s3Key}`) + }) +}) diff --git a/packages/pocl-job/src/transport/storeS3MetaData.js b/packages/pocl-job/src/transport/storeS3MetaData.js new file mode 100644 index 0000000000..a60db0e40d --- /dev/null +++ b/packages/pocl-job/src/transport/storeS3MetaData.js @@ -0,0 +1,21 @@ + +import moment from 'moment' +import { salesApi } from '@defra-fish/connectors-lib' +import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../staging/constants.js' +import { updateFileStagingTable } from '../io/db.js' + +export async function storeS3Metadata (md5, fileSize, filename, s3Key, receiptMoment) { + console.log(`Storing metadata for ${s3Key}`) + await updateFileStagingTable({ filename, md5, fileSize, s3Key, stage: FILE_STAGE.Pending }) + + await salesApi.upsertTransactionFile(filename, { + status: DYNAMICS_IMPORT_STAGE.Pending, + dataSource: POST_OFFICE_DATASOURCE, + fileSize: fileSize, + salesDate: moment(receiptMoment).subtract(1, 'days').toISOString(), + receiptTimestamp: receiptMoment.toISOString(), + notes: 'Retrieved from the remote server and awaiting processing' + }) + + console.log(`Stored metadata for ${s3Key}`) +} From c7901b19286c3e743910d34ef38c3029e1f8bd6a Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Mon, 21 Oct 2024 11:45:02 +0100 Subject: [PATCH 15/19] rename file and undo removal of stores3 --- packages/pocl-job/src/transport/storeS3MetaData.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/pocl-job/src/transport/storeS3MetaData.js b/packages/pocl-job/src/transport/storeS3MetaData.js index a60db0e40d..31eb6a4810 100644 --- a/packages/pocl-job/src/transport/storeS3MetaData.js +++ b/packages/pocl-job/src/transport/storeS3MetaData.js @@ -1,4 +1,3 @@ - import moment from 'moment' import { salesApi } from '@defra-fish/connectors-lib' import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../staging/constants.js' From 0fc4c944f65403cce50bb9963c4fd2e2ea0a990b Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Mon, 21 Oct 2024 11:54:49 +0100 Subject: [PATCH 16/19] add missing mock --- packages/pocl-job/src/__mocks__/ssh2-sftp-client.js | 12 ++++++++++++ .../pocl-job/src/__tests__/pocl-processor.spec.js | 1 + 2 files changed, 13 insertions(+) create mode 100644 packages/pocl-job/src/__mocks__/ssh2-sftp-client.js diff --git a/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js b/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js new file mode 100644 index 0000000000..b6e7121c0e --- /dev/null +++ b/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js @@ -0,0 +1,12 @@ +const ssh2sftpClient = jest.genMockFromModule('ssh2-sftp-client') + +export const mockedFtpMethods = { + connect: jest.fn(), + list: jest.fn(), + fastGet: jest.fn(), + delete: jest.fn(), + end: jest.fn() +} +ssh2sftpClient.mockImplementation(() => mockedFtpMethods) + +export default ssh2sftpClient diff --git a/packages/pocl-job/src/__tests__/pocl-processor.spec.js b/packages/pocl-job/src/__tests__/pocl-processor.spec.js index f358861b78..ade97101d7 100644 --- a/packages/pocl-job/src/__tests__/pocl-processor.spec.js +++ b/packages/pocl-job/src/__tests__/pocl-processor.spec.js @@ -41,6 +41,7 @@ jest.mock('../config.js', () => ({ bucket: 'testbucket' } })) +jest.mock('../transport/storeS3MetaData.js') jest.mock('../transport/s3-to-local.js') jest.mock('../io/db.js') jest.mock('../io/s3.js') From 3201f3a4b5c235fe51c8b3fda48b8c67525aa11d Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Mon, 21 Oct 2024 13:23:25 +0100 Subject: [PATCH 17/19] Remove ssh2 mock --- packages/pocl-job/src/__mocks__/ssh2-sftp-client.js | 12 ------------ packages/pocl-job/src/io/__tests__/s3.spec.js | 2 -- 2 files changed, 14 deletions(-) delete mode 100644 packages/pocl-job/src/__mocks__/ssh2-sftp-client.js diff --git a/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js b/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js deleted file mode 100644 index b6e7121c0e..0000000000 --- a/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js +++ /dev/null @@ -1,12 +0,0 @@ -const ssh2sftpClient = jest.genMockFromModule('ssh2-sftp-client') - -export const mockedFtpMethods = { - connect: jest.fn(), - list: jest.fn(), - fastGet: jest.fn(), - delete: jest.fn(), - end: jest.fn() -} -ssh2sftpClient.mockImplementation(() => mockedFtpMethods) - -export default ssh2sftpClient diff --git a/packages/pocl-job/src/io/__tests__/s3.spec.js b/packages/pocl-job/src/io/__tests__/s3.spec.js index 4a1a2da226..26c7fe9be1 100644 --- a/packages/pocl-job/src/io/__tests__/s3.spec.js +++ b/packages/pocl-job/src/io/__tests__/s3.spec.js @@ -5,7 +5,6 @@ import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../.. import { salesApi } from '@defra-fish/connectors-lib' import fs from 'fs' import AwsMock from 'aws-sdk' -import { mockedFtpMethods } from 'ssh2-sftp-client' jest.mock('fs') jest.mock('md5-file') @@ -159,7 +158,6 @@ describe('s3 operations', () => { }) it('skips file processing if a file has already been marked as processed in Dynamics', async () => { - mockedFtpMethods.list.mockResolvedValue([{ name: 'test-already-processed.xml' }]) fs.createReadStream.mockReturnValueOnce('teststream') fs.statSync.mockReturnValueOnce({ size: 1024 }) salesApi.getTransactionFile.mockResolvedValueOnce({ status: { description: 'Processed' } }) From 76a068eae5e790d2716aed0584b635a04d329c57 Mon Sep 17 00:00:00 2001 From: ScottDormand96 Date: Thu, 31 Oct 2024 09:34:13 +0000 Subject: [PATCH 18/19] remove functionality from config --- .../src/__tests__/config.spec.js | 33 +-------- packages/fulfilment-job/src/config.js | 68 +----------------- .../pocl-job/src/__tests__/config.spec.js | 31 -------- packages/pocl-job/src/config.js | 70 ------------------- 4 files changed, 2 insertions(+), 200 deletions(-) diff --git a/packages/fulfilment-job/src/__tests__/config.spec.js b/packages/fulfilment-job/src/__tests__/config.spec.js index 1cbecefe04..883b6c86c6 100644 --- a/packages/fulfilment-job/src/__tests__/config.spec.js +++ b/packages/fulfilment-job/src/__tests__/config.spec.js @@ -17,11 +17,6 @@ const clearEnvVars = () => { const envVars = Object.freeze({ FULFILMENT_FILE_SIZE: 1234, - FULFILMENT_FTP_HOST: 'test-host', - FULFILMENT_FTP_PORT: 2222, - FULFILMENT_FTP_PATH: '/remote/share', - FULFILMENT_FTP_USERNAME: 'test-user', - FULFILMENT_FTP_KEY_SECRET_ID: 'test-secret-id', FULFILMENT_S3_BUCKET: 'test-bucket', FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID: 'pgp-key-secret-id', FULFILMENT_SEND_UNENCRYPTED_FILE: 'false' @@ -44,32 +39,6 @@ describe('config', () => { }) }) - describe('ftp', () => { - it('provides properties relating the use of SFTP', async () => { - expect(config.ftp).toEqual( - expect.objectContaining({ - host: 'test-host', - port: '2222', - path: '/remote/share', - username: 'test-user', - privateKey: 'test-ssh-key', - algorithms: { cipher: expect.any(Array), kex: expect.any(Array) }, - // Wait up to 60 seconds for the SSH handshake - readyTimeout: expect.any(Number), - // Retry 5 times over a minute - retries: expect.any(Number), - retry_minTimeout: expect.any(Number), - debug: expect.any(Function) - }) - ) - }) - it('defaults the sftp port to 22 if the environment variable is not configured', async () => { - delete process.env.FULFILMENT_FTP_PORT - await config.initialise() - expect(config.ftp.port).toEqual('22') - }) - }) - describe('s3', () => { it('provides properties relating the use of Amazon S3', async () => { expect(config.s3.bucket).toEqual('test-bucket') @@ -79,7 +48,7 @@ describe('config', () => { describe('pgp config', () => { const init = async (samplePublicKey = 'sample-pgp-key') => { - AwsMock.SecretsManager.__setNextResponses('getSecretValue', { SecretString: 'test-ssh-key' }, { SecretString: samplePublicKey }) + AwsMock.SecretsManager.__setNextResponses('getSecretValue', { SecretString: samplePublicKey }) await config.initialise() } beforeAll(setEnvVars) diff --git a/packages/fulfilment-job/src/config.js b/packages/fulfilment-job/src/config.js index bc0620fe45..e0f9c01315 100644 --- a/packages/fulfilment-job/src/config.js +++ b/packages/fulfilment-job/src/config.js @@ -1,45 +1,6 @@ import { AWS } from '@defra-fish/connectors-lib' -import db from 'debug' -const { secretsManager } = AWS() -/** - * Key exchange algorithms for public key authentication - in descending order of priority - * @type {string[]} - */ -export const SFTP_KEY_EXCHANGE_ALGORITHMS = [ - 'curve25519-sha256@libssh.org', - 'curve25519-sha256', - 'ecdh-sha2-nistp521', - 'ecdh-sha2-nistp384', - 'ecdh-sha2-nistp256', - 'diffie-hellman-group-exchange-sha256', - 'diffie-hellman-group14-sha256', - 'diffie-hellman-group16-sha512', - 'diffie-hellman-group18-sha512', - 'diffie-hellman-group14-sha1', - 'diffie-hellman-group-exchange-sha1', - 'diffie-hellman-group1-sha1' -] -/** - * Ciphers for SFTP support - in descending order of priority - * @type {string[]} - */ -export const SFTP_CIPHERS = [ - // http://tools.ietf.org/html/rfc4344#section-4 - 'aes256-ctr', - 'aes192-ctr', - 'aes128-ctr', - 'aes256-gcm', - 'aes256-gcm@openssh.com', - 'aes128-gcm', - 'aes128-gcm@openssh.com', - 'aes256-cbc', - 'aes192-cbc', - 'aes128-cbc', - 'blowfish-cbc', - '3des-cbc', - 'cast128-cbc' -] +const { secretsManager } = AWS() const falseRegEx = /(false|0)/i const trueRegEx = /(true|1)/i const toBoolean = val => { @@ -54,7 +15,6 @@ const toBoolean = val => { class Config { _file - _ftp _s3 _pgp @@ -68,20 +28,6 @@ class Config { */ partFileSize: Math.min(Number.parseInt(process.env.FULFILMENT_FILE_SIZE), 999) } - this.ftp = { - host: process.env.FULFILMENT_FTP_HOST, - port: process.env.FULFILMENT_FTP_PORT || '22', - path: process.env.FULFILMENT_FTP_PATH, - username: process.env.FULFILMENT_FTP_USERNAME, - privateKey: (await secretsManager.getSecretValue({ SecretId: process.env.FULFILMENT_FTP_KEY_SECRET_ID }).promise()).SecretString, - algorithms: { cipher: SFTP_CIPHERS, kex: SFTP_KEY_EXCHANGE_ALGORITHMS }, - // Wait up to 60 seconds for the SSH handshake - readyTimeout: 60000, - // Retry 5 times over a minute - retries: 5, - retry_minTimeout: 12000, - debug: db('fulfilment:ftp') - } this.s3 = { bucket: process.env.FULFILMENT_S3_BUCKET } @@ -104,18 +50,6 @@ class Config { this._file = cfg } - /** - * FTP configuration settings - * @type {object} - */ - get ftp () { - return this._ftp - } - - set ftp (cfg) { - this._ftp = cfg - } - /** * S3 configuration settings * @type {object} diff --git a/packages/pocl-job/src/__tests__/config.spec.js b/packages/pocl-job/src/__tests__/config.spec.js index f0c1c6d8eb..5fd427d0f6 100644 --- a/packages/pocl-job/src/__tests__/config.spec.js +++ b/packages/pocl-job/src/__tests__/config.spec.js @@ -11,11 +11,6 @@ describe('config', () => { process.env.POCL_RECORD_STAGING_TABLE = 'test-record-staging-table' process.env.POCL_STAGING_TTL = 1234 - process.env.POCL_FTP_HOST = 'test-host' - process.env.POCL_FTP_PORT = 2222 - process.env.POCL_FTP_PATH = '/remote/share' - process.env.POCL_FTP_USERNAME = 'test-user' - process.env.POCL_FTP_KEY_SECRET_ID = 'test-secret-id' process.env.POCL_S3_BUCKET = 'test-bucket' await config.initialise() }) @@ -38,32 +33,6 @@ describe('config', () => { }) }) - describe('ftp', () => { - it('provides properties relating the use of SFTP', async () => { - expect(config.ftp).toEqual( - expect.objectContaining({ - host: 'test-host', - port: '2222', - path: '/remote/share', - username: 'test-user', - privateKey: 'test-ssh-key', - algorithms: { cipher: expect.any(Array), kex: expect.any(Array) }, - // Wait up to 60 seconds for the SSH handshake - readyTimeout: expect.any(Number), - // Retry 5 times over a minute - retries: expect.any(Number), - retry_minTimeout: expect.any(Number), - debug: expect.any(Function) - }) - ) - }) - it('defaults the sftp port to 22 if the environment variable is not configured', async () => { - delete process.env.POCL_FTP_PORT - await config.initialise() - expect(config.ftp.port).toEqual('22') - }) - }) - describe('s3', () => { it('provides properties relating the use of Amazon S3', async () => { expect(config.s3.bucket).toEqual('test-bucket') diff --git a/packages/pocl-job/src/config.js b/packages/pocl-job/src/config.js index 0430b4a457..c85a0a74be 100644 --- a/packages/pocl-job/src/config.js +++ b/packages/pocl-job/src/config.js @@ -1,46 +1,3 @@ -import { AWS } from '@defra-fish/connectors-lib' -import db from 'debug' -const { secretsManager } = AWS() - -/** - * Key exchange algorithms for public key authentication - in descending order of priority - * @type {string[]} - */ -export const SFTP_KEY_EXCHANGE_ALGORITHMS = [ - 'curve25519-sha256@libssh.org', - 'curve25519-sha256', - 'ecdh-sha2-nistp521', - 'ecdh-sha2-nistp384', - 'ecdh-sha2-nistp256', - 'diffie-hellman-group-exchange-sha256', - 'diffie-hellman-group14-sha256', - 'diffie-hellman-group16-sha512', - 'diffie-hellman-group18-sha512', - 'diffie-hellman-group14-sha1', - 'diffie-hellman-group-exchange-sha1', - 'diffie-hellman-group1-sha1' -] -/** - * Ciphers for SFTP support - in descending order of priority - * @type {string[]} - */ -export const SFTP_CIPHERS = [ - // http://tools.ietf.org/html/rfc4344#section-4 - 'aes256-ctr', - 'aes192-ctr', - 'aes128-ctr', - 'aes256-gcm', - 'aes256-gcm@openssh.com', - 'aes128-gcm', - 'aes128-gcm@openssh.com', - 'aes256-cbc', - 'aes192-cbc', - 'aes128-cbc', - 'blowfish-cbc', - '3des-cbc', - 'cast128-cbc' -] - class Config { _db _ftp @@ -53,21 +10,6 @@ class Config { stagingTtlDelta: Number.parseInt(process.env.POCL_STAGING_TTL || 60 * 60 * 168) } - this.ftp = { - host: process.env.POCL_FTP_HOST, - port: process.env.POCL_FTP_PORT || '22', - path: process.env.POCL_FTP_PATH, - username: process.env.POCL_FTP_USERNAME, - privateKey: (await secretsManager.getSecretValue({ SecretId: process.env.POCL_FTP_KEY_SECRET_ID }).promise()).SecretString, - algorithms: { cipher: SFTP_CIPHERS, kex: SFTP_KEY_EXCHANGE_ALGORITHMS }, - // Wait up to 60 seconds for the SSH handshake - readyTimeout: 60000, - // Retry 5 times over a minute - retries: 5, - retry_minTimeout: 12000, - debug: db('pocl:ftp') - } - this.s3 = { bucket: process.env.POCL_S3_BUCKET } @@ -85,18 +27,6 @@ class Config { this._db = cfg } - /** - * FTP configuration settings - * @type {object} - */ - get ftp () { - return this._ftp - } - - set ftp (cfg) { - this._ftp = cfg - } - /** * S3 configuration settings * @type {object} From 0778979803c69ea08c16770389702e52233c6343 Mon Sep 17 00:00:00 2001 From: Scott Dormand Date: Fri, 8 Nov 2024 08:34:22 +0000 Subject: [PATCH 19/19] remove ftp reference --- packages/pocl-job/src/config.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/pocl-job/src/config.js b/packages/pocl-job/src/config.js index c85a0a74be..d08132335b 100644 --- a/packages/pocl-job/src/config.js +++ b/packages/pocl-job/src/config.js @@ -1,6 +1,5 @@ class Config { _db - _ftp _s3 async initialise () {