diff --git a/packages/fulfilment-job/package-lock.json b/packages/fulfilment-job/package-lock.json index d7d17dc1fc..abb8549caa 100644 --- a/packages/fulfilment-job/package-lock.json +++ b/packages/fulfilment-job/package-lock.json @@ -16,8 +16,7 @@ "merge2": "^1.4.1", "moment": "^2.29.1", "openpgp": "^5.0.0-1", - "pluralize": "^8.0.0", - "ssh2-sftp-client": "^6.0.1" + "pluralize": "^8.0.0" }, "engines": { "node": ">=18.17" @@ -50,20 +49,10 @@ "node": ">=10" } }, - "node_modules/@dabh/diagnostics": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz", - "integrity": "sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==", - "dependencies": { - "colorspace": "1.1.x", - "enabled": "2.0.x", - "kuler": "^2.0.0" - } - }, "node_modules/@defra-fish/connectors-lib": { - "version": "1.50.0-rc.7", - "resolved": "https://registry.npmjs.org/@defra-fish/connectors-lib/-/connectors-lib-1.50.0-rc.7.tgz", - "integrity": "sha512-DV8J4CnxRuYZTfbX8JCkGhm1gQ4hgScIxdRuoy/X5THMIPaLgDSxBkOH90UT/2D61U8A5QGVfgKjPuKSisfGrA==", + "version": "1.50.0-rc.8", + "resolved": "https://registry.npmjs.org/@defra-fish/connectors-lib/-/connectors-lib-1.50.0-rc.8.tgz", + "integrity": "sha512-aXPMRNfXBF6xytV+TD7C3fbeEmMb0m2VPsaDiq4uN4ShqqTg2q3OuAo6YUZypKOkX2tH6/Hj4Qpb3UMerNJ6hg==", "dependencies": { "@airbrake/node": "^2.1.7", "aws-sdk": "^2.1074.0", @@ -77,9 +66,9 @@ } }, "node_modules/@defra-fish/dynamics-lib": { - "version": "1.50.0-rc.7", - "resolved": "https://registry.npmjs.org/@defra-fish/dynamics-lib/-/dynamics-lib-1.50.0-rc.7.tgz", - "integrity": "sha512-6ttk/rlLlHR12L5Jz4Gn61f+hUer9IEp0LxczOqXyEvQqGwKiivJHZkh80jcL4OlmuFqOOj4Ft1neS1puZ0mwQ==", + "version": "1.50.0-rc.8", + "resolved": "https://registry.npmjs.org/@defra-fish/dynamics-lib/-/dynamics-lib-1.50.0-rc.8.tgz", + "integrity": "sha512-K0d15rfayclGfqcoIrrv9XaHRTXqnvhPu+1cnRmoWHgaDBeKr/YrGB/H2Y0S5Mq4tk78E1QdYIupJY9oXAWuxA==", "dependencies": { "cache-manager": "^3.6.0", "cache-manager-ioredis": "^2.1.0", @@ -202,14 +191,6 @@ "node": ">= 6.0.0" } }, - "node_modules/asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "dependencies": { - "safer-buffer": "~2.1.0" - } - }, "node_modules/asn1.js": { "version": "5.4.1", "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", @@ -293,14 +274,6 @@ } ] }, - "node_modules/bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "dependencies": { - "tweetnacl": "^0.14.3" - } - }, "node_modules/bintrees": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", @@ -326,11 +299,6 @@ "isarray": "^1.0.0" } }, - "node_modules/buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" - }, "node_modules/cache-manager": { "version": "3.6.3", "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-3.6.3.tgz", @@ -378,54 +346,6 @@ "node": ">=0.10.0" } }, - "node_modules/color": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", - "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==", - "dependencies": { - "color-convert": "^1.9.1", - "color-string": "^1.5.2" - } - }, - "node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" - }, - "node_modules/color-string": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.4.tgz", - "integrity": "sha512-57yF5yt8Xa3czSEW1jfQDE79Idk0+AkN/4KWad6tbdxUmAs3MvjxlWSWD4deYytcRfoZ9nhKyFl1kj5tBvidbw==", - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, - "node_modules/colors": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", - "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/colorspace": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz", - "integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==", - "dependencies": { - "color": "3.0.x", - "text-hex": "1.0.x" - } - }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -445,25 +365,6 @@ "node": ">= 10" } }, - "node_modules/concat-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", - "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "engines": [ - "node >= 6.0" - ], - "dependencies": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.0.2", - "typedarray": "^0.0.6" - } - }, - "node_modules/core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, "node_modules/cross-fetch": { "version": "3.1.8", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", @@ -529,16 +430,6 @@ "https-proxy-agent": "^5.0.0" } }, - "node_modules/enabled": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", - "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" - }, - "node_modules/err-code": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", - "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==" - }, "node_modules/error-stack-parser": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.1.4.tgz", @@ -574,21 +465,6 @@ "node": ">=0.4.x" } }, - "node_modules/fast-safe-stringify": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", - "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" - }, - "node_modules/fecha": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.0.tgz", - "integrity": "sha512-aN3pcx/DSmtyoovUudctc8+6Hl4T+hI9GBBHLjA76jdZl7+b1sgh5g4k+u/GL3dTy1/pnYzKp69FpJ0OicE3Wg==" - }, - "node_modules/fn.name": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", - "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" - }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -780,11 +656,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -810,14 +681,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", - "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==", - "engines": { - "node": ">=8" - } - }, "node_modules/is-typed-array": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", @@ -857,11 +720,6 @@ "@sideway/pinpoint": "^2.0.0" } }, - "node_modules/kuler": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", - "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" - }, "node_modules/lodash.clonedeep": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", @@ -882,18 +740,6 @@ "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" }, - "node_modules/logform": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz", - "integrity": "sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg==", - "dependencies": { - "colors": "^1.2.1", - "fast-safe-stringify": "^2.0.4", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "triple-beam": "^1.3.0" - } - }, "node_modules/lru-cache": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", @@ -969,14 +815,6 @@ } } }, - "node_modules/one-time": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", - "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", - "dependencies": { - "fn.name": "1.x.x" - } - }, "node_modules/openpgp": { "version": "5.0.0-1", "resolved": "https://registry.npmjs.org/openpgp/-/openpgp-5.0.0-1.tgz", @@ -1012,28 +850,11 @@ "node": ">= 0.4" } }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, "node_modules/promise-polyfill": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/promise-polyfill/-/promise-polyfill-8.3.0.tgz", "integrity": "sha512-H5oELycFml5yto/atYqmjyigJoAo3+OXwolYiH7OfQuYlAqhxNvTfiNMbV9hsC6Yp83yE5r2KTVmtrG6R9i6Pg==" }, - "node_modules/promise-retry": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", - "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", - "dependencies": { - "err-code": "^2.0.2", - "retry": "^0.12.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -1048,19 +869,6 @@ "node": ">=0.4.x" } }, - "node_modules/readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/redis-commands": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", @@ -1096,33 +904,6 @@ "node": ">=8.0.0" } }, - "node_modules/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=", - "engines": { - "node": ">= 4" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -1160,57 +941,6 @@ "joi": "^17.3.0" } }, - "node_modules/simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=", - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, - "node_modules/ssh2": { - "version": "0.8.9", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-0.8.9.tgz", - "integrity": "sha512-GmoNPxWDMkVpMFa9LVVzQZHF6EW3WKmBwL+4/GeILf2hFmix5Isxm7Amamo8o7bHiU0tC+wXsGcUXOxp8ChPaw==", - "dependencies": { - "ssh2-streams": "~0.4.10" - }, - "engines": { - "node": ">=5.2.0" - } - }, - "node_modules/ssh2-sftp-client": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-6.0.1.tgz", - "integrity": "sha512-Glut2SmK/XpNOBiEuzqlKZGKkIyha2XMbuWVXR2hFUJkNsbyl/wmlZSeUEPxKFp/dC9UEvUKzanKydgLmNdfkw==", - "dependencies": { - "concat-stream": "^2.0.0", - "promise-retry": "^2.0.1", - "ssh2": "^0.8.9", - "winston": "^3.3.3" - } - }, - "node_modules/ssh2-streams": { - "version": "0.4.10", - "resolved": "https://registry.npmjs.org/ssh2-streams/-/ssh2-streams-0.4.10.tgz", - "integrity": "sha512-8pnlMjvnIZJvmTzUIIA5nT4jr2ZWNNVHwyXfMGdRJbug9TpI3kd99ffglgfSWqujVv/0gxwMsDn9j9RVst8yhQ==", - "dependencies": { - "asn1": "~0.2.0", - "bcrypt-pbkdf": "^1.0.2", - "streamsearch": "~0.1.2" - }, - "engines": { - "node": ">=5.2.0" - } - }, - "node_modules/stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=", - "engines": { - "node": "*" - } - }, "node_modules/stackframe": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.3.4.tgz", @@ -1221,22 +951,6 @@ "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" }, - "node_modules/streamsearch": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", - "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, "node_modules/tdigest": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", @@ -1245,31 +959,11 @@ "bintrees": "1.0.2" } }, - "node_modules/text-hex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", - "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" - }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, - "node_modules/triple-beam": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", - "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" - }, - "node_modules/tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" - }, - "node_modules/typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" - }, "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", @@ -1296,11 +990,6 @@ "which-typed-array": "^1.1.2" } }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, "node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -1341,64 +1030,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/winston": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/winston/-/winston-3.3.3.tgz", - "integrity": "sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw==", - "dependencies": { - "@dabh/diagnostics": "^2.0.2", - "async": "^3.1.0", - "is-stream": "^2.0.0", - "logform": "^2.2.0", - "one-time": "^1.0.0", - "readable-stream": "^3.4.0", - "stack-trace": "0.0.x", - "triple-beam": "^1.3.0", - "winston-transport": "^4.4.0" - }, - "engines": { - "node": ">= 6.4.0" - } - }, - "node_modules/winston-transport": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.0.tgz", - "integrity": "sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw==", - "dependencies": { - "readable-stream": "^2.3.7", - "triple-beam": "^1.2.0" - }, - "engines": { - "node": ">= 6.4.0" - } - }, - "node_modules/winston-transport/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/winston-transport/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/winston-transport/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, "node_modules/xml2js": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", @@ -1425,4 +1056,4 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" } } -} \ No newline at end of file +} diff --git a/packages/fulfilment-job/package.json b/packages/fulfilment-job/package.json index 6c9c690d07..470c287620 100644 --- a/packages/fulfilment-job/package.json +++ b/packages/fulfilment-job/package.json @@ -42,7 +42,6 @@ "merge2": "^1.4.1", "moment": "^2.29.1", "openpgp": "^5.0.0-1", - "pluralize": "^8.0.0", - "ssh2-sftp-client": "^6.0.1" + "pluralize": "^8.0.0" } } diff --git a/packages/fulfilment-job/src/__mocks__/ssh2-sftp-client.js b/packages/fulfilment-job/src/__mocks__/ssh2-sftp-client.js deleted file mode 100644 index 124d4777bb..0000000000 --- a/packages/fulfilment-job/src/__mocks__/ssh2-sftp-client.js +++ /dev/null @@ -1,9 +0,0 @@ -const ssh2sftpClient = jest.genMockFromModule('ssh2-sftp-client') - -export const mockedFtpMethods = { - connect: jest.fn(async () => {}), - put: jest.fn(async () => {}), - end: jest.fn() -} -ssh2sftpClient.mockImplementation(() => mockedFtpMethods) -export default ssh2sftpClient diff --git a/packages/fulfilment-job/src/__tests__/config.spec.js b/packages/fulfilment-job/src/__tests__/config.spec.js index 1cbecefe04..883b6c86c6 100644 --- a/packages/fulfilment-job/src/__tests__/config.spec.js +++ b/packages/fulfilment-job/src/__tests__/config.spec.js @@ -17,11 +17,6 @@ const clearEnvVars = () => { const envVars = Object.freeze({ FULFILMENT_FILE_SIZE: 1234, - FULFILMENT_FTP_HOST: 'test-host', - FULFILMENT_FTP_PORT: 2222, - FULFILMENT_FTP_PATH: '/remote/share', - FULFILMENT_FTP_USERNAME: 'test-user', - FULFILMENT_FTP_KEY_SECRET_ID: 'test-secret-id', FULFILMENT_S3_BUCKET: 'test-bucket', FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID: 'pgp-key-secret-id', FULFILMENT_SEND_UNENCRYPTED_FILE: 'false' @@ -44,32 +39,6 @@ describe('config', () => { }) }) - describe('ftp', () => { - it('provides properties relating the use of SFTP', async () => { - expect(config.ftp).toEqual( - expect.objectContaining({ - host: 'test-host', - port: '2222', - path: '/remote/share', - username: 'test-user', - privateKey: 'test-ssh-key', - algorithms: { cipher: expect.any(Array), kex: expect.any(Array) }, - // Wait up to 60 seconds for the SSH handshake - readyTimeout: expect.any(Number), - // Retry 5 times over a minute - retries: expect.any(Number), - retry_minTimeout: expect.any(Number), - debug: expect.any(Function) - }) - ) - }) - it('defaults the sftp port to 22 if the environment variable is not configured', async () => { - delete process.env.FULFILMENT_FTP_PORT - await config.initialise() - expect(config.ftp.port).toEqual('22') - }) - }) - describe('s3', () => { it('provides properties relating the use of Amazon S3', async () => { expect(config.s3.bucket).toEqual('test-bucket') @@ -79,7 +48,7 @@ describe('config', () => { describe('pgp config', () => { const init = async (samplePublicKey = 'sample-pgp-key') => { - AwsMock.SecretsManager.__setNextResponses('getSecretValue', { SecretString: 'test-ssh-key' }, { SecretString: samplePublicKey }) + AwsMock.SecretsManager.__setNextResponses('getSecretValue', { SecretString: samplePublicKey }) await config.initialise() } beforeAll(setEnvVars) diff --git a/packages/fulfilment-job/src/config.js b/packages/fulfilment-job/src/config.js index bc0620fe45..e0f9c01315 100644 --- a/packages/fulfilment-job/src/config.js +++ b/packages/fulfilment-job/src/config.js @@ -1,45 +1,6 @@ import { AWS } from '@defra-fish/connectors-lib' -import db from 'debug' -const { secretsManager } = AWS() -/** - * Key exchange algorithms for public key authentication - in descending order of priority - * @type {string[]} - */ -export const SFTP_KEY_EXCHANGE_ALGORITHMS = [ - 'curve25519-sha256@libssh.org', - 'curve25519-sha256', - 'ecdh-sha2-nistp521', - 'ecdh-sha2-nistp384', - 'ecdh-sha2-nistp256', - 'diffie-hellman-group-exchange-sha256', - 'diffie-hellman-group14-sha256', - 'diffie-hellman-group16-sha512', - 'diffie-hellman-group18-sha512', - 'diffie-hellman-group14-sha1', - 'diffie-hellman-group-exchange-sha1', - 'diffie-hellman-group1-sha1' -] -/** - * Ciphers for SFTP support - in descending order of priority - * @type {string[]} - */ -export const SFTP_CIPHERS = [ - // http://tools.ietf.org/html/rfc4344#section-4 - 'aes256-ctr', - 'aes192-ctr', - 'aes128-ctr', - 'aes256-gcm', - 'aes256-gcm@openssh.com', - 'aes128-gcm', - 'aes128-gcm@openssh.com', - 'aes256-cbc', - 'aes192-cbc', - 'aes128-cbc', - 'blowfish-cbc', - '3des-cbc', - 'cast128-cbc' -] +const { secretsManager } = AWS() const falseRegEx = /(false|0)/i const trueRegEx = /(true|1)/i const toBoolean = val => { @@ -54,7 +15,6 @@ const toBoolean = val => { class Config { _file - _ftp _s3 _pgp @@ -68,20 +28,6 @@ class Config { */ partFileSize: Math.min(Number.parseInt(process.env.FULFILMENT_FILE_SIZE), 999) } - this.ftp = { - host: process.env.FULFILMENT_FTP_HOST, - port: process.env.FULFILMENT_FTP_PORT || '22', - path: process.env.FULFILMENT_FTP_PATH, - username: process.env.FULFILMENT_FTP_USERNAME, - privateKey: (await secretsManager.getSecretValue({ SecretId: process.env.FULFILMENT_FTP_KEY_SECRET_ID }).promise()).SecretString, - algorithms: { cipher: SFTP_CIPHERS, kex: SFTP_KEY_EXCHANGE_ALGORITHMS }, - // Wait up to 60 seconds for the SSH handshake - readyTimeout: 60000, - // Retry 5 times over a minute - retries: 5, - retry_minTimeout: 12000, - debug: db('fulfilment:ftp') - } this.s3 = { bucket: process.env.FULFILMENT_S3_BUCKET } @@ -104,18 +50,6 @@ class Config { this._file = cfg } - /** - * FTP configuration settings - * @type {object} - */ - get ftp () { - return this._ftp - } - - set ftp (cfg) { - this._ftp = cfg - } - /** * S3 configuration settings * @type {object} diff --git a/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js b/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js index 83e2145335..ca157ec115 100644 --- a/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js +++ b/packages/fulfilment-job/src/staging/__tests__/deliver-fulfilment-files.spec.js @@ -1,7 +1,6 @@ import { Readable, PassThrough, Writable } from 'stream' import { deliverFulfilmentFiles } from '../deliver-fulfilment-files.js' import { createS3WriteStream, readS3PartFiles } from '../../transport/s3.js' -import { createFtpWriteStream } from '../../transport/ftp.js' import { FULFILMENT_FILE_STATUS_OPTIONSET, getOptionSetEntry } from '../staging-common.js' import { FulfilmentRequestFile, executeQuery, persist } from '@defra-fish/dynamics-lib' import openpgp from 'openpgp' @@ -10,7 +9,6 @@ import streamHelper from '../streamHelper.js' import merge2 from 'merge2' jest.mock('../../transport/s3.js') -jest.mock('../../transport/ftp.js') jest.mock('openpgp', () => ({ readKey: jest.fn(() => ({})), encrypt: jest.fn(({ message: readableStream }) => readableStream), @@ -46,20 +44,10 @@ describe('deliverFulfilmentFiles', () => { executeQuery.mockResolvedValue([{ entity: mockFulfilmentRequestFile2 }, { entity: mockFulfilmentRequestFile1 }]) // Streams for file1 - const { - s3DataStreamFile: s3DataStreamFile1, - ftpDataStreamFile: ftpDataStreamFile1, - s3HashStreamFile: s3HashStreamFile1, - ftpHashStreamFile: ftpHashStreamFile1 - } = createMockFileStreams() + const { s3DataStreamFile: s3DataStreamFile1, s3HashStreamFile: s3HashStreamFile1 } = createMockFileStreams() // Streams for file2 - const { - s3DataStreamFile: s3DataStreamFile2, - ftpDataStreamFile: ftpDataStreamFile2, - s3HashStreamFile: s3HashStreamFile2, - ftpHashStreamFile: ftpHashStreamFile2 - } = createMockFileStreams() + const { s3DataStreamFile: s3DataStreamFile2, s3HashStreamFile: s3HashStreamFile2 } = createMockFileStreams() // Run the delivery await expect(deliverFulfilmentFiles()).resolves.toBeUndefined() @@ -69,22 +57,14 @@ describe('deliverFulfilmentFiles', () => { // File 1 expectations expect(createS3WriteStream).toHaveBeenNthCalledWith(1, 'EAFF202006180001.json') expect(createS3WriteStream).toHaveBeenNthCalledWith(3, 'EAFF202006180001.json.sha256') - expect(createFtpWriteStream).toHaveBeenNthCalledWith(1, 'EAFF202006180001.json') - expect(createFtpWriteStream).toHaveBeenNthCalledWith(3, 'EAFF202006180001.json.sha256') expect(JSON.parse(s3DataStreamFile1.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] }) - expect(JSON.parse(ftpDataStreamFile1.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] }) expect(s3HashStreamFile1.dataProcessed).toEqual(fileShaHash) // validated - expect(ftpHashStreamFile1.dataProcessed).toEqual(fileShaHash) // validated // File 2 expectations expect(createS3WriteStream).toHaveBeenNthCalledWith(4, 'EAFF202006180002.json') expect(createS3WriteStream).toHaveBeenNthCalledWith(6, 'EAFF202006180002.json.sha256') - expect(createFtpWriteStream).toHaveBeenNthCalledWith(4, 'EAFF202006180002.json') - expect(createFtpWriteStream).toHaveBeenNthCalledWith(6, 'EAFF202006180002.json.sha256') expect(JSON.parse(s3DataStreamFile2.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] }) - expect(JSON.parse(ftpDataStreamFile2.dataProcessed)).toEqual({ licences: [{ part: 0 }, { part: 1 }] }) expect(s3HashStreamFile2.dataProcessed).toEqual(fileShaHash) // validated - expect(ftpHashStreamFile2.dataProcessed).toEqual(fileShaHash) // validated // Persist to dynamics for file 1 expect(persist).toHaveBeenNthCalledWith(1, [ @@ -197,10 +177,7 @@ describe('deliverFulfilmentFiles', () => { const s3 = createTestableStream() streamHelper.pipelinePromise.mockResolvedValue() openpgp.encrypt.mockResolvedValue(s2) - merge2 - .mockReturnValueOnce(s1) - .mockReturnValueOnce(s2) - .mockReturnValueOnce(s3) + merge2.mockReturnValueOnce(s1).mockReturnValueOnce(s2).mockReturnValueOnce(s3) await mockExecuteQuery() createMockFileStreams() @@ -227,27 +204,18 @@ const createMockFulfilmentRequestFile = async (fileName, date) => const createMockFileStreams = () => { const s3DataStreamFile = createTestableStream() - const ftpDataStreamFile = createTestableStream() createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3DataStreamFile, managedUpload: Promise.resolve() }) - createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpDataStreamFile, managedUpload: Promise.resolve() }) const s3EncryptedDataStreamFile = createTestableStream() - const ftpEncryptedDataStreamFile = createTestableStream() createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3EncryptedDataStreamFile, managedUpload: Promise.resolve() }) - createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpEncryptedDataStreamFile, managedUpload: Promise.resolve() }) const s3HashStreamFile = createTestableStream() - const ftpHashStreamFile = createTestableStream() createS3WriteStream.mockReturnValueOnce({ s3WriteStream: s3HashStreamFile, managedUpload: Promise.resolve() }) - createFtpWriteStream.mockReturnValueOnce({ ftpWriteStream: ftpHashStreamFile, managedUpload: Promise.resolve() }) return { s3DataStreamFile, - ftpDataStreamFile, s3EncryptedDataStreamFile, - ftpEncryptedDataStreamFile, - s3HashStreamFile, - ftpHashStreamFile + s3HashStreamFile } } diff --git a/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js b/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js index 09c24ceb80..25036678dc 100644 --- a/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js +++ b/packages/fulfilment-job/src/staging/deliver-fulfilment-files.js @@ -4,7 +4,6 @@ import merge2 from 'merge2' import moment from 'moment' import { executeQuery, persist, findFulfilmentFiles } from '@defra-fish/dynamics-lib' import { createS3WriteStream, readS3PartFiles } from '../transport/s3.js' -import { createFtpWriteStream } from '../transport/ftp.js' import { FULFILMENT_FILE_STATUS_OPTIONSET, getOptionSetEntry } from './staging-common.js' import db from 'debug' import openpgp from 'openpgp' @@ -69,11 +68,6 @@ const createEncryptedDataReadStream = async file => { */ const deliver = async (targetFileName, readableStream, ...transforms) => { const { s3WriteStream: s3DataStream, managedUpload: s3DataManagedUpload } = createS3WriteStream(targetFileName) - const { ftpWriteStream: ftpDataStream, managedUpload: ftpDataManagedUpload } = createFtpWriteStream(targetFileName) - await Promise.all([ - streamHelper.pipelinePromise([readableStream, ...transforms, s3DataStream, ftpDataStream]), - s3DataManagedUpload, - ftpDataManagedUpload - ]) + await Promise.all([streamHelper.pipelinePromise([readableStream, ...transforms, s3DataStream]), s3DataManagedUpload]) } diff --git a/packages/fulfilment-job/src/transport/__tests__/ftp.spec.js b/packages/fulfilment-job/src/transport/__tests__/ftp.spec.js deleted file mode 100644 index bf6e868ac4..0000000000 --- a/packages/fulfilment-job/src/transport/__tests__/ftp.spec.js +++ /dev/null @@ -1,63 +0,0 @@ -import { createFtpWriteStream } from '../ftp.js' -import { mockedFtpMethods } from 'ssh2-sftp-client' - -jest.mock('stream') -jest.mock('../../config.js', () => ({ - ftp: { - host: 'testhost', - port: 2222, - path: 'testpath/', - username: 'testusername', - privateKey: 'testprivatekey' - } -})) - -describe('ftp', () => { - beforeEach(() => { - jest.clearAllMocks() - }) - - describe('createFtpWriteStream', () => { - it('creates a stream to write to the configured FTP server', async () => { - const { ftpWriteStream, managedUpload } = createFtpWriteStream('testfile.json') - ftpWriteStream.write('Some data') - ftpWriteStream.end() - await managedUpload - expect(mockedFtpMethods.connect).toHaveBeenCalledWith( - expect.objectContaining({ - host: 'testhost', - port: 2222, - username: 'testusername', - privateKey: 'testprivatekey' - }) - ) - expect(mockedFtpMethods.put).toHaveBeenCalledWith(ftpWriteStream, 'testpath/testfile.json', { - flags: 'w', - encoding: 'UTF-8', - autoClose: false - }) - expect(mockedFtpMethods.end).toHaveBeenCalled() - }) - - it('rejects the managed upload promise if an FTP upload error occurs', async () => { - const testError = new Error('Test error') - mockedFtpMethods.put.mockImplementationOnce(() => Promise.reject(testError)) - const { ftpWriteStream, managedUpload } = createFtpWriteStream('testfile.json') - await expect(managedUpload).rejects.toThrow('Test error') - expect(mockedFtpMethods.connect).toHaveBeenCalledWith( - expect.objectContaining({ - host: 'testhost', - port: 2222, - username: 'testusername', - privateKey: 'testprivatekey' - }) - ) - expect(mockedFtpMethods.put).toHaveBeenCalledWith(ftpWriteStream, 'testpath/testfile.json', { - flags: 'w', - encoding: 'UTF-8', - autoClose: false - }) - expect(mockedFtpMethods.end).toHaveBeenCalled() - }) - }) -}) diff --git a/packages/fulfilment-job/src/transport/ftp.js b/packages/fulfilment-job/src/transport/ftp.js deleted file mode 100644 index 0ce3f41c9c..0000000000 --- a/packages/fulfilment-job/src/transport/ftp.js +++ /dev/null @@ -1,28 +0,0 @@ -import FtpClient from 'ssh2-sftp-client' -import Path from 'path' -import { PassThrough } from 'stream' -import config from '../config.js' -import db from 'debug' -const debug = db('fulfilment:transport') - -/** - * Create a stream to write to the configured FTP server - * - * @param {string} filename The name of the file to be written to the remote server - * @returns {{ftpWriteStream: module:stream.internal.PassThrough, managedUpload: Promise<*>}} - */ -export const createFtpWriteStream = filename => { - const sftp = new FtpClient() - const passThrough = new PassThrough() - const remoteFilePath = Path.join(config.ftp.path, filename) - return { - ftpWriteStream: passThrough, - managedUpload: sftp - .connect(config.ftp) - .then(() => sftp.put(passThrough, remoteFilePath, { flags: 'w', encoding: 'UTF-8', autoClose: false })) - .then(() => - debug('File successfully uploaded to fulfilment provider at sftp://%s:%s%s', config.ftp.host, config.ftp.port, remoteFilePath) - ) - .finally(() => sftp.end()) - } -} diff --git a/packages/pocl-job/package-lock.json b/packages/pocl-job/package-lock.json index 0b2a7780ba..ab1fc908c7 100644 --- a/packages/pocl-job/package-lock.json +++ b/packages/pocl-job/package-lock.json @@ -17,8 +17,7 @@ "md5-file": "^5.0.0", "moment": "^2.29.1", "moment-timezone": "^0.5.34", - "sax-stream": "^1.3.0", - "ssh2-sftp-client": "^6.0.1" + "sax-stream": "^1.3.0" }, "engines": { "node": ">=18.17" @@ -51,20 +50,10 @@ "node": ">=10" } }, - "node_modules/@dabh/diagnostics": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz", - "integrity": "sha512-+A1YivoVDNNVCdfozHSR8v/jyuuLTMXwjWuxPFlFlUapXoGc+Gj9mDlTDDfrwl7rXCl2tNZ0kE8sIBO6YOn96Q==", - "dependencies": { - "colorspace": "1.1.x", - "enabled": "2.0.x", - "kuler": "^2.0.0" - } - }, "node_modules/@defra-fish/business-rules-lib": { - "version": "1.50.0-rc.7", - "resolved": "https://registry.npmjs.org/@defra-fish/business-rules-lib/-/business-rules-lib-1.50.0-rc.7.tgz", - "integrity": "sha512-Zc0wkDOx5H4rADDl1TYtVK3HIFl8uB5udcL0VmlXrCGFG3SOZEgeGElR4I0Y4rEz3Y4F/eP/uYSzscf1HGIsCg==", + "version": "1.50.0-rc.8", + "resolved": "https://registry.npmjs.org/@defra-fish/business-rules-lib/-/business-rules-lib-1.50.0-rc.8.tgz", + "integrity": "sha512-Xo+lI+8DON5vahQKQe8ZX1fLpmE2TQm7eThYaFeUxBAEwbhhP7u7l3Ck49YR9hzWsAUWMLWGKgUrWAhCdId/3g==", "dependencies": { "joi": "^17.6.0", "moment": "^2.29.1", @@ -75,9 +64,9 @@ } }, "node_modules/@defra-fish/connectors-lib": { - "version": "1.50.0-rc.7", - "resolved": "https://registry.npmjs.org/@defra-fish/connectors-lib/-/connectors-lib-1.50.0-rc.7.tgz", - "integrity": "sha512-DV8J4CnxRuYZTfbX8JCkGhm1gQ4hgScIxdRuoy/X5THMIPaLgDSxBkOH90UT/2D61U8A5QGVfgKjPuKSisfGrA==", + "version": "1.50.0-rc.8", + "resolved": "https://registry.npmjs.org/@defra-fish/connectors-lib/-/connectors-lib-1.50.0-rc.8.tgz", + "integrity": "sha512-aXPMRNfXBF6xytV+TD7C3fbeEmMb0m2VPsaDiq4uN4ShqqTg2q3OuAo6YUZypKOkX2tH6/Hj4Qpb3UMerNJ6hg==", "dependencies": { "@airbrake/node": "^2.1.7", "aws-sdk": "^2.1074.0", @@ -155,19 +144,6 @@ "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" }, - "node_modules/asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "dependencies": { - "safer-buffer": "~2.1.0" - } - }, - "node_modules/async": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz", - "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==" - }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -240,14 +216,6 @@ } ] }, - "node_modules/bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "dependencies": { - "tweetnacl": "^0.14.3" - } - }, "node_modules/bintrees": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", @@ -268,11 +236,6 @@ "isarray": "^1.0.0" } }, - "node_modules/buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" - }, "node_modules/call-bind": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz", @@ -299,54 +262,6 @@ "node": ">=0.10.0" } }, - "node_modules/color": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/color/-/color-3.0.0.tgz", - "integrity": "sha512-jCpd5+s0s0t7p3pHQKpnJ0TpQKKdleP71LWcA0aqiljpiuAkOSUFN/dyH8ZwF0hRmFlrIuRhufds1QyEP9EB+w==", - "dependencies": { - "color-convert": "^1.9.1", - "color-string": "^1.5.2" - } - }, - "node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" - }, - "node_modules/color-string": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.4.tgz", - "integrity": "sha512-57yF5yt8Xa3czSEW1jfQDE79Idk0+AkN/4KWad6tbdxUmAs3MvjxlWSWD4deYytcRfoZ9nhKyFl1kj5tBvidbw==", - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, - "node_modules/colors": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", - "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/colorspace": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.2.tgz", - "integrity": "sha512-vt+OoIP2d76xLhjwbBaucYlNSpPsrJWPlBTtwCpQKIu6/CSMutyzX93O/Do0qzpH3YoHEes8YEFXyZ797rEhzQ==", - "dependencies": { - "color": "3.0.x", - "text-hex": "1.0.x" - } - }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -366,25 +281,6 @@ "node": ">= 10" } }, - "node_modules/concat-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", - "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", - "engines": [ - "node >= 6.0" - ], - "dependencies": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.0.2", - "typedarray": "^0.0.6" - } - }, - "node_modules/core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, "node_modules/cross-fetch": { "version": "3.1.8", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", @@ -441,16 +337,6 @@ "node": ">=0.10" } }, - "node_modules/enabled": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", - "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" - }, - "node_modules/err-code": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", - "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==" - }, "node_modules/error-stack-parser": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.1.4.tgz", @@ -486,16 +372,6 @@ "node": ">=0.4.x" } }, - "node_modules/fast-safe-stringify": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", - "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" - }, - "node_modules/fecha": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.0.tgz", - "integrity": "sha512-aN3pcx/DSmtyoovUudctc8+6Hl4T+hI9GBBHLjA76jdZl7+b1sgh5g4k+u/GL3dTy1/pnYzKp69FpJ0OicE3Wg==" - }, "node_modules/filesize": { "version": "6.4.0", "resolved": "https://registry.npmjs.org/filesize/-/filesize-6.4.0.tgz", @@ -504,11 +380,6 @@ "node": ">= 0.4.0" } }, - "node_modules/fn.name": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", - "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" - }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -675,11 +546,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -705,14 +571,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", - "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==", - "engines": { - "node": ">=8" - } - }, "node_modules/is-typed-array": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", @@ -752,11 +610,6 @@ "@sideway/pinpoint": "^2.0.0" } }, - "node_modules/kuler": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", - "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" - }, "node_modules/lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", @@ -772,18 +625,6 @@ "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==" }, - "node_modules/logform": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.2.0.tgz", - "integrity": "sha512-N0qPlqfypFx7UHNn4B3lzS/b0uLqt2hmuoa+PpuXNYgozdJYAyauF5Ky0BWVjrxDlMWiT3qN4zPq3vVAfZy7Yg==", - "dependencies": { - "colors": "^1.2.1", - "fast-safe-stringify": "^2.0.4", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "triple-beam": "^1.3.0" - } - }, "node_modules/md5-file": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/md5-file/-/md5-file-5.0.0.tgz", @@ -857,14 +698,6 @@ } } }, - "node_modules/one-time": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", - "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", - "dependencies": { - "fn.name": "1.x.x" - } - }, "node_modules/p-map": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-2.1.0.tgz", @@ -881,28 +714,11 @@ "node": ">= 0.4" } }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" - }, "node_modules/promise-polyfill": { "version": "8.3.0", "resolved": "https://registry.npmjs.org/promise-polyfill/-/promise-polyfill-8.3.0.tgz", "integrity": "sha512-H5oELycFml5yto/atYqmjyigJoAo3+OXwolYiH7OfQuYlAqhxNvTfiNMbV9hsC6Yp83yE5r2KTVmtrG6R9i6Pg==" }, - "node_modules/promise-retry": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", - "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", - "dependencies": { - "err-code": "^2.0.2", - "retry": "^0.12.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", @@ -917,19 +733,6 @@ "node": ">=0.4.x" } }, - "node_modules/readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/redis-commands": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/redis-commands/-/redis-commands-1.7.0.tgz", @@ -965,38 +768,6 @@ "node": ">=8.0.0" } }, - "node_modules/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", - "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=", - "engines": { - "node": ">= 4" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, "node_modules/sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", @@ -1040,57 +811,6 @@ "node": ">= 0.4" } }, - "node_modules/simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha1-pNprY1/8zMoz9w0Xy5JZLeleVXo=", - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, - "node_modules/ssh2": { - "version": "0.8.9", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-0.8.9.tgz", - "integrity": "sha512-GmoNPxWDMkVpMFa9LVVzQZHF6EW3WKmBwL+4/GeILf2hFmix5Isxm7Amamo8o7bHiU0tC+wXsGcUXOxp8ChPaw==", - "dependencies": { - "ssh2-streams": "~0.4.10" - }, - "engines": { - "node": ">=5.2.0" - } - }, - "node_modules/ssh2-sftp-client": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ssh2-sftp-client/-/ssh2-sftp-client-6.0.1.tgz", - "integrity": "sha512-Glut2SmK/XpNOBiEuzqlKZGKkIyha2XMbuWVXR2hFUJkNsbyl/wmlZSeUEPxKFp/dC9UEvUKzanKydgLmNdfkw==", - "dependencies": { - "concat-stream": "^2.0.0", - "promise-retry": "^2.0.1", - "ssh2": "^0.8.9", - "winston": "^3.3.3" - } - }, - "node_modules/ssh2-streams": { - "version": "0.4.10", - "resolved": "https://registry.npmjs.org/ssh2-streams/-/ssh2-streams-0.4.10.tgz", - "integrity": "sha512-8pnlMjvnIZJvmTzUIIA5nT4jr2ZWNNVHwyXfMGdRJbug9TpI3kd99ffglgfSWqujVv/0gxwMsDn9j9RVst8yhQ==", - "dependencies": { - "asn1": "~0.2.0", - "bcrypt-pbkdf": "^1.0.2", - "streamsearch": "~0.1.2" - }, - "engines": { - "node": ">=5.2.0" - } - }, - "node_modules/stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=", - "engines": { - "node": "*" - } - }, "node_modules/stackframe": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.3.4.tgz", @@ -1101,22 +821,6 @@ "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==" }, - "node_modules/streamsearch": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", - "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dependencies": { - "safe-buffer": "~5.2.0" - } - }, "node_modules/tdigest": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", @@ -1125,31 +829,11 @@ "bintrees": "1.0.2" } }, - "node_modules/text-hex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", - "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" - }, "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, - "node_modules/triple-beam": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", - "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==" - }, - "node_modules/tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" - }, - "node_modules/typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" - }, "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", @@ -1176,11 +860,6 @@ "which-typed-array": "^1.1.2" } }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, "node_modules/uuid": { "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", @@ -1221,64 +900,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/winston": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/winston/-/winston-3.3.3.tgz", - "integrity": "sha512-oEXTISQnC8VlSAKf1KYSSd7J6IWuRPQqDdo8eoRNaYKLvwSb5+79Z3Yi1lrl6KDpU6/VWaxpakDAtb1oQ4n9aw==", - "dependencies": { - "@dabh/diagnostics": "^2.0.2", - "async": "^3.1.0", - "is-stream": "^2.0.0", - "logform": "^2.2.0", - "one-time": "^1.0.0", - "readable-stream": "^3.4.0", - "stack-trace": "0.0.x", - "triple-beam": "^1.3.0", - "winston-transport": "^4.4.0" - }, - "engines": { - "node": ">= 6.4.0" - } - }, - "node_modules/winston-transport": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.4.0.tgz", - "integrity": "sha512-Lc7/p3GtqtqPBYYtS6KCN3c77/2QCev51DvcJKbkFPQNoj1sinkGwLGFDxkXY9J6p9+EPnYs+D90uwbnaiURTw==", - "dependencies": { - "readable-stream": "^2.3.7", - "triple-beam": "^1.2.0" - }, - "engines": { - "node": ">= 6.4.0" - } - }, - "node_modules/winston-transport/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/winston-transport/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/winston-transport/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, "node_modules/xml2js": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", @@ -1300,4 +921,4 @@ } } } -} \ No newline at end of file +} diff --git a/packages/pocl-job/package.json b/packages/pocl-job/package.json index 85668d4698..596b6abe95 100644 --- a/packages/pocl-job/package.json +++ b/packages/pocl-job/package.json @@ -43,7 +43,6 @@ "md5-file": "^5.0.0", "moment": "^2.29.1", "moment-timezone": "^0.5.34", - "sax-stream": "^1.3.0", - "ssh2-sftp-client": "^6.0.1" + "sax-stream": "^1.3.0" } } diff --git a/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js b/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js deleted file mode 100644 index f24599f0b1..0000000000 --- a/packages/pocl-job/src/__mocks__/ssh2-sftp-client.js +++ /dev/null @@ -1,11 +0,0 @@ -const ssh2sftpClient = jest.genMockFromModule('ssh2-sftp-client') - -export const mockedFtpMethods = { - connect: jest.fn(), - list: jest.fn(), - fastGet: jest.fn(), - delete: jest.fn(), - end: jest.fn() -} -ssh2sftpClient.mockImplementation(() => mockedFtpMethods) -export default ssh2sftpClient diff --git a/packages/pocl-job/src/__tests__/config.spec.js b/packages/pocl-job/src/__tests__/config.spec.js index f0c1c6d8eb..5fd427d0f6 100644 --- a/packages/pocl-job/src/__tests__/config.spec.js +++ b/packages/pocl-job/src/__tests__/config.spec.js @@ -11,11 +11,6 @@ describe('config', () => { process.env.POCL_RECORD_STAGING_TABLE = 'test-record-staging-table' process.env.POCL_STAGING_TTL = 1234 - process.env.POCL_FTP_HOST = 'test-host' - process.env.POCL_FTP_PORT = 2222 - process.env.POCL_FTP_PATH = '/remote/share' - process.env.POCL_FTP_USERNAME = 'test-user' - process.env.POCL_FTP_KEY_SECRET_ID = 'test-secret-id' process.env.POCL_S3_BUCKET = 'test-bucket' await config.initialise() }) @@ -38,32 +33,6 @@ describe('config', () => { }) }) - describe('ftp', () => { - it('provides properties relating the use of SFTP', async () => { - expect(config.ftp).toEqual( - expect.objectContaining({ - host: 'test-host', - port: '2222', - path: '/remote/share', - username: 'test-user', - privateKey: 'test-ssh-key', - algorithms: { cipher: expect.any(Array), kex: expect.any(Array) }, - // Wait up to 60 seconds for the SSH handshake - readyTimeout: expect.any(Number), - // Retry 5 times over a minute - retries: expect.any(Number), - retry_minTimeout: expect.any(Number), - debug: expect.any(Function) - }) - ) - }) - it('defaults the sftp port to 22 if the environment variable is not configured', async () => { - delete process.env.POCL_FTP_PORT - await config.initialise() - expect(config.ftp.port).toEqual('22') - }) - }) - describe('s3', () => { it('provides properties relating the use of Amazon S3', async () => { expect(config.s3.bucket).toEqual('test-bucket') diff --git a/packages/pocl-job/src/__tests__/pocl-processor.spec.js b/packages/pocl-job/src/__tests__/pocl-processor.spec.js index f3a5e4696a..ade97101d7 100644 --- a/packages/pocl-job/src/__tests__/pocl-processor.spec.js +++ b/packages/pocl-job/src/__tests__/pocl-processor.spec.js @@ -41,7 +41,7 @@ jest.mock('../config.js', () => ({ bucket: 'testbucket' } })) -jest.mock('../transport/ftp-to-s3.js') +jest.mock('../transport/storeS3MetaData.js') jest.mock('../transport/s3-to-local.js') jest.mock('../io/db.js') jest.mock('../io/s3.js') diff --git a/packages/pocl-job/src/config.js b/packages/pocl-job/src/config.js index 0430b4a457..d08132335b 100644 --- a/packages/pocl-job/src/config.js +++ b/packages/pocl-job/src/config.js @@ -1,49 +1,5 @@ -import { AWS } from '@defra-fish/connectors-lib' -import db from 'debug' -const { secretsManager } = AWS() - -/** - * Key exchange algorithms for public key authentication - in descending order of priority - * @type {string[]} - */ -export const SFTP_KEY_EXCHANGE_ALGORITHMS = [ - 'curve25519-sha256@libssh.org', - 'curve25519-sha256', - 'ecdh-sha2-nistp521', - 'ecdh-sha2-nistp384', - 'ecdh-sha2-nistp256', - 'diffie-hellman-group-exchange-sha256', - 'diffie-hellman-group14-sha256', - 'diffie-hellman-group16-sha512', - 'diffie-hellman-group18-sha512', - 'diffie-hellman-group14-sha1', - 'diffie-hellman-group-exchange-sha1', - 'diffie-hellman-group1-sha1' -] -/** - * Ciphers for SFTP support - in descending order of priority - * @type {string[]} - */ -export const SFTP_CIPHERS = [ - // http://tools.ietf.org/html/rfc4344#section-4 - 'aes256-ctr', - 'aes192-ctr', - 'aes128-ctr', - 'aes256-gcm', - 'aes256-gcm@openssh.com', - 'aes128-gcm', - 'aes128-gcm@openssh.com', - 'aes256-cbc', - 'aes192-cbc', - 'aes128-cbc', - 'blowfish-cbc', - '3des-cbc', - 'cast128-cbc' -] - class Config { _db - _ftp _s3 async initialise () { @@ -53,21 +9,6 @@ class Config { stagingTtlDelta: Number.parseInt(process.env.POCL_STAGING_TTL || 60 * 60 * 168) } - this.ftp = { - host: process.env.POCL_FTP_HOST, - port: process.env.POCL_FTP_PORT || '22', - path: process.env.POCL_FTP_PATH, - username: process.env.POCL_FTP_USERNAME, - privateKey: (await secretsManager.getSecretValue({ SecretId: process.env.POCL_FTP_KEY_SECRET_ID }).promise()).SecretString, - algorithms: { cipher: SFTP_CIPHERS, kex: SFTP_KEY_EXCHANGE_ALGORITHMS }, - // Wait up to 60 seconds for the SSH handshake - readyTimeout: 60000, - // Retry 5 times over a minute - retries: 5, - retry_minTimeout: 12000, - debug: db('pocl:ftp') - } - this.s3 = { bucket: process.env.POCL_S3_BUCKET } @@ -85,18 +26,6 @@ class Config { this._db = cfg } - /** - * FTP configuration settings - * @type {object} - */ - get ftp () { - return this._ftp - } - - set ftp (cfg) { - this._ftp = cfg - } - /** * S3 configuration settings * @type {object} diff --git a/packages/pocl-job/src/io/__tests__/s3.spec.js b/packages/pocl-job/src/io/__tests__/s3.spec.js index 4a1a2da226..26c7fe9be1 100644 --- a/packages/pocl-job/src/io/__tests__/s3.spec.js +++ b/packages/pocl-job/src/io/__tests__/s3.spec.js @@ -5,7 +5,6 @@ import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../.. import { salesApi } from '@defra-fish/connectors-lib' import fs from 'fs' import AwsMock from 'aws-sdk' -import { mockedFtpMethods } from 'ssh2-sftp-client' jest.mock('fs') jest.mock('md5-file') @@ -159,7 +158,6 @@ describe('s3 operations', () => { }) it('skips file processing if a file has already been marked as processed in Dynamics', async () => { - mockedFtpMethods.list.mockResolvedValue([{ name: 'test-already-processed.xml' }]) fs.createReadStream.mockReturnValueOnce('teststream') fs.statSync.mockReturnValueOnce({ size: 1024 }) salesApi.getTransactionFile.mockResolvedValueOnce({ status: { description: 'Processed' } }) diff --git a/packages/pocl-job/src/io/s3.js b/packages/pocl-job/src/io/s3.js index c24d3ba3d0..66a97cdbfb 100644 --- a/packages/pocl-job/src/io/s3.js +++ b/packages/pocl-job/src/io/s3.js @@ -2,7 +2,7 @@ import moment from 'moment' import filesize from 'filesize' import config from '../config.js' import { DYNAMICS_IMPORT_STAGE } from '../staging/constants.js' -import { storeS3Metadata } from '../transport/ftp-to-s3.js' +import { storeS3Metadata } from '../transport/storeS3MetaData.js' import { AWS, salesApi } from '@defra-fish/connectors-lib' const { s3 } = AWS() diff --git a/packages/pocl-job/src/transport/__tests__/ftp-to-s3.spec.js b/packages/pocl-job/src/transport/__tests__/ftp-to-s3.spec.js deleted file mode 100644 index 7b5259da27..0000000000 --- a/packages/pocl-job/src/transport/__tests__/ftp-to-s3.spec.js +++ /dev/null @@ -1,145 +0,0 @@ -import { ftpToS3 } from '../ftp-to-s3.js' -import moment from 'moment' -import { updateFileStagingTable } from '../../io/db.js' -import { getTempDir } from '../../io/file.js' -import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../../staging/constants.js' -import { salesApi } from '@defra-fish/connectors-lib' -import fs from 'fs' -import md5File from 'md5-file' -import AwsMock from 'aws-sdk' -import { mockedFtpMethods } from 'ssh2-sftp-client' - -jest.mock('fs') -jest.mock('md5-file') -jest.mock('../../io/db.js') -jest.mock('../../io/file.js') - -jest.mock('@defra-fish/connectors-lib', () => { - const actual = jest.requireActual('@defra-fish/connectors-lib') - return { - AWS: actual.AWS, - salesApi: { - ...Object.keys(actual.salesApi).reduce((acc, k) => ({ ...acc, [k]: jest.fn(async () => {}) }), {}) - } - } -}) - -jest.mock('../../config.js', () => ({ - ftp: { - path: '/ftpservershare/' - }, - s3: { - bucket: 'testbucket' - } -})) - -describe('ftp-to-s3', () => { - beforeAll(() => { - getTempDir.mockReturnValue('/local/tmp') - md5File.mockResolvedValue('example-md5') - }) - beforeEach(() => { - jest.clearAllMocks() - AwsMock.__resetAll() - }) - - it('retrieves files from SFTP and stores in S3', async () => { - mockedFtpMethods.list.mockResolvedValue([{ name: 'test1.xml' }, { name: 'test2.xml' }]) - fs.createReadStream.mockReturnValueOnce('test1stream') - fs.createReadStream.mockReturnValueOnce('test2stream') - fs.statSync.mockReturnValueOnce({ size: 1024 }) - fs.statSync.mockReturnValueOnce({ size: 2048 }) - await ftpToS3() - - const localPath1 = '/local/tmp/test1.xml' - const localPath2 = '/local/tmp/test2.xml' - - const s3Key1 = `${moment().format('YYYY-MM-DD')}/test1.xml` - const s3Key2 = `${moment().format('YYYY-MM-DD')}/test2.xml` - - expect(mockedFtpMethods.fastGet).toHaveBeenNthCalledWith(1, '/ftpservershare/test1.xml', localPath1, {}) - expect(mockedFtpMethods.fastGet).toHaveBeenNthCalledWith(2, '/ftpservershare/test2.xml', localPath2, {}) - expect(AwsMock.S3.mockedMethods.putObject).toHaveBeenNthCalledWith(1, { - Bucket: 'testbucket', - Key: s3Key1, - Body: 'test1stream' - }) - expect(AwsMock.S3.mockedMethods.putObject).toHaveBeenNthCalledWith(2, { - Bucket: 'testbucket', - Key: s3Key2, - Body: 'test2stream' - }) - expect(updateFileStagingTable).toHaveBeenNthCalledWith(1, { - filename: 'test1.xml', - md5: 'example-md5', - fileSize: '1 KB', - stage: FILE_STAGE.Pending, - s3Key: s3Key1 - }) - expect(updateFileStagingTable).toHaveBeenNthCalledWith(2, { - filename: 'test2.xml', - md5: 'example-md5', - fileSize: '2 KB', - stage: FILE_STAGE.Pending, - s3Key: s3Key2 - }) - expect(salesApi.upsertTransactionFile).toHaveBeenNthCalledWith(1, 'test1.xml', { - status: DYNAMICS_IMPORT_STAGE.Pending, - dataSource: POST_OFFICE_DATASOURCE, - fileSize: '1 KB', - receiptTimestamp: expect.any(String), - salesDate: expect.any(String), - notes: 'Retrieved from the remote server and awaiting processing' - }) - expect(salesApi.upsertTransactionFile).toHaveBeenNthCalledWith(2, 'test2.xml', { - status: DYNAMICS_IMPORT_STAGE.Pending, - dataSource: POST_OFFICE_DATASOURCE, - fileSize: '2 KB', - receiptTimestamp: expect.any(String), - salesDate: expect.any(String), - notes: 'Retrieved from the remote server and awaiting processing' - }) - expect(fs.unlinkSync).toHaveBeenNthCalledWith(1, localPath1) - expect(fs.unlinkSync).toHaveBeenNthCalledWith(2, localPath2) - expect(mockedFtpMethods.end).toHaveBeenCalledTimes(1) - }) - - it('moves the file to s3 but skips file processing if a file has already been marked as processed in Dynamics', async () => { - mockedFtpMethods.list.mockResolvedValue([{ name: 'test-already-processed.xml' }]) - fs.createReadStream.mockReturnValueOnce('teststream') - fs.statSync.mockReturnValueOnce({ size: 1024 }) - salesApi.getTransactionFile.mockResolvedValueOnce({ status: { description: 'Processed' } }) - const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(jest.fn()) - await ftpToS3() - const localPath = '/local/tmp/test-already-processed.xml' - const s3Key = `${moment().format('YYYY-MM-DD')}/test-already-processed.xml` - expect(mockedFtpMethods.fastGet).toHaveBeenCalledWith('/ftpservershare/test-already-processed.xml', localPath, {}) - expect(AwsMock.S3.mockedMethods.putObject).toHaveBeenCalledWith({ - Bucket: 'testbucket', - Key: s3Key, - Body: 'teststream' - }) - expect(updateFileStagingTable).not.toHaveBeenCalled() - expect(salesApi.upsertTransactionFile).not.toHaveBeenCalled() - expect(consoleErrorSpy).toHaveBeenCalled() - expect(fs.unlinkSync).toHaveBeenCalledWith(localPath) - expect(mockedFtpMethods.end).toHaveBeenCalledTimes(1) - }) - - it('logs and propogates errors back up the stack', async () => { - const testError = new Error('Test error') - mockedFtpMethods.list.mockRejectedValue(testError) - const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {}) - await expect(ftpToS3).rejects.toThrow(testError) - expect(consoleErrorSpy).toHaveBeenCalled() - }) - - it('ignores non-xml files', async () => { - mockedFtpMethods.list.mockResolvedValue([{ name: 'test1.pdf' }, { name: 'test2.md' }]) - await ftpToS3() - expect(mockedFtpMethods.fastGet).not.toHaveBeenCalled() - expect(AwsMock.S3.mockedMethods.putObject).not.toHaveBeenCalled() - expect(fs.unlinkSync).not.toHaveBeenCalled() - expect(mockedFtpMethods.end).toHaveBeenCalledTimes(1) - }) -}) diff --git a/packages/pocl-job/src/transport/__tests__/storeS3MetaData.spec.js b/packages/pocl-job/src/transport/__tests__/storeS3MetaData.spec.js new file mode 100644 index 0000000000..57d4434d65 --- /dev/null +++ b/packages/pocl-job/src/transport/__tests__/storeS3MetaData.spec.js @@ -0,0 +1,64 @@ +import moment from 'moment' +import { salesApi } from '@defra-fish/connectors-lib' +import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../../staging/constants.js' +import { updateFileStagingTable } from '../../io/db.js' +import { storeS3Metadata } from '../storeS3MetaData.js' + +jest.mock('../../io/db.js', () => ({ + updateFileStagingTable: jest.fn() +})) +jest.mock('@defra-fish/connectors-lib', () => ({ + salesApi: { + upsertTransactionFile: jest.fn() + } +})) + +describe('storeS3Metadata', () => { + const md5 = 'mockMd5Hash' + const fileSize = 12345 + const filename = 'testfile' + const s3Key = 'mock/s3/key/testfile' + const receiptMoment = new Date('2024-10-17T00:00:00Z') + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('console log should output "Storing metadata for s3Key"', async () => { + const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(jest.fn()) + await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) + expect(consoleLogSpy).toHaveBeenCalledWith(`Storing metadata for ${s3Key}`) + }) + + it('should call updateFileStagingTable with correct arguments', async () => { + await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) + expect(updateFileStagingTable).toHaveBeenCalledWith({ + filename, + md5, + fileSize, + s3Key, + stage: FILE_STAGE.Pending + }) + }) + + it('should call salesApi.upsertTransactionFile with correct arguments', async () => { + const expectedSalesDate = moment(receiptMoment).subtract(1, 'days').toISOString() + const expectedReceiptTimestamp = receiptMoment.toISOString() + + await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) + expect(salesApi.upsertTransactionFile).toHaveBeenCalledWith(filename, { + status: DYNAMICS_IMPORT_STAGE.Pending, + dataSource: POST_OFFICE_DATASOURCE, + fileSize: fileSize, + salesDate: expectedSalesDate, + receiptTimestamp: expectedReceiptTimestamp, + notes: 'Retrieved from the remote server and awaiting processing' + }) + }) + + test('should log "Stored metadata for s3Key"', async () => { + const consoleLogSpy = jest.spyOn(console, 'log').mockImplementation(jest.fn()) + await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) + expect(consoleLogSpy).toHaveBeenCalledWith(`Stored metadata for ${s3Key}`) + }) +}) diff --git a/packages/pocl-job/src/transport/ftp-to-s3.js b/packages/pocl-job/src/transport/ftp-to-s3.js deleted file mode 100644 index b3c9376448..0000000000 --- a/packages/pocl-job/src/transport/ftp-to-s3.js +++ /dev/null @@ -1,91 +0,0 @@ -import FtpClient from 'ssh2-sftp-client' -import moment from 'moment' -import Path from 'path' -import fs from 'fs' -import db from 'debug' -import md5File from 'md5-file' -import filesize from 'filesize' -import config from '../config.js' -import { getTempDir } from '../io/file.js' -import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../staging/constants.js' -import { AWS, salesApi } from '@defra-fish/connectors-lib' -import { updateFileStagingTable } from '../io/db.js' -const { s3 } = AWS() - -const debug = db('pocl:transport') -const sftp = new FtpClient() - -export async function ftpToS3 () { - try { - debug('Connecting to SFTP endpoint at sftp://%s:%s%s', config.ftp.host, config.ftp.port, config.ftp.path) - await sftp.connect(config.ftp) - const fileList = await sftp.list(config.ftp.path) - debug('Discovered the following files on the SFTP server: %o', fileList) - const xmlFiles = fileList.filter(f => Path.extname(f.name).toLowerCase() === '.xml') - - if (!xmlFiles.length) { - debug('No XML files were waiting to be processed on the SFTP server.') - } else { - await retrieveAllFiles(xmlFiles) - } - } catch (e) { - console.error('Error migrating files from the SFTP endpoint', e) - throw e - } finally { - debug('Closing SFTP connection.') - await sftp.end() - } -} - -export async function storeS3Metadata (md5, fileSize, filename, s3Key, receiptMoment) { - console.log(`Storing metadata for ${s3Key}`) - await updateFileStagingTable({ filename, md5, fileSize, s3Key, stage: FILE_STAGE.Pending }) - - await salesApi.upsertTransactionFile(filename, { - status: DYNAMICS_IMPORT_STAGE.Pending, - dataSource: POST_OFFICE_DATASOURCE, - fileSize: fileSize, - salesDate: moment(receiptMoment).subtract(1, 'days').toISOString(), - receiptTimestamp: receiptMoment.toISOString(), - notes: 'Retrieved from the remote server and awaiting processing' - }) - - console.log(`Stored metadata for ${s3Key}`) -} - -const retrieveAllFiles = async xmlFiles => { - const tempDir = getTempDir('ftp') - - for (const fileEntry of xmlFiles) { - const filename = fileEntry.name - const remoteFilePath = Path.join(config.ftp.path, filename) - const localFilePath = Path.resolve(tempDir, filename) - - // Retrieve from FTP server to local temporary directory - debug('Transferring %s to %s', remoteFilePath, localFilePath) - await sftp.fastGet(remoteFilePath, localFilePath, {}) - - // Transfer to S3 - const receiptMoment = moment() - const s3Key = Path.join(receiptMoment.format('YYYY-MM-DD'), filename) - debug('Transferring file to S3 bucket %s with key %s', config.s3.bucket, s3Key) - await s3.putObject({ Bucket: config.s3.bucket, Key: s3Key, Body: fs.createReadStream(localFilePath) }).promise() - - const dynamicsRecord = await salesApi.getTransactionFile(filename) - if (dynamicsRecord && DYNAMICS_IMPORT_STAGE.isAlreadyProcessed(dynamicsRecord.status.description)) { - console.error( - 'Retrieved file %s from SFTP and stored in S3, however an entry already exists in Dynamics with this filename. Skipping import.', - filename - ) - } else { - const md5 = await md5File(localFilePath) - const fileSize = filesize(fs.statSync(localFilePath).size) - await storeS3Metadata(md5, fileSize, filename, s3Key, receiptMoment) - } - - // Remove from FTP server and local tmp - debug('Removing remote file %s', remoteFilePath) - await sftp.delete(remoteFilePath) - fs.unlinkSync(localFilePath) - } -} diff --git a/packages/pocl-job/src/transport/storeS3MetaData.js b/packages/pocl-job/src/transport/storeS3MetaData.js new file mode 100644 index 0000000000..31eb6a4810 --- /dev/null +++ b/packages/pocl-job/src/transport/storeS3MetaData.js @@ -0,0 +1,20 @@ +import moment from 'moment' +import { salesApi } from '@defra-fish/connectors-lib' +import { DYNAMICS_IMPORT_STAGE, FILE_STAGE, POST_OFFICE_DATASOURCE } from '../staging/constants.js' +import { updateFileStagingTable } from '../io/db.js' + +export async function storeS3Metadata (md5, fileSize, filename, s3Key, receiptMoment) { + console.log(`Storing metadata for ${s3Key}`) + await updateFileStagingTable({ filename, md5, fileSize, s3Key, stage: FILE_STAGE.Pending }) + + await salesApi.upsertTransactionFile(filename, { + status: DYNAMICS_IMPORT_STAGE.Pending, + dataSource: POST_OFFICE_DATASOURCE, + fileSize: fileSize, + salesDate: moment(receiptMoment).subtract(1, 'days').toISOString(), + receiptTimestamp: receiptMoment.toISOString(), + notes: 'Retrieved from the remote server and awaiting processing' + }) + + console.log(`Stored metadata for ${s3Key}`) +}