From 0469101ef29033b25d1896e96ab3ea74f77c8a05 Mon Sep 17 00:00:00 2001 From: Scott Dormand <61588381+ScottDormand96@users.noreply.github.com> Date: Mon, 25 Nov 2024 14:02:06 +0000 Subject: [PATCH] Remove ftp image and build (#2070) * Remove ftp functionality https://eaflood.atlassian.net/browse/IWTF-4278 Remove ftp functionality as package we use in fulfilment and pocl jobs (ssh2-sftp-client) has critical vulnerability * remove ssh2 sftp client * remove ssh2 sftp client and fix lint * remove reference * fix tests * fix tests and lint * update tests * refactor * refactor * update tests * refactor tests * s3spec * undefined token * rename file and undo removal of stores3 * rename file and undo removal of stores3 * add missing mock * Remove ssh2 mock * Remove ftp image and build https://eaflood.atlassian.net/browse/IWTF-4279 remove any FTP images from our Docker config * remove functionality from config * remove ftp reference * remove extra line --- docker/env/fulfilment_job.env.example | 5 -- docker/infrastructure.yml | 27 ------ .../resources/infrastructure/sftp/Dockerfile | 11 --- .../infrastructure/sftp/files/entrypoint.sh | 85 ------------------- .../infrastructure/sftp/files/sshd_config | 24 ------ docker/volumes/sftp/fulfilment/README.md | 1 - docker/volumes/sftp/pocl/README.md | 1 - packages/fulfilment-job/README.md | 25 +++--- 8 files changed, 10 insertions(+), 169 deletions(-) delete mode 100644 docker/resources/infrastructure/sftp/Dockerfile delete mode 100755 docker/resources/infrastructure/sftp/files/entrypoint.sh delete mode 100644 docker/resources/infrastructure/sftp/files/sshd_config delete mode 100644 docker/volumes/sftp/fulfilment/README.md delete mode 100644 docker/volumes/sftp/pocl/README.md diff --git a/docker/env/fulfilment_job.env.example b/docker/env/fulfilment_job.env.example index 36356b4102..a616195603 100644 --- a/docker/env/fulfilment_job.env.example +++ b/docker/env/fulfilment_job.env.example @@ -20,11 +20,6 @@ FULFILMENT_S3_BUCKET=fulfilment-audit FULFILMENT_FILE_SIZE=5000 # FTP Settings -FULFILMENT_FTP_HOST=host.docker.internal -FULFILMENT_FTP_PORT=2222 -FULFILMENT_FTP_PATH=/share/fulfilment -FULFILMENT_FTP_USERNAME=test_sftp_user -FULFILMENT_FTP_KEY_SECRET_ID=/dev/fsh/local/sftp/ssh_ed25519_host_key FULFILMENT_SEND_UNENCRYPTED_FILE=true FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID=/dev/fsh/local/ diff --git a/docker/infrastructure.yml b/docker/infrastructure.yml index 89d387f4fb..acaf6a6ec2 100644 --- a/docker/infrastructure.yml +++ b/docker/infrastructure.yml @@ -107,30 +107,3 @@ services: deploy: restart_policy: condition: on-failure - - ####################################################### - # Test SFTP server - ####################################################### - ftp: - image: rod_licensing/ftp:${TAG:-latest} - build: - dockerfile: Dockerfile - context: ./resources/infrastructure/sftp - depends_on: - - localstack - ports: - - '2222:22' - volumes: - - ./volumes/sftp:/home/test_sftp_user/share - environment: - SFTP_USER: test_sftp_user - SFTP_FOLDERS: share/pocl;share/fulfilment - AWS_SECRETSMANAGER_ENDPOINT: http://host.docker.internal:4566 - AWS_DEFAULT_REGION: eu-west-2 - AWS_ACCESS_KEY_ID: local - AWS_SECRET_ACCESS_KEY: local - SSH_HOST_ED25519_SECRET_ID: /dev/fsh/local/sftp/ssh_ed25519_host_key - SSH_HOST_RSA_SECRET_ID: /dev/fsh/local/sftp/ssh_rsa_host_key - deploy: - restart_policy: - condition: on-failure diff --git a/docker/resources/infrastructure/sftp/Dockerfile b/docker/resources/infrastructure/sftp/Dockerfile deleted file mode 100644 index 2c9f4ff96b..0000000000 --- a/docker/resources/infrastructure/sftp/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM alpine:latest - -RUN apk --no-cache -U -v upgrade \ - && apk --no-cache -U -v add bash openssh openssh-sftp-server aws-cli \ - && mkdir -p /var/run/sshd \ - && rm -f /etc/ssh/ssh_host_*key* - -COPY ./files/sshd_config /etc/ssh/sshd_config -COPY ./files/entrypoint.sh / -EXPOSE 22 -ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/resources/infrastructure/sftp/files/entrypoint.sh b/docker/resources/infrastructure/sftp/files/entrypoint.sh deleted file mode 100755 index 7e853335a0..0000000000 --- a/docker/resources/infrastructure/sftp/files/entrypoint.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash -############################################################################### -# SFTP Initialisation script -############################################################################### -set -e -trap 'exit 1' INT - -ED25519_KEY_PATH="/etc/ssh/ssh_host_ed25519_key" -RSA_KEY_PATH="/etc/ssh/ssh_host_rsa_key" -SFTP_USER=${SFTP_USER:=test} -SFTP_FOLDERS=${SFTP_FOLDERS:=share} - -AWS_CLI_ARGS=() -if [ -n "${AWS_SECRETSMANAGER_ENDPOINT}" ]; then - AWS_CLI_ARGS+=('--endpoint' "${AWS_SECRETSMANAGER_ENDPOINT}") -fi - -######################### -# Create SSH keys -######################### -if [ -n "${SSH_HOST_ED25519_SECRET_ID}" ]; then - echo "Retrieving SSH_HOST_ED25519_KEY from aws secrets manager" - aws "${AWS_CLI_ARGS[@]}" secretsmanager get-secret-value --secret-id "${SSH_HOST_ED25519_SECRET_ID}" --query SecretString --output text | (umask 177; cat > "${ED25519_KEY_PATH}") -elif [ -n "${SSH_HOST_ED25519_KEY}" ]; then - echo "Using SSH_HOST_ED25519_KEY defined in environment" - echo "${SSH_HOST_ED25519_KEY}" | (umask 177; cat > "${ED25519_KEY_PATH}") -else - echo "Generating new SSH_HOST_ED25519_KEY" - ssh-keygen -t ed25519 -f "${ED25519_KEY_PATH}" -N '' - cat "${ED25519_KEY_PATH}" -fi -ssh-keygen -y -f "${ED25519_KEY_PATH}" > "${ED25519_KEY_PATH}.pub" -ssh-keygen -lvf "${ED25519_KEY_PATH}" - -if [ -n "${SSH_HOST_RSA_SECRET_ID}" ]; then - echo "Retrieving SSH_HOST_RSA_SECRET_ID from aws secrets manager" - aws "${AWS_CLI_ARGS[@]}" secretsmanager get-secret-value --secret-id "${SSH_HOST_RSA_SECRET_ID}" --query SecretString --output text | (umask 177; cat > "${RSA_KEY_PATH}") -elif [ -n "${SSH_HOST_RSA_KEY}" ]; then - echo "Using SSH_HOST_RSA_KEY defined in environment" - echo "${SSH_HOST_RSA_KEY}" | (umask 177; cat > "${RSA_KEY_PATH}") -else - echo "Generating new SSH_HOST_RSA_KEY" - ssh-keygen -t rsa -b 4096 -f "${RSA_KEY_PATH}" -N '' - cat "${RSA_KEY_PATH}" -fi -ssh-keygen -y -f "${RSA_KEY_PATH}" > "${RSA_KEY_PATH}.pub" -ssh-keygen -lvf "${RSA_KEY_PATH}" - -######################### -# Create test user -######################### -echo "Creating user ${SFTP_USER} with random password" -adduser "${SFTP_USER}" > /dev/null 2>&1 || true -echo "${SFTP_USER}:$(base64 /dev/urandom | tr -d '/+' | fold -w 32 | head -n1)" | chpasswd -e > /dev/null 2>&1 - -######################### -# Add authorised keys -######################### -echo "Adding authorised keys" -mkdir -p "/home/${SFTP_USER}/.ssh/keys/" -cp "${ED25519_KEY_PATH}.pub" "/home/${SFTP_USER}/.ssh/keys/id_ed25519.pub" -cp "${RSA_KEY_PATH}.pub" "/home/${SFTP_USER}/.ssh/keys/id_rsa.pub" -for publickey in "/home/${SFTP_USER}/.ssh/keys"/*; do - cat "${publickey}" >> "/home/${SFTP_USER}/.ssh/authorized_keys" -done - - -######################### -# Create default folders -######################### -IFS=';' -read -ra FOLDERS <<< "${SFTP_FOLDERS}" -for folder in "${FOLDERS[@]}"; do - echo "Creating folder /home/${SFTP_USER}/${folder}" - mkdir -p "/home/${SFTP_USER}/${folder}" -done - -######################### -# Set permissions -######################### -chown -R "${SFTP_USER}" "/home/${SFTP_USER}/" -chown root:root "/home/${SFTP_USER}" -chmod 755 "/home/${SFTP_USER}" - -exec /usr/sbin/sshd -D -e diff --git a/docker/resources/infrastructure/sftp/files/sshd_config b/docker/resources/infrastructure/sftp/files/sshd_config deleted file mode 100644 index 96e7861832..0000000000 --- a/docker/resources/infrastructure/sftp/files/sshd_config +++ /dev/null @@ -1,24 +0,0 @@ -# Secure defaults -Protocol 2 -HostKey /etc/ssh/ssh_host_ed25519_key -HostKey /etc/ssh/ssh_host_rsa_key - -# IPv4 Only -AddressFamily inet - -# Faster connection -UseDNS no - -# Limit access -PermitRootLogin no -X11Forwarding no -AllowTcpForwarding no -PasswordAuthentication no - -# Force sftp and chroot jail -Subsystem sftp internal-sftp -ForceCommand internal-sftp -ChrootDirectory %h - -# Enable this for more logs -LogLevel VERBOSE diff --git a/docker/volumes/sftp/fulfilment/README.md b/docker/volumes/sftp/fulfilment/README.md deleted file mode 100644 index c919ee9d42..0000000000 --- a/docker/volumes/sftp/fulfilment/README.md +++ /dev/null @@ -1 +0,0 @@ -> Used to provide a remote FTP server folder for fulfilment - do not remove diff --git a/docker/volumes/sftp/pocl/README.md b/docker/volumes/sftp/pocl/README.md deleted file mode 100644 index fa717a9d65..0000000000 --- a/docker/volumes/sftp/pocl/README.md +++ /dev/null @@ -1 +0,0 @@ -> Used to provide a remote FTP server folder for POCL - do not remove diff --git a/packages/fulfilment-job/README.md b/packages/fulfilment-job/README.md index 566f9b5ff7..393cc1eca7 100644 --- a/packages/fulfilment-job/README.md +++ b/packages/fulfilment-job/README.md @@ -19,21 +19,16 @@ provider. # Environment variables -| name | description | required | default | valid | notes | -| ----------------------------------- | ----------------------------------------------------------------------------------------- | :------: | ------- | ----------------------------------------------------------------------- | ----- | -| NODE_ENV | Node environment | no | | development, test, production | | -| FULFILMENT_FILE_SIZE | The maximum number of records written to an aggregated fulfilment file | yes | | | | -| FULFILMENT_FTP_HOST | The hostname of the target FTP server | yes | | | | -| FULFILMENT_FTP_PORT | The port of the FTP service on the target server | yes | | | | -| FULFILMENT_FTP_PATH | The base path under which files should be written to the FTP server | yes | | | | -| FULFILMENT_FTP_USERNAME | The username used to authenticate with the FTP server | yes | | | | -| FULFILMENT_FTP_KEY_SECRET_ID | The ID of the secret in AWS secrets manager which contains the SSH key for authentication | yes | | | | -| FULFILMENT_S3_BUCKET | The name of the AWS S3 bucket in which to stage and aggregate fulfilment data | yes | | | | -| FULFILMENT_SEND_UNENCRYPTED_FILE | Flag for whether to send the unencrypted fulfilment file | no | false | true, false, 0, 1 | | -| FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID | The secret id for the file encryption public key | yes | | | | -| DEBUG | Use to enable output of debug information to the console | yes | | fulfilment:\*, fulfilment:staging, fulfilment:transport, fulfilment:ftp | | -| AIRBRAKE_HOST | URL of airbrake host | no | | | | -| AIRBRAKE_PROJECT_KEY | Project key for airbrake logging | no | | | | +| name | description | required | default | valid | notes | +| ----------------------------------- | ----------------------------------------------------------------------------- | :------: | ------- | ----------------------------------------------------------------------- | ----- | +| NODE_ENV | Node environment | no | | development, test, production | | +| FULFILMENT_FILE_SIZE | The maximum number of records written to an aggregated fulfilment file | yes | | | | +| FULFILMENT_S3_BUCKET | The name of the AWS S3 bucket in which to stage and aggregate fulfilment data | yes | | | | +| FULFILMENT_SEND_UNENCRYPTED_FILE | Flag for whether to send the unencrypted fulfilment file | no | false | true, false, 0, 1 | | +| FULFILMENT_PGP_PUBLIC_KEY_SECRET_ID | The secret id for the file encryption public key | yes | | | | +| DEBUG | Use to enable output of debug information to the console | yes | | fulfilment:\*, fulfilment:staging, fulfilment:transport, fulfilment:ftp | | +| AIRBRAKE_HOST | URL of airbrake host | no | | | | +| AIRBRAKE_PROJECT_KEY | Project key for airbrake logging | no | | | | ### See also: