diff --git a/.github/workflows/build-pkgs.yml b/.github/workflows/build-pkgs.yml index cba54534..0420cb6a 100644 --- a/.github/workflows/build-pkgs.yml +++ b/.github/workflows/build-pkgs.yml @@ -51,7 +51,7 @@ jobs: run: rpmlint ${{ steps.rpm.outputs.rpm_dir_path }} - name: Upload artifact - uses: actions/upload-artifact@v4.3.1 + uses: actions/upload-artifact@v4.3.6 with: name: Binary and Source RPMs path: | diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c975d066..b67559c0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -48,7 +48,7 @@ jobs: # Build and push Docker image # https://github.com/docker/build-push-action name: Build and push Docker image - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v6.7.0 with: # Only push containers to the registry on GitHub pushes, # not pull requests. GitHub won't let a rogue PR create a container diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 5be689e1..63c500e5 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -4,31 +4,40 @@ on: [push, pull_request] jobs: unit-test: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 # 20.04 to allow for Py 3.6 strategy: fail-fast: false matrix: - python-version: ['3.x'] + # Python versions on Rocky 8, Ubuntu 20.04, Rocky 9 + python-version: ['3.6', '3.8', '3.9'] name: Python ${{ matrix.python-version }} test steps: - uses: actions/checkout@v4 - - name: Set up Python + + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Set up dependencies for python-ldap run: sudo apt-get install libsasl2-dev libldap2-dev libssl-dev + - name: Base requirements for SSM run: pip install -r requirements.txt + - name: Additional requirements for the unit and coverage tests run: pip install -r requirements-test.txt + - name: Pre-test set up run: | export TMPDIR=$PWD/tmp mkdir $TMPDIR export PYTHONPATH=$PYTHONPATH:`pwd -P` cd test + - name: Run unit tests run: coverage run --branch --source=ssm,bin -m unittest discover --buffer + - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3.1.4 + uses: codecov/codecov-action@v4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 86c0ad80..de1e3c08 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ # See https://pre-commit.com for more information repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v4.1.0 # Python 3.6 compatible hooks: # Python related checks - id: check-ast @@ -13,9 +13,13 @@ repos: files: 'test/.*' # Other checks - id: check-added-large-files + - id: check-case-conflict - id: check-merge-conflict - id: check-yaml - id: debug-statements + - id: detect-private-key + # This file has a test cert and key + exclude: 'test_ssm.py' - id: end-of-file-fixer - id: mixed-line-ending name: Force line endings to LF diff --git a/.travis.yml b/.travis.yml index 70e2d56c..40af6510 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,6 @@ os: linux language: python python: - "2.7" - - "3.8" # Cache the dependencies installed by pip cache: pip diff --git a/CHANGELOG b/CHANGELOG index c35d5537..b4145013 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,10 @@ +* Fri Aug 30 2024 Adrian Coveney - 3.4.1-1 + - Improved error logging to store full traceback on unexpected exceptions. + - Changed more code to use pyOpenSSL to improve compatibility with newer OpenSSL versions. + - Added a check to prevent a host certificate being to used for target server encryption. + - Changed which version of exit function is used to avoid edge case. + - Various changes and improvements to build scripts and processes. + * Wed Feb 21 2024 Adrian Coveney - 3.4.0-1 - Fixed compatability with newer versions of OpenSSL that only provide comma separated DNs. - Fixed Python 3 compatability (indirectly fixing EL8+ compatability) by performing explicit diff --git a/Dockerfile b/Dockerfile index aacd05f4..8c6b5e8b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,9 @@ -FROM centos:7 -MAINTAINER APEL Administrators +FROM rockylinux:9 +LABEL org.opencontainers.image.authors="apel-admins@stfc.ac.uk" +LABEL org.opencontainers.image.title="APEL SSM" +LABEL org.opencontainers.image.description="Secure STOMP Messenger (SSM) is designed to simply send messages using the STOMP protocol or via the ARGO Messaging Service (AMS)." +LABEL org.opencontainers.image.source="https://github.com/apel/ssm" +LABEL org.opencontainers.image.licenses="Apache License, Version 2.0" # Copy the SSM Git repository to /tmp/ssm COPY . /tmp/ssm @@ -9,10 +13,10 @@ WORKDIR /tmp/ssm # Add the EPEL repo so we can get pip RUN yum -y install epel-release && yum clean all # Then get pip -RUN yum -y install python-pip && yum clean all +RUN yum -y install python3-pip && yum clean all # Install the system requirements of python-ldap -RUN yum -y install gcc python-devel openldap-devel && yum clean all +RUN yum -y install gcc python3-devel openldap-devel && yum clean all # Install libffi, a requirement of openssl RUN yum -y install libffi-devel && yum clean all @@ -21,9 +25,9 @@ RUN yum -y install libffi-devel && yum clean all RUN yum -y install openssl && yum clean all # Install the python requirements of SSM -RUN pip install -r requirements.txt +RUN pip install -r requirements-docker.txt # Then install the SSM -RUN python setup.py install +RUN python3 setup.py install # Set the working directory back to / WORKDIR / diff --git a/apel-ssm.spec b/apel-ssm.spec index 82628a78..a23cc74d 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -4,7 +4,7 @@ %endif Name: apel-ssm -Version: 3.4.0 +Version: 3.4.1 %define releasenumber 1 Release: %{releasenumber}%{?dist} Summary: Secure stomp messenger @@ -100,6 +100,13 @@ rm -rf $RPM_BUILD_ROOT %doc %_defaultdocdir/%{name} %changelog +* Fri Aug 30 2024 Adrian Coveney - 3.4.1-1 + - Improved error logging to store full traceback on unexpected exceptions. + - Changed more code to use pyOpenSSL to improve compatibility with newer OpenSSL versions. + - Added a check to prevent a host certificate being to used for target server encryption. + - Changed which version of exit function is used to avoid edge case. + - Various changes and improvements to build scripts and processes. + * Wed Feb 21 2024 Adrian Coveney - 3.4.0-1 - Fixed compatability with newer versions of OpenSSL that only provide comma separated DNs. - Fixed Python 3 compatability (indirectly fixing EL8+ compatability) by performing explicit diff --git a/bin/receiver.py b/bin/receiver.py index dac81f3b..82674495 100644 --- a/bin/receiver.py +++ b/bin/receiver.py @@ -66,7 +66,7 @@ def main(): cp.read(options.config) else: print("Config file not found at", options.config) - exit(1) + sys.exit(1) # Check for pidfile pidfile = cp.get('daemon', 'pidfile') diff --git a/bin/sender.py b/bin/sender.py index f6d08e98..a058bbc4 100644 --- a/bin/sender.py +++ b/bin/sender.py @@ -24,6 +24,7 @@ import logging from optparse import OptionParser import os +import sys try: import ConfigParser @@ -57,7 +58,7 @@ def main(): cp.read(options.config) else: print("Config file not found at", options.config) - exit(1) + sys.exit(1) ssm.agents.logging_helper(cp) diff --git a/requirements-docker.txt b/requirements-docker.txt new file mode 100644 index 00000000..4765b7c5 --- /dev/null +++ b/requirements-docker.txt @@ -0,0 +1,12 @@ +# Base requirements for ssm + +argo-ams-library +pyopenssl +cryptography +stomp.py +python-daemon +python-ldap +setuptools # Required for pkg_resources (also happens to be a dependency of python-ldap) + +# Dependencies for optional dirq based sending +dirq diff --git a/scripts/ssm-build-deb.sh b/scripts/ssm-build-deb.sh index 707cc048..cc5df4f8 100755 --- a/scripts/ssm-build-deb.sh +++ b/scripts/ssm-build-deb.sh @@ -16,7 +16,7 @@ set -eu -TAG=3.4.0-1 +TAG=3.4.1-1 SOURCE_DIR=~/debbuild/source BUILD_DIR=~/debbuild/build diff --git a/scripts/ssm-build-rpm.sh b/scripts/ssm-build-rpm.sh index e6d1502d..f0c37a5b 100644 --- a/scripts/ssm-build-rpm.sh +++ b/scripts/ssm-build-rpm.sh @@ -10,7 +10,7 @@ rpmdev-setuptree RPMDIR=/home/rpmb/rpmbuild -VERSION=3.4.0-1 +VERSION=3.4.1-1 SSMDIR=apel-ssm-$VERSION # Remove old sources and RPMS diff --git a/scripts/ssm-build.sh b/scripts/ssm-build.sh index d2eb8248..9301006c 100755 --- a/scripts/ssm-build.sh +++ b/scripts/ssm-build.sh @@ -5,10 +5,13 @@ # @Author: Nicholas Whyatt (RedProkofiev@github.com) # Script runs well with FPM 1.14.2 on ruby 2.7.1, setuptools 51.3.3 on RHEL and Deb platforms -# Download ruby (if you're locked to 2.5, use RVM) and then run: + +# Download ruby (if you're locked to 2.5, use RVM, https://www.tecmint.com/install-ruby-on-centos-rhel-8/#installrubyrvm) and then run: # sudo gem install fpm -v 1.14.2 +# (may need to be run without the 'sudo') + # for RPM builds, you will also need: -# sudo yum install rpm-build | sudo apt-get install rpm +# sudo yum install rpm-build rpmlint | sudo apt-get install rpm lintian # ./ssm-build.sh (deb | rpm) # e.g. # ./ssm-build.sh deb 3.4.0 1 /usr/lib/python3.6 @@ -107,6 +110,7 @@ rm -f "$TAR_FILE" # Get supplied Python version PY_VERSION="$(basename "$PYTHON_ROOT_DIR")" PY_NUM=${PY_VERSION#python} +OS_EXTENSION="$(uname -r | grep -o 'el[7-9]' || echo '_all')" # Universal FPM Call FPM_CORE="fpm -s python \ @@ -127,25 +131,27 @@ if [[ ${PY_NUM:0:1} == "3" ]]; then if [[ "$PACK_TYPE" = "deb" ]]; then FPM_PYTHON="--depends python3 \ - --depends python-pip3 \ - --depends 'python-stomp' \ - --depends python-ldap \ + --depends python3-pip \ + --depends python3-cryptography \ + --depends python3-openssl \ + --depends python3-daemon \ + --depends 'python3-stomp' \ + --depends python3-ldap \ --depends libssl-dev \ --depends libsasl2-dev \ --depends openssl " - OS_EXTENSION="_all" - # Currently builds for el8 elif [[ "$PACK_TYPE" = "rpm" ]]; then FPM_PYTHON="--depends python3 \ --depends python3-stomppy \ --depends python3-pip \ + --depends python3-cryptography \ + --depends python3-pyOpenSSL \ + --depends python3-daemon \ --depends python3-ldap \ --depends openssl \ --depends openssl-devel " - - OS_EXTENSION="el8" fi elif [[ ${PY_NUM:0:1} == "2" ]]; then @@ -156,22 +162,24 @@ elif [[ ${PY_NUM:0:1} == "2" ]]; then --depends python-pip \ --depends 'python-stomp < 5.0.0' \ --depends python-ldap \ + --depends python-cryptography \ + --depends python-openssl \ + --depends python-daemon \ --depends libssl-dev \ --depends libsasl2-dev \ --depends openssl " - OS_EXTENSION="_all" - # el7 and below, due to yum package versions elif [[ "$PACK_TYPE" = "rpm" ]]; then FPM_PYTHON="--depends python2 \ --depends python2-pip \ + --depends python2-cryptography \ + --depends python2-pyOpenSSL \ + --depends python2-daemon \ --depends stomppy \ --depends python-ldap \ --depends openssl \ --depends openssl-devel " - - OS_EXTENSION="el7" fi fi @@ -179,6 +187,7 @@ fi PACKAGE_VERSION="--$PACK_TYPE-changelog $SOURCE_DIR/ssm-$VERSION-$ITERATION/CHANGELOG \ --$PACK_TYPE-dist $OS_EXTENSION \ --python-bin /usr/bin/$PY_VERSION \ + --python-install-bin /usr/bin \ --python-install-lib $PYTHON_ROOT_DIR$LIB_EXTENSION \ --exclude *.pyc \ --package $BUILD_DIR \ @@ -203,3 +212,18 @@ fpm -s pleaserun -t "$PACK_TYPE" \ --depends apel-ssm \ --package "$BUILD_DIR" \ /usr/bin/ssmreceive + +echo "Possible Issues to Fix:" +if [ "$OS_EXTENSION" == "_all" ] +then + # Check the resultant debs for 'lint' + TAG="$VERSION-$ITERATION" + DEBDIR="$(dirname "$BUILD_DIR")" + + lintian "$DEBDIR"/apel-ssm_"${TAG}"_all.deb + lintian "$DEBDIR"/apel-ssm-service_"${TAG}"_all.deb +else + # Check for errors in SPEC and built packages + # For instance; Given $(dirname /root/rpmb/rpmbuild/source) will output "/root/rpmb/rpmbuild". + rpmlint "$(dirname "$SOURCE_DIR")" +fi diff --git a/setup.py b/setup.py index c33e93c9..6ca3c7fb 100644 --- a/setup.py +++ b/setup.py @@ -22,14 +22,30 @@ from ssm import __version__ +def setup_temp_files(): + """Create temporary files with deployment names. """ + copyfile('bin/receiver.py', 'bin/ssmreceive') + copyfile('bin/sender.py', 'bin/ssmsend') + copyfile('scripts/apel-ssm.logrotate', 'conf/apel-ssm') + copyfile('README.md', 'apel-ssm') + + def main(): """Called when run as script, e.g. 'python setup.py install'.""" - # Create temporary files with deployment names - if 'install' in sys.argv: - copyfile('bin/receiver.py', 'bin/ssmreceive') - copyfile('bin/sender.py', 'bin/ssmsend') - copyfile('scripts/apel-ssm.logrotate', 'conf/apel-ssm') - copyfile('README.md', 'apel-ssm') + supported_commands = { + "install", + "build", + "bdist", + "develop", + "build_scripts", + "install_scripts", + "install_data", + "bdist_dumb", + "bdist_egg", + } + + if supported_commands.intersection(sys.argv): + setup_temp_files() # conf_files will later be copied to conf_dir conf_dir = '/etc/apel/' @@ -51,15 +67,15 @@ def main(): download_url='https://github.com/apel/ssm/releases', license='Apache License, Version 2.0', install_requires=[ - 'cryptography==3.3.2', - 'stomp.py<5.0.0', - 'python-ldap<3.4.0', + 'cryptography', + 'stomp.py', + 'python-ldap', 'setuptools', - 'pyopenssl >=19.1.0, <=21.0.0', + 'pyopenssl', ], extras_require={ - 'AMS': ['argo-ams-library', 'certifi<2020.4.5.2', ], - 'daemon': ['python-daemon<=2.3.0', ], + 'AMS': ['argo-ams-library', ], + 'daemon': ['python-daemon', ], 'dirq': ['dirq'], }, packages=find_packages(exclude=['bin', 'test']), @@ -79,7 +95,7 @@ def main(): ) # Remove temporary files with deployment names - if 'install' in sys.argv: + if supported_commands.intersection(sys.argv): remove('bin/ssmreceive') remove('bin/ssmsend') remove('conf/apel-ssm') diff --git a/ssm/__init__.py b/ssm/__init__.py index 904c0c7d..79ecfe73 100644 --- a/ssm/__init__.py +++ b/ssm/__init__.py @@ -19,7 +19,7 @@ import logging import sys -__version__ = (3, 4, 0) +__version__ = (3, 4, 1) LOG_BREAK = '========================================' diff --git a/ssm/agents.py b/ssm/agents.py index 07ebeed7..4726cbb2 100644 --- a/ssm/agents.py +++ b/ssm/agents.py @@ -222,6 +222,12 @@ def run_sender(protocol, brokers, project, token, cp, log): host_dn = get_certificate_subject(_from_file(host_cert)) log.info('Messages will be signed using %s', host_dn) + if server_cert == host_cert: + raise Ssm2Exception( + "server certificate is the same as host certificate in config file. " + "Do you really mean to encrypt messages with this certificate?" + ) + sender = Ssm2(brokers, cp.get('messaging', 'path'), path_type=path_type, @@ -246,10 +252,10 @@ def run_sender(protocol, brokers, project, token, cp, log): except (Ssm2Exception, CryptoException) as e: print('SSM failed to complete successfully. See log file for details.') log.error('SSM failed to complete successfully: %s', e) + sender_failed = True except Exception as e: print('SSM failed to complete successfully. See log file for details.') - log.error('Unexpected exception in SSM: %s', e) - log.error('Exception type: %s', e.__class__) + log.exception('Unexpected exception in SSM. See traceback below.') sender_failed = True else: sender_failed = False @@ -351,8 +357,7 @@ def run_receiver(protocol, brokers, project, token, cp, log, dn_file): dc.close() receiver_failed = True except Exception as e: - log.error('Unexpected exception: %s', e) - log.error('Exception type: %s', e.__class__) + log.exception('Unexpected exception in SSM. See traceback below.') log.error('The SSM will exit.') ssm.shutdown() dc.close() diff --git a/ssm/crypto.py b/ssm/crypto.py index cef91ae4..69066b91 100644 --- a/ssm/crypto.py +++ b/ssm/crypto.py @@ -49,10 +49,7 @@ def _from_file(filename): def check_cert_key(certpath, keypath): - """Check that a certificate and a key match. - - Uses openssl directly to fetch the modulus of each, which must be the same. - """ + """Check that a certificate and a key match.""" try: cert = _from_file(certpath) key = _from_file(keypath) @@ -64,23 +61,32 @@ def check_cert_key(certpath, keypath): if cert == key: return False - p1 = Popen(['openssl', 'x509', '-pubkey', '-noout'], - stdin=PIPE, stdout=PIPE, stderr=PIPE, universal_newlines=True) - pubkey1, error = p1.communicate(cert) + try: + certificate = OpenSSL.crypto.load_certificate( + OpenSSL.crypto.FILETYPE_PEM, cert + ) + crypto_public_key = certificate.get_pubkey() + certificate_public_key = OpenSSL.crypto.dump_publickey( + OpenSSL.crypto.FILETYPE_PEM, crypto_public_key + ) - if error != '': + except OpenSSL.crypto.Error as error: log.error(error) return False - p2 = Popen(['openssl', 'pkey', '-pubout'], - stdin=PIPE, stdout=PIPE, stderr=PIPE, universal_newlines=True) - pubkey2, error = p2.communicate(key) + try: + private_key = OpenSSL.crypto.load_privatekey( + OpenSSL.crypto.FILETYPE_PEM, key + ) + private_public_key = OpenSSL.crypto.dump_publickey( + OpenSSL.crypto.FILETYPE_PEM, private_key + ) - if error != '': + except OpenSSL.crypto.Error as error: log.error(error) return False - return pubkey1.strip() == pubkey2.strip() + return certificate_public_key.strip() == private_public_key.strip() def sign(text, certpath, keypath): """Sign the message using the certificate and key in the files specified.