From fba8f511a54e6ff7b4064006c6221b8a4d10304e Mon Sep 17 00:00:00 2001 From: DanielPerkins7 <141025447+DanielPerkins7@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:20:33 +0100 Subject: [PATCH 01/31] Fix logo URLs and files have moved --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 47762b75..fb890868 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ For more information about SSM, see the [EGI wiki](https://wiki.egi.eu/wiki/APEL ## Acknowledgements - STFC logo - EU flag - EOSC-hub logo + STFC logo + EU flag + EOSC-hub logo SSM is provided by [STFC](https://stfc.ukri.org/), a part of [UK Research and Innovation](https://www.ukri.org/), and is co-funded by the [EOSC-hub](https://www.eosc-hub.eu/) project (Horizon 2020) under Grant number 777536. Licensed under the [Apache 2 License](http://www.apache.org/licenses/LICENSE-2.0). From ccbcfd42b13b6546ad2bc074a86742e781a99d04 Mon Sep 17 00:00:00 2001 From: DanielPerkins7 <141025447+DanielPerkins7@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:29:22 +0100 Subject: [PATCH 02/31] Remove mention of RHEL 6 from README No longer pacakged for RHEL 6 --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index fb890868..62e3991e 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,7 @@ using the [STOMP protocol](http://stomp.github.io/) or via the ARGO Messaging Se Messages are signed and may be encrypted during transit. Persistent queues should be used to guarantee delivery. -SSM is written in Python. Packages are available for RHEL 6 and 7, and - Ubuntu Trusty. +SSM is written in Python. Packages are available for RHEL 7, and Ubuntu Trusty. For more information about SSM, see the [EGI wiki](https://wiki.egi.eu/wiki/APEL/SSM). From 3a870cd360f357b66350e93ea192c59322bcdddf Mon Sep 17 00:00:00 2001 From: DanielPerkins7 <141025447+DanielPerkins7@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:35:15 +0100 Subject: [PATCH 03/31] Remove EOSC-hub acknowledgement and logos --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 62e3991e..cf4c6d68 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,10 @@ For more information about SSM, see the [EGI wiki](https://wiki.egi.eu/wiki/APEL ## Acknowledgements - STFC logo - EU flag - EOSC-hub logo + STFC logo -SSM is provided by [STFC](https://stfc.ukri.org/), a part of [UK Research and Innovation](https://www.ukri.org/), and is co-funded by the [EOSC-hub](https://www.eosc-hub.eu/) project (Horizon 2020) under Grant number 777536. Licensed under the [Apache 2 License](http://www.apache.org/licenses/LICENSE-2.0). +SSM is provided by [STFC](https://stfc.ukri.org/), a part of [UK Research and Innovation](https://www.ukri.org/). Licensed under the [Apache 2 License](http://www.apache.org/licenses/LICENSE-2.0). ## Installing the RPM From cee0d9e18997b0581f94d69304246b9b2056307a Mon Sep 17 00:00:00 2001 From: DanielPerkins7 <141025447+DanielPerkins7@users.noreply.github.com> Date: Mon, 31 Jul 2023 13:17:14 +0100 Subject: [PATCH 04/31] Add LDAP version requirment to README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cf4c6d68..fb647589 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ The Python STOMP library (N.B. versions between 3.1.1 (inclusive) and 5.0.0 The Python AMS library. This is only required if you want to use AMS. See here for details on obtaining an RPM: https://github.com/ARGOeu/argo-ams-library/ -The Python ldap library +The Python ldap library (N.B. versions before 3.4.0 (exclusive) are currently supported) * `yum install python-ldap` Optionally, the Python dirq library (N.B. this is only required if your messages From cb8957ad27ee50a327a0cf928bc4b3438253ff4c Mon Sep 17 00:00:00 2001 From: DanielPerkins7 Date: Tue, 1 Aug 2023 08:19:56 +0000 Subject: [PATCH 05/31] redundant line and comment removed resolve #236 --- ssm/agents.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ssm/agents.py b/ssm/agents.py index 3d6ef885..673a9906 100644 --- a/ssm/agents.py +++ b/ssm/agents.py @@ -141,10 +141,6 @@ def get_ssm_args(protocol, cp, log): elif protocol == Ssm2.AMS_MESSAGING: # Then we are setting up an SSM to connect to a AMS. - # TODO: See if setting use_ssl directly in Ssm2 constructor is ok. - # 'use_ssl' isn't checked when using AMS (SSL is always used), but it - # is needed for the call to the Ssm2 constructor below. - use_ssl = None try: # We only need a hostname, not a port host = cp.get('broker', 'host') From 83e33c783c0bc2c15fb801408a0a7b1f0c3cb02f Mon Sep 17 00:00:00 2001 From: rowan04 Date: Tue, 18 Jul 2023 14:42:17 +0000 Subject: [PATCH 06/31] Update logging in test_crypto - removes old comment - gets rid of logging.getLogger('SSM') --- test/test_crypto.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_crypto.py b/test/test_crypto.py index db13fdbb..0dcb4001 100644 --- a/test/test_crypto.py +++ b/test/test_crypto.py @@ -17,9 +17,7 @@ verify_cert, \ CryptoException -# Set up logging - is this necessary? logging.basicConfig() -log = logging.getLogger('SSM') class TestEncryptUtils(unittest.TestCase): From 59f0f0ae18ddc632e42f758dfb81e146553e565c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Sep 2023 06:42:14 +0000 Subject: [PATCH 07/31] Bump actions/upload-artifact from 3.1.2 to 3.1.3 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3.1.2 to 3.1.3. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v3.1.2...v3.1.3) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/build-pkgs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-pkgs.yml b/.github/workflows/build-pkgs.yml index a48e35ea..d2b1232b 100644 --- a/.github/workflows/build-pkgs.yml +++ b/.github/workflows/build-pkgs.yml @@ -51,7 +51,7 @@ jobs: run: rpmlint ${{ steps.rpm.outputs.rpm_dir_path }} - name: Upload artifact - uses: actions/upload-artifact@v3.1.2 + uses: actions/upload-artifact@v3.1.3 with: name: Binary and Source RPMs path: | From aad7194f26f56c5c79a1ff39c00406d2d285950f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Sep 2023 16:08:46 +0000 Subject: [PATCH 08/31] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build-pkgs.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/unit-test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-pkgs.yml b/.github/workflows/build-pkgs.yml index d2b1232b..43589e31 100644 --- a/.github/workflows/build-pkgs.yml +++ b/.github/workflows/build-pkgs.yml @@ -21,7 +21,7 @@ jobs: run: | sudo apt-get update sudo apt-get install rpmlint - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # Get all branches and tags so the latest tag can be found for VERSION fetch-depth: 0 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 29501c12..37232599 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -34,7 +34,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index f7f89548..cb3048c0 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -11,7 +11,7 @@ jobs: python-version: ['2.x', '3.x'] name: Python ${{ matrix.python-version }} test steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: From 7594e5ca898f5f2f496e8199f8bb0064af891ac3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Sep 2023 16:11:41 +0000 Subject: [PATCH 09/31] Bump docker/build-push-action from 4.1.1 to 5.0.0 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4.1.1 to 5.0.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v4.1.1...v5.0.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6190ce31..4f016aa2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -48,7 +48,7 @@ jobs: # Build and push Docker image # https://github.com/docker/build-push-action name: Build and push Docker image - uses: docker/build-push-action@v4.1.1 + uses: docker/build-push-action@v5.0.0 with: # Only push containers to the registry on GitHub pushes, # not pull requests. GitHub won't let a rogue PR create a container From 1d2d622a76fc4b8a2e69a1a1d5645ac2301e8e96 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 06:58:14 +0000 Subject: [PATCH 10/31] Bump docker/metadata-action from 4 to 5 Bumps [docker/metadata-action](https://github.com/docker/metadata-action) from 4 to 5. - [Release notes](https://github.com/docker/metadata-action/releases) - [Upgrade guide](https://github.com/docker/metadata-action/blob/master/UPGRADE.md) - [Commits](https://github.com/docker/metadata-action/compare/v4...v5) --- updated-dependencies: - dependency-name: docker/metadata-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4f016aa2..1bfbc292 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -40,7 +40,7 @@ jobs: # https://github.com/docker/metadata-action name: Extract Docker metadata id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} From 0ec0b316e07b835670f1f3f8b4e7288cf2cd2fb9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 08:05:24 +0000 Subject: [PATCH 11/31] Bump docker/login-action from 2 to 3 Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1bfbc292..59f06ebc 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -30,7 +30,7 @@ jobs: # Login against a Docker registry # https://github.com/docker/login-action name: Login to ${{ env.REGISTRY }} - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} From 7b2011589499920f093611345ee59adb18f3ff5c Mon Sep 17 00:00:00 2001 From: DanielPerkins7 Date: Tue, 1 Aug 2023 08:44:13 +0000 Subject: [PATCH 12/31] else case added to issue 242 resolve #242 --- ssm/ssm2.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 72099ad8..4d3a0e79 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -392,6 +392,9 @@ def _send_msg_ams(self, text, msgid): argo_response = self._ams.publish(self._dest, message, retry=3) return argo_response['messageIds'][0] + else: + return None + def pull_msg_ams(self): """Pull 1 message from the AMS and acknowledge it.""" From 2cef8c9a7ddabb32afe187a107c431fa84e304ec Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 18 Sep 2023 11:14:46 +0100 Subject: [PATCH 13/31] Add explanatory comment --- ssm/ssm2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 4d3a0e79..51efdfa8 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -393,6 +393,8 @@ def _send_msg_ams(self, text, msgid): argo_response = self._ams.publish(self._dest, message, retry=3) return argo_response['messageIds'][0] else: + # We ignore empty messages as there is no point sending them. + # (STOMP did require empty messages to keep the connection alive.) return None From b248d7a57d88d03776135a319fda64466459dca2 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 18 Sep 2023 11:19:51 +0100 Subject: [PATCH 14/31] Remove extra line --- ssm/ssm2.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 51efdfa8..21e225b7 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -397,7 +397,6 @@ def _send_msg_ams(self, text, msgid): # (STOMP did require empty messages to keep the connection alive.) return None - def pull_msg_ams(self): """Pull 1 message from the AMS and acknowledge it.""" if self._protocol != Ssm2.AMS_MESSAGING: From 2034e650dc7a5e58608f355206cb09db2fd2a0a0 Mon Sep 17 00:00:00 2001 From: Nicholas Whyatt Date: Mon, 18 Sep 2023 08:38:12 +0000 Subject: [PATCH 15/31] Minor timeout changes Add timeout to prevent TCP spinlocks --- ssm/ssm2.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ssm/ssm2.py b/ssm/ssm2.py index 21e225b7..9cbc28e2 100644 --- a/ssm/ssm2.py +++ b/ssm/ssm2.py @@ -390,7 +390,7 @@ def _send_msg_ams(self, text, msgid): message = AmsMessage(data=to_send, attributes={'empaid': msgid}).dict() - argo_response = self._ams.publish(self._dest, message, retry=3) + argo_response = self._ams.publish(self._dest, message, retry=3, timeout=10) return argo_response['messageIds'][0] else: # We ignore empty messages as there is no point sending them. @@ -415,7 +415,8 @@ def pull_msg_ams(self): for msg_ack_id, msg in self._ams.pull_sub(self._listen, messages_to_pull, - retry=3): + retry=3, + timeout=10): # Get the AMS message id msgid = msg.get_msgid() # Get the SSM dirq id @@ -444,7 +445,7 @@ def pull_msg_ams(self): # it can move the offset for the next subscription pull # (basically acknowledging pulled messages) if ackids: - self._ams.ack_sub(self._listen, ackids, retry=3) + self._ams.ack_sub(self._listen, ackids, retry=3, timeout=10) def send_ping(self): """Perform connection stay-alive steps. From e2fa9f4b336992dbf7506dc7e1e679749f1c89f4 Mon Sep 17 00:00:00 2001 From: RedProkofiev Date: Mon, 18 Sep 2023 10:41:02 +0000 Subject: [PATCH 16/31] Added warning message to LDAP initialisation. --- ssm/brokers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ssm/brokers.py b/ssm/brokers.py index 598824e0..e6013db6 100644 --- a/ssm/brokers.py +++ b/ssm/brokers.py @@ -43,6 +43,8 @@ class StompBrokerGetter(object): def __init__(self, bdii_url): """Set up the LDAP connection and strings which are re-used.""" # Set up the LDAP connection + logging.warning('LDAP is deprecated and will be removed in an upcoming version, ' + + 'migrate to AMS.') log.debug('Connecting to %s...', bdii_url) self._ldap_conn = ldap.initialize(bdii_url) @@ -99,7 +101,7 @@ def _get_broker_details(self, service_type): return broker_details def _broker_in_network(self, broker_id, network): - """Check that a GlueServiceUniqueID is part of a specified netowrk.""" + """Check that a GlueServiceUniqueID is part of a specified network.""" ldap_filter = '(&(GlueServiceDataKey=cluster)(GlueChunkKey=GlueServiceUniqueID=%s))' \ % broker_id attrs = [self._service_data_value_key] From 961c652fea995d054e9327a9f99ef4e97860a100 Mon Sep 17 00:00:00 2001 From: RedProkofiev <87487335+RedProkofiev@users.noreply.github.com> Date: Mon, 18 Sep 2023 12:30:36 +0100 Subject: [PATCH 17/31] Update ssm/brokers.py Change wording of warning message Co-authored-by: Adrian Coveney <4836233+tofu-rocketry@users.noreply.github.com> --- ssm/brokers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ssm/brokers.py b/ssm/brokers.py index e6013db6..28826c34 100644 --- a/ssm/brokers.py +++ b/ssm/brokers.py @@ -43,8 +43,8 @@ class StompBrokerGetter(object): def __init__(self, bdii_url): """Set up the LDAP connection and strings which are re-used.""" # Set up the LDAP connection - logging.warning('LDAP is deprecated and will be removed in an upcoming version, ' + - 'migrate to AMS.') + logging.warning('LDAP is deprecated and will be removed in an upcoming version, ' + 'please set host locally in SSM config.') log.debug('Connecting to %s...', bdii_url) self._ldap_conn = ldap.initialize(bdii_url) From 261945025cec9935dec1cef4e4d6433e0d1d67cb Mon Sep 17 00:00:00 2001 From: gregcorbett Date: Mon, 18 Sep 2023 16:01:22 +0100 Subject: [PATCH 18/31] Cap version of certifi used below 2020.4.5.2 (#264) Cap version of certifi used below 2020.4.5.2 2020.4.5.2 dropped Python 2 support: - https://github.com/certifi/python-certifi/commit/5efdd48f719d9c3c7c8f9a812da2256d088eab78 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 23c85f7a..e2c93c97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # Base requirements for ssm argo-ams-library +certifi<2020.4.5.2 # Used by AMS (via requests), 2020.4.5.2 dropped support for Python 2 stomp.py<5.0.0 python-daemon<=2.3.0 # 2.3.1 dropped support for Python 2 python-ldap<3.4.0 # python-ldap-3.4.0 dropped support for Python 2 From 6f7a21c47ff3baf105ed35b63c874be20842651a Mon Sep 17 00:00:00 2001 From: Rose <83274393+RoseECooper@users.noreply.github.com> Date: Mon, 18 Sep 2023 16:22:34 +0100 Subject: [PATCH 19/31] Add sys.exit() to both sender and receiver (#263) * Add sys.exit(1) to both sender and receiver. Return 1 instead of 0 when failing due to an exception * Added a flag to both sender and receiver to return true when an exception is thrown. Moved sys.exit() to the end of the sender and receiver functions. --- ssm/agents.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ssm/agents.py b/ssm/agents.py index 673a9906..07ebeed7 100644 --- a/ssm/agents.py +++ b/ssm/agents.py @@ -250,6 +250,9 @@ def run_sender(protocol, brokers, project, token, cp, log): print('SSM failed to complete successfully. See log file for details.') log.error('Unexpected exception in SSM: %s', e) log.error('Exception type: %s', e.__class__) + sender_failed = True + else: + sender_failed = False try: sender.close_connection() @@ -259,6 +262,8 @@ def run_sender(protocol, brokers, project, token, cp, log): log.info('SSM has shut down.') log.info(LOG_BREAK) + if sender_failed: + sys.exit(1) def run_receiver(protocol, brokers, project, token, cp, log, dn_file): @@ -344,15 +349,24 @@ def run_receiver(protocol, brokers, project, token, cp, log, dn_file): log.info('Received the shutdown signal: %s', e) ssm.shutdown() dc.close() + receiver_failed = True except Exception as e: log.error('Unexpected exception: %s', e) log.error('Exception type: %s', e.__class__) log.error('The SSM will exit.') ssm.shutdown() dc.close() + receiver_failed = True + # Currently won't run the else statement due to the while loop in the reciever + # Leaving here in case of future refactoring, but commented out so the unreachable + # code isn't flagged by tests + # else: + # receiver_failed = False log.info('Receiving SSM has shut down.') log.info(LOG_BREAK) + if receiver_failed: + sys.exit(1) def get_dns(dn_file, log): From 835ac6e25fd2db81308704992ea406bda0e59a4d Mon Sep 17 00:00:00 2001 From: gregcorbett Date: Mon, 18 Sep 2023 16:12:16 +0100 Subject: [PATCH 20/31] Fix copy/paste error in receiver docs --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fb647589..7cf98716 100644 --- a/README.md +++ b/README.md @@ -208,8 +208,8 @@ add your messages using the `add` method. ``` docker run \ -d --entrypoint ssmreceive \ - -v /path/to/downloaded/config/sender.cfg:/etc/apel/sender.cfg \ - -v /path/to/read/messages:/var/spool/apel/outgoing \ + -v /path/to/downloaded/config/receiver.cfg:/etc/apel/receiver.cfg \ + -v /path/to/read/messages:/var/spool/apel/ \ -v /path/to/dns/file:/etc/apel/dns \ -v /etc/grid-security:/etc/grid-security \ -v /path/to/persistently/log:/var/log/apel \ From dfa3db5cc9bba2df007093a335618b5f8b55ad17 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 26 Sep 2023 09:55:32 +0100 Subject: [PATCH 21/31] Remove Py 2 from GH Action CI run Python 2 is not supported on GH runners --- .github/workflows/unit-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index cb3048c0..6d58f556 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['2.x', '3.x'] + python-version: ['3.x'] name: Python ${{ matrix.python-version }} test steps: - uses: actions/checkout@v4 From 15fc9a2d48a7e00b2b68020466351133ceb0b524 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 26 Sep 2023 12:34:07 +0100 Subject: [PATCH 22/31] Update pre-commit repo and add a couple of checks --- .pre-commit-config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f80092a2..6305e79c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,6 +15,7 @@ repos: - id: check-added-large-files - id: check-merge-conflict - id: check-yaml + - id: debug-statements - id: end-of-file-fixer - id: mixed-line-ending name: Force line endings to LF @@ -22,8 +23,9 @@ repos: - id: trailing-whitespace - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.9.0 + rev: v1.10.0 hooks: - id: python-check-mock-methods - id: python-no-eval - id: python-no-log-warn + - id: python-use-type-annotations From 1204fcc91179564d5816b2dc1c7a7d9595b73da7 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Tue, 26 Sep 2023 12:58:10 +0100 Subject: [PATCH 23/31] Add pre-commit CI config Disables autofixes on PRs and reduces autoupdate frequency (from weekly) as at the moment we have to use the versions specified for Py2 compatability. --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6305e79c..86c0ad80 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,3 +29,8 @@ repos: - id: python-no-eval - id: python-no-log-warn - id: python-use-type-annotations + +# Pre-commit CI config, see https://pre-commit.ci/ +ci: + autofix_prs: false + autoupdate_schedule: quarterly From afe46aab24bf5b6c9aa34c19041a6aee0bfd4293 Mon Sep 17 00:00:00 2001 From: DanielPerkins7 Date: Tue, 1 Aug 2023 14:53:03 +0000 Subject: [PATCH 24/31] Check if item is a file when using directory queue This fixes issues with the item count being wrong and whole directories being passed on to the sender. --- ssm/message_directory.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index b7a70c65..667ea8e8 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -110,7 +110,10 @@ def _get_messages(self, sort_by_mtime=False): for (file_name, _mtime) in sorted(unsorted_messages, key=lambda tup: tup[1]): # Store the sorted file paths in a class element. - sorted_messages.append(file_name) + if os.path.isfile(file_name): + sorted_messages.append(file_name) + else: + continue # Return the sorted list. return sorted_messages From 3fcbb5413f5c3bba956fda6a316bd06c54bef8e1 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 9 Oct 2023 13:22:05 +0100 Subject: [PATCH 25/31] Revert "Check if item is a file when using directory queue" This reverts commit afe46aab24bf5b6c9aa34c19041a6aee0bfd4293 as the file filtering needs to be implmented earlier in the code. --- ssm/message_directory.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index 667ea8e8..b7a70c65 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -110,10 +110,7 @@ def _get_messages(self, sort_by_mtime=False): for (file_name, _mtime) in sorted(unsorted_messages, key=lambda tup: tup[1]): # Store the sorted file paths in a class element. - if os.path.isfile(file_name): - sorted_messages.append(file_name) - else: - continue + sorted_messages.append(file_name) # Return the sorted list. return sorted_messages From 132410a56ce25aba480bb9fb0b101ae4c08ec2a9 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 9 Oct 2023 13:54:30 +0100 Subject: [PATCH 26/31] Add unit test for dir in mess dir issue --- test/test_message_directory.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test_message_directory.py b/test/test_message_directory.py index af78bdc7..9e2cdcb2 100644 --- a/test/test_message_directory.py +++ b/test/test_message_directory.py @@ -137,6 +137,15 @@ def test_remove(self): # Check the count method returns the expected value. self.assertEqual(self.message_directory.count(), 0) + def test_dir_in_dir(self): + """Check that directories inside the queue are being ignored.""" + self.longMessage = True # Include normal unittest output before custom message. + + with tempfile.TemporaryFile(dir=self.tmp_dir): + tempfile.mkdtemp(prefix='extra_directory_', dir=self.tmp_dir) + self.assertEqual(self.message_directory.count(), 1, "Expected just one file, " + "but greater result implies that directory is being counted.") + def tearDown(self): """Remove test directory and all contents.""" try: From bc56fd343dbd190dd16d51f6b61e6718346e0429 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 9 Oct 2023 13:55:47 +0100 Subject: [PATCH 27/31] Filter file list to exclude directories --- ssm/message_directory.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ssm/message_directory.py b/ssm/message_directory.py index b7a70c65..62dad40d 100644 --- a/ssm/message_directory.py +++ b/ssm/message_directory.py @@ -87,8 +87,9 @@ def _get_messages(self, sort_by_mtime=False): """ try: # Get a list of files under self.directory_path - # in an arbitrary order. - file_name_list = os.listdir(self.directory_path) + # in an arbitrary order (ignoring directories). + file_name_list = [file for file in os.listdir(self.directory_path) + if os.path.isfile(os.path.join(self.directory_path, file))] if sort_by_mtime: # Working space to hold the unsorted messages From 1defe699af08710b7e06cef31cf635a854f93457 Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 9 Oct 2023 13:57:04 +0100 Subject: [PATCH 28/31] Add underscore to temp dir name to make readable tempfile creates files and folders with a random name, but adds the prefix directly before it, so this makes it more readable by separating those parts. --- test/test_message_directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_message_directory.py b/test/test_message_directory.py index 9e2cdcb2..fae85203 100644 --- a/test/test_message_directory.py +++ b/test/test_message_directory.py @@ -27,7 +27,7 @@ class TestMessageDirectory(unittest.TestCase): def setUp(self): """Create a MessageDirectory class on top of a temporary directory.""" - self.tmp_dir = tempfile.mkdtemp(prefix='message_directory') + self.tmp_dir = tempfile.mkdtemp(prefix='message_directory_') self.message_directory = MessageDirectory(self.tmp_dir) def test_add_and_get(self): From b8a08e63a473e0cf38c48f170fb3e6e98261725c Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 9 Oct 2023 14:46:22 +0100 Subject: [PATCH 29/31] Rewrite test to work on Unix Previous version of test failed in Ubuntu Travis environment, possibly due to the file being kept open in the context manager. --- test/test_message_directory.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/test/test_message_directory.py b/test/test_message_directory.py index fae85203..2e3d8541 100644 --- a/test/test_message_directory.py +++ b/test/test_message_directory.py @@ -14,6 +14,7 @@ """This module contains test cases for the MessageDirectory class.""" from __future__ import print_function +import os import shutil import tempfile import time @@ -141,10 +142,14 @@ def test_dir_in_dir(self): """Check that directories inside the queue are being ignored.""" self.longMessage = True # Include normal unittest output before custom message. - with tempfile.TemporaryFile(dir=self.tmp_dir): - tempfile.mkdtemp(prefix='extra_directory_', dir=self.tmp_dir) - self.assertEqual(self.message_directory.count(), 1, "Expected just one file, " - "but greater result implies that directory is being counted.") + # Add a single test file (closing it to ensure this works on Unix) + handle, _path = tempfile.mkstemp(dir=self.tmp_dir) + os.close(handle) + # Add a directory (to ignore) + tempfile.mkdtemp(prefix='extra_directory_', dir=self.tmp_dir) + + self.assertEqual(self.message_directory.count(), 1, "Expected just one file, " + "but greater result implies that directory is being counted.") def tearDown(self): """Remove test directory and all contents.""" From c862f921208098d3da3f53cee09aa4d679f2abaf Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 9 Oct 2023 15:40:10 +0100 Subject: [PATCH 30/31] Update version numbers for 3.3.1 --- apel-ssm.spec | 2 +- scripts/ssm-build-deb.sh | 2 +- scripts/ssm-build-rpm.sh | 2 +- ssm/__init__.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apel-ssm.spec b/apel-ssm.spec index 5be15b28..64ca9672 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -4,7 +4,7 @@ %endif Name: apel-ssm -Version: 3.3.0 +Version: 3.3.1 %define releasenumber 1 Release: %{releasenumber}%{?dist} Summary: Secure stomp messenger diff --git a/scripts/ssm-build-deb.sh b/scripts/ssm-build-deb.sh index 1a4db00c..a8e9aee1 100755 --- a/scripts/ssm-build-deb.sh +++ b/scripts/ssm-build-deb.sh @@ -16,7 +16,7 @@ set -eu -TAG=3.3.0-1 +TAG=3.3.1-1 SOURCE_DIR=~/debbuild/source BUILD_DIR=~/debbuild/build diff --git a/scripts/ssm-build-rpm.sh b/scripts/ssm-build-rpm.sh index e99d3725..b71ce358 100644 --- a/scripts/ssm-build-rpm.sh +++ b/scripts/ssm-build-rpm.sh @@ -10,7 +10,7 @@ rpmdev-setuptree RPMDIR=/home/rpmb/rpmbuild -VERSION=3.3.0-1 +VERSION=3.3.1-1 SSMDIR=apel-ssm-$VERSION # Remove old sources and RPMS diff --git a/ssm/__init__.py b/ssm/__init__.py index 74f42008..738a7e51 100644 --- a/ssm/__init__.py +++ b/ssm/__init__.py @@ -19,7 +19,7 @@ import logging import sys -__version__ = (3, 3, 0) +__version__ = (3, 3, 1) LOG_BREAK = '========================================' From bed0e273dc96c680d3caf28115fa7091fccef56a Mon Sep 17 00:00:00 2001 From: Adrian Coveney Date: Mon, 9 Oct 2023 16:07:17 +0100 Subject: [PATCH 31/31] Update changelogs for 3.3.1 --- CHANGELOG | 9 +++++++++ apel-ssm.spec | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index b0a161aa..84efd27c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,14 @@ Changelog for ssm ================= +* Mon Oct 09 2023 Adrian Coveney - 3.3.0-1 + - Added warning that BDII broker fetching will be deprecated in a future version. + - Added non-zero exit status if sender or receiver crash. + - Fixed SSM hanging if TCP connection drops by adding timeout. + - Fixed directory queue system picking up sub-directories. + - Fixed AMS messaging library dependency issue. + - Fixed documentation for running a containerised receiver. + - Fixed a few minor code issues. + * Thu Jun 29 2023 Adrian Coveney - 3.3.0-1 - Added destination queue to the log during startup to aid troubleshooting. - Added check that the config file exists to allow for better error messages. diff --git a/apel-ssm.spec b/apel-ssm.spec index 64ca9672..5333bdc0 100644 --- a/apel-ssm.spec +++ b/apel-ssm.spec @@ -100,6 +100,15 @@ rm -rf $RPM_BUILD_ROOT %doc %_defaultdocdir/%{name} %changelog +* Mon Oct 09 2023 Adrian Coveney - 3.3.0-1 + - Added warning that BDII broker fetching will be deprecated in a future version. + - Added non-zero exit status if sender or receiver crash. + - Fixed SSM hanging if TCP connection drops by adding timeout. + - Fixed directory queue system picking up sub-directories. + - Fixed AMS messaging library dependency issue. + - Fixed documentation for running a containerised receiver. + - Fixed a few minor code issues. + * Thu Jun 29 2023 Adrian Coveney - 3.3.0-1 - Added destination queue to the log during startup to aid troubleshooting. - Added check that the config file exists to allow for better error messages.