Add dependabot config (#92)

* Add dependabot

* Add verified commit

* Patch lovely-pytest-docker to use `docker compose`

* Add rename check to logging tests

* Add upterm debugging

* Clear logs before test runs. And add debug logging

* Stop test intentionally failing

* Correct version number

* bump version v24.30.0 -> v24.32.0

* Add daily test run

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/daily_test.yml

@@ -0,0 +1,45 @@
+name: Daily Test
+
+env:
+  DEFAULT_PYTHON: "3.11"
+  PRE_COMMIT_CACHE: ~/.cache/pre-commit
+  KEY_PREFIX: base-venv
+  CACHE_VERSION: 1
+# yamllint disable-line rule:truthy
+on:
+  schedule:
+    - cron: "0 2 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  prepare-base:
+    name: Prepare base dependencies
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/[email protected]
+      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
+        id: python
+        uses: actions/[email protected]
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON }}
+          check-latest: true
+      - name: Create Python virtual environment
+        run: |
+          python -m venv venv
+          . venv/bin/activate
+          python -m pip install -U pip setuptools wheel
+          pip install -U ".[dev]"
+      - name: Run tests
+        env:
+          SMTP_USERNAME: ${{ secrets.SMTP_USERNAME }}
+          SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
+        run: |
+          . venv/bin/activate
+          pip install -U .
+          python -m pytest

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+# v24.32.0
+
+- Add dependabot config.
+- Add debug logging to `get_latest_log_file` function
+- Fix test for `get_latest_log_file` function to clear the log directory before running the test
+
 # v24.30.0
 
 - Add `supportsPosixRename` option to SFTP source so that post copy actions will work on server that don't support POSIX renames

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "opentaskpy"
-version = "v24.30.0"
+version = "v24.32.0"
 authors = [{ name = "Adam McDonagh", email = "[email protected]" }]
 license = { text = "GPLv3" }
 classifiers = [
@@ -71,7 +71,7 @@ otf-batch-validator = "opentaskpy.cli.batch_validator:main"
 profile = 'black'
 
 [tool.bumpver]
-current_version = "v24.30.0"
+current_version = "v24.32.0"
 version_pattern = "vYY.WW.PATCH[-TAG]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

src/opentaskpy/otflogging.py

@@ -236,18 +236,24 @@ def get_latest_log_file(task_id: str, task_type: str) -> str | None:
     # Also, we don't want to limit to running jobs, only failed or successful ones
     log_file_name = log_file_name.replace("_running", "(_failed)*")
 
+    logger.debug(f"Looking for log file: {log_file_name}")
     if not os.path.exists(os.path.dirname(log_file_name)):
         return None
 
     # List the contents of the directory
     log_files = os.listdir(os.path.dirname(log_file_name))
+    logger.debug(f"Found {len(log_files)} files")
+    logger.debug(f"Log files: {log_files}")
     # Filter the list to only include files that match the log_file_name, and contain a valid date/time prefix
     log_files = [
         f
         for f in log_files
         if re.match(os.path.basename(log_file_name), f)
         and re.match(r"\d{8}-\d{6}\.\d{3}", f.split("_")[0])
     ]
+    logger.debug(
+        f"Log files after filtering out logs that dont have valid date prefix and {log_file_name}: {log_files}"
+    )
 
     # Unless another date is given, only look at today's logs
     batch_resume_date = datetime.now().date()
@@ -263,9 +269,13 @@ def get_latest_log_file(task_id: str, task_type: str) -> str | None:
         if datetime.strptime(f.split("_")[0], "%Y%m%d-%H%M%S.%f").date()
         == batch_resume_date
     ]
+    logger.debug(
+        f"Log files after filtering out logs that dont match date: {log_files}"
+    )
 
     # Sort the list by the date/time in the filename
     log_files.sort(key=lambda x: datetime.strptime(x.split("_")[0], "%Y%m%d-%H%M%S.%f"))
+    logger.debug(f"Log files after sorting: {log_files}")
     # Get the latest log file
     if log_files:
         log_file_name = f"{os.path.dirname(log_file_name)}/{log_files[-1]}"

tests/conftest.py

@@ -0,0 +1,19 @@
+# monkey patch lovely pytest docker plugin
+# lovely-pytest-docker is no longer maintained, so we need to monkey patch it
+# to use docker compose instead as docker-compose v1 is no longer part of the
+# GitHub Actions runner
+from lovely.pytest.docker.compose import DockerComposeExecutor, execute
+
+
+def docker_compose_executor(self, *subcommand):
+    command = ["docker", "compose", "--project-directory", self.project_directory]
+    for compose_file in self._compose_files:  # pylint: disable=protected-access
+        command.append("-f")
+        command.append(compose_file)
+    command.append("-p")
+    command.append(self._project_name)  # pylint: disable=protected-access
+    command += subcommand
+    return execute(command)
+
+
+DockerComposeExecutor.execute = docker_compose_executor

tests/test_logging.py

@@ -3,11 +3,12 @@
 import logging
 import os
 import re
+import shutil
 import time
 from datetime import datetime
 
 import opentaskpy.otflogging
-from tests.fixtures.ssh_clients import *  # noqa: F403
+from tests.fixtures.ssh_clients import *  # noqa: F403, F401
 
 
 def test_define_log_file_name(env_vars, tmpdir, top_level_root_dir):
@@ -120,9 +121,14 @@ def test_get_latest_log_file(env_vars):
     # Setup some dummy log files
     log_path = "test/testLogs"
     os.environ["OTF_LOG_DIRECTORY"] = log_path
+    # Set debug logging
+    opentaskpy.otflogging.logger.setLevel(logging.DEBUG)
 
     last_created_file = None
 
+    # Empty the log directory
+    shutil.rmtree(log_path, ignore_errors=True)
+
     # loop 1 to 10
     for _ in range(1, 11):
         log_file_name = opentaskpy.otflogging._define_log_file_name(None, "B")
@@ -146,14 +152,23 @@ def test_get_latest_log_file(env_vars):
     # Rename the last created file to remove the _running suffix
     os.rename(last_created_file, last_created_file.replace("_running", ""))
     last_created_file = last_created_file.replace("_running", "")
+
+    # Check the rename worked
+    assert os.path.exists(last_created_file)
+
     # Run the function again and validate that it still returns nothing
     assert opentaskpy.otflogging.get_latest_log_file(None, "B") is None
 
     # Rename this file to _failed
-    os.rename(last_created_file, last_created_file.replace("_B", "_B_failed"))
-    last_created_file = last_created_file.replace("_B", "_B_failed")
+    failed_file = last_created_file.replace("_B", "_B_failed")
+    os.rename(last_created_file, failed_file)
+    # Check the rename worked
+    assert os.path.exists(failed_file)
+
     # Run the function again and validate that it returns this file
-    assert opentaskpy.otflogging.get_latest_log_file(None, "B") == last_created_file
+    print("Testing finding a failed log to resume from")
+    print(f"Expect to find {failed_file}")
+    assert opentaskpy.otflogging.get_latest_log_file(None, "B") == failed_file
 
     # Create a new file that has succeeded, make sure it still returns None, as the latest
     # state is success