diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..6a7695c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/daily_test.yml b/.github/workflows/daily_test.yml new file mode 100644 index 0000000..d902882 --- /dev/null +++ b/.github/workflows/daily_test.yml @@ -0,0 +1,45 @@ +name: Daily Test + +env: + DEFAULT_PYTHON: "3.11" + PRE_COMMIT_CACHE: ~/.cache/pre-commit + KEY_PREFIX: base-venv + CACHE_VERSION: 1 +# yamllint disable-line rule:truthy +on: + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + prepare-base: + name: Prepare base dependencies + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out code from GitHub + uses: actions/checkout@v3.5.2 + - name: Set up Python ${{ env.DEFAULT_PYTHON }} + id: python + uses: actions/setup-python@v4.6.0 + with: + python-version: ${{ env.DEFAULT_PYTHON }} + check-latest: true + - name: Create Python virtual environment + run: | + python -m venv venv + . venv/bin/activate + python -m pip install -U pip setuptools wheel + pip install -U ".[dev]" + - name: Run tests + env: + SMTP_USERNAME: ${{ secrets.SMTP_USERNAME }} + SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }} + run: | + . venv/bin/activate + pip install -U . + python -m pytest diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d8538b..9cb467b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +# v24.32.0 + +- Add dependabot config. +- Add debug logging to `get_latest_log_file` function +- Fix test for `get_latest_log_file` function to clear the log directory before running the test + # v24.30.0 - Add `supportsPosixRename` option to SFTP source so that post copy actions will work on server that don't support POSIX renames diff --git a/pyproject.toml b/pyproject.toml index 62d7a00..9aaf6ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "opentaskpy" -version = "v24.30.0" +version = "v24.32.0" authors = [{ name = "Adam McDonagh", email = "adam@elitemonkey.net" }] license = { text = "GPLv3" } classifiers = [ @@ -71,7 +71,7 @@ otf-batch-validator = "opentaskpy.cli.batch_validator:main" profile = 'black' [tool.bumpver] -current_version = "v24.30.0" +current_version = "v24.32.0" version_pattern = "vYY.WW.PATCH[-TAG]" commit_message = "bump version {old_version} -> {new_version}" commit = true diff --git a/src/opentaskpy/otflogging.py b/src/opentaskpy/otflogging.py index 7b02b24..c02d2a0 100644 --- a/src/opentaskpy/otflogging.py +++ b/src/opentaskpy/otflogging.py @@ -236,11 +236,14 @@ def get_latest_log_file(task_id: str, task_type: str) -> str | None: # Also, we don't want to limit to running jobs, only failed or successful ones log_file_name = log_file_name.replace("_running", "(_failed)*") + logger.debug(f"Looking for log file: {log_file_name}") if not os.path.exists(os.path.dirname(log_file_name)): return None # List the contents of the directory log_files = os.listdir(os.path.dirname(log_file_name)) + logger.debug(f"Found {len(log_files)} files") + logger.debug(f"Log files: {log_files}") # Filter the list to only include files that match the log_file_name, and contain a valid date/time prefix log_files = [ f @@ -248,6 +251,9 @@ def get_latest_log_file(task_id: str, task_type: str) -> str | None: if re.match(os.path.basename(log_file_name), f) and re.match(r"\d{8}-\d{6}\.\d{3}", f.split("_")[0]) ] + logger.debug( + f"Log files after filtering out logs that dont have valid date prefix and {log_file_name}: {log_files}" + ) # Unless another date is given, only look at today's logs batch_resume_date = datetime.now().date() @@ -263,9 +269,13 @@ def get_latest_log_file(task_id: str, task_type: str) -> str | None: if datetime.strptime(f.split("_")[0], "%Y%m%d-%H%M%S.%f").date() == batch_resume_date ] + logger.debug( + f"Log files after filtering out logs that dont match date: {log_files}" + ) # Sort the list by the date/time in the filename log_files.sort(key=lambda x: datetime.strptime(x.split("_")[0], "%Y%m%d-%H%M%S.%f")) + logger.debug(f"Log files after sorting: {log_files}") # Get the latest log file if log_files: log_file_name = f"{os.path.dirname(log_file_name)}/{log_files[-1]}" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..1fd3e1c --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,19 @@ +# monkey patch lovely pytest docker plugin +# lovely-pytest-docker is no longer maintained, so we need to monkey patch it +# to use docker compose instead as docker-compose v1 is no longer part of the +# GitHub Actions runner +from lovely.pytest.docker.compose import DockerComposeExecutor, execute + + +def docker_compose_executor(self, *subcommand): + command = ["docker", "compose", "--project-directory", self.project_directory] + for compose_file in self._compose_files: # pylint: disable=protected-access + command.append("-f") + command.append(compose_file) + command.append("-p") + command.append(self._project_name) # pylint: disable=protected-access + command += subcommand + return execute(command) + + +DockerComposeExecutor.execute = docker_compose_executor diff --git a/tests/test_logging.py b/tests/test_logging.py index 170f591..1dc4d15 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -3,11 +3,12 @@ import logging import os import re +import shutil import time from datetime import datetime import opentaskpy.otflogging -from tests.fixtures.ssh_clients import * # noqa: F403 +from tests.fixtures.ssh_clients import * # noqa: F403, F401 def test_define_log_file_name(env_vars, tmpdir, top_level_root_dir): @@ -120,9 +121,14 @@ def test_get_latest_log_file(env_vars): # Setup some dummy log files log_path = "test/testLogs" os.environ["OTF_LOG_DIRECTORY"] = log_path + # Set debug logging + opentaskpy.otflogging.logger.setLevel(logging.DEBUG) last_created_file = None + # Empty the log directory + shutil.rmtree(log_path, ignore_errors=True) + # loop 1 to 10 for _ in range(1, 11): log_file_name = opentaskpy.otflogging._define_log_file_name(None, "B") @@ -146,14 +152,23 @@ def test_get_latest_log_file(env_vars): # Rename the last created file to remove the _running suffix os.rename(last_created_file, last_created_file.replace("_running", "")) last_created_file = last_created_file.replace("_running", "") + + # Check the rename worked + assert os.path.exists(last_created_file) + # Run the function again and validate that it still returns nothing assert opentaskpy.otflogging.get_latest_log_file(None, "B") is None # Rename this file to _failed - os.rename(last_created_file, last_created_file.replace("_B", "_B_failed")) - last_created_file = last_created_file.replace("_B", "_B_failed") + failed_file = last_created_file.replace("_B", "_B_failed") + os.rename(last_created_file, failed_file) + # Check the rename worked + assert os.path.exists(failed_file) + # Run the function again and validate that it returns this file - assert opentaskpy.otflogging.get_latest_log_file(None, "B") == last_created_file + print("Testing finding a failed log to resume from") + print(f"Expect to find {failed_file}") + assert opentaskpy.otflogging.get_latest_log_file(None, "B") == failed_file # Create a new file that has succeeded, make sure it still returns None, as the latest # state is success