From 6c97b250fc450f06de13b1e5f1cac38dff820eb9 Mon Sep 17 00:00:00 2001 From: Russ Allbery Date: Fri, 26 Jul 2024 16:54:11 -0700 Subject: [PATCH 1/4] Add UWS support to FastAPI Safir app template Add a new configuration option for the FastAPI Safir app project template that allows the user to indicate whether they are building a UWS app with a separate backend and a queuing system (defaulting to no). If that option is enabled, add in the machinery for a UWS application, including stub models and a stub worker function. This template uncovered a Ruff diagnostic for unused class method parameters that would fire on the initial application as generated from the template. Add an ignore rule for that diagnostic for the same reason as the similar diagnostic for unused method arguments. --- .../fastapi_safir_app/SConscript | 11 ++ .../fastapi_safir_app/cookiecutter.json | 1 + .../example-uws/.dockerignore | 142 ++++++++++++++++++ .../example-uws/.github/dependabot.yml | 11 ++ .../example-uws/.github/workflows/ci.yaml | 87 +++++++++++ .../.github/workflows/periodic-ci.yaml | 57 +++++++ .../fastapi_safir_app/example-uws/.gitignore | 137 +++++++++++++++++ .../example-uws/.pre-commit-config.yaml | 14 ++ .../example-uws/CHANGELOG.md | 8 + .../fastapi_safir_app/example-uws/Dockerfile | 68 +++++++++ .../example-uws/Dockerfile.worker | 28 ++++ .../fastapi_safir_app/example-uws/LICENSE | 21 +++ .../fastapi_safir_app/example-uws/MANIFEST.in | 0 .../fastapi_safir_app/example-uws/Makefile | 48 ++++++ .../fastapi_safir_app/example-uws/README.md | 6 + .../changelog.d/_template.md.jinja | 7 + .../example-uws/pyproject.toml | 112 ++++++++++++++ .../example-uws/requirements/dev.in | 23 +++ .../example-uws/requirements/main.in | 18 +++ .../example-uws/requirements/tox.in | 15 ++ .../example-uws/ruff-shared.toml | 126 ++++++++++++++++ .../scripts/install-base-packages.sh | 34 +++++ .../scripts/install-dependency-packages.sh | 34 +++++ .../scripts/install-worker-packages.sh | 23 +++ .../example-uws/scripts/install-worker.sh | 25 +++ .../example-uws/scripts/start-worker.sh | 18 +++ .../example-uws/src/exampleuws/__init__.py | 14 ++ .../example-uws/src/exampleuws/cli.py | 42 ++++++ .../example-uws/src/exampleuws/config.py | 55 +++++++ .../src/exampleuws/dependencies.py | 28 ++++ .../example-uws/src/exampleuws/domain.py | 15 ++ .../src/exampleuws/handlers/__init__.py | 0 .../src/exampleuws/handlers/external.py | 52 +++++++ .../src/exampleuws/handlers/internal.py | 42 ++++++ .../example-uws/src/exampleuws/main.py | 67 +++++++++ .../example-uws/src/exampleuws/models.py | 41 +++++ .../src/exampleuws/workers/__init__.py | 0 .../src/exampleuws/workers/exampleuws.py | 97 ++++++++++++ .../example-uws/src/exampleuws/workers/uws.py | 19 +++ .../example-uws/tests/__init__.py | 0 .../example-uws/tests/conftest.py | 63 ++++++++ .../example-uws/tests/handlers/__init__.py | 0 .../tests/handlers/external_test.py | 22 +++ .../tests/handlers/internal_test.py | 21 +++ .../fastapi_safir_app/example-uws/tox.ini | 63 ++++++++ .../example/ruff-shared.toml | 1 + .../example/src/example/main.py | 2 +- .../hooks/post_gen_project.py | 14 ++ .../fastapi_safir_app/templatekit.yaml | 4 + .../.github/workflows/ci.yaml | 8 + .../{{cookiecutter.name}}/Dockerfile.worker | 29 ++++ .../requirements/main.in | 4 + .../{{cookiecutter.name}}/requirements/tox.in | 3 + .../{{cookiecutter.name}}/ruff-shared.toml | 1 + .../scripts/install-worker-packages.sh | 24 +++ .../scripts/install-worker.sh | 26 ++++ .../scripts/start-worker.sh | 19 +++ .../src/{{cookiecutter.module_name}}/cli.py | 43 ++++++ .../{{cookiecutter.module_name}}/config.py | 30 +++- .../dependencies.py | 29 ++++ .../{{cookiecutter.module_name}}/domain.py | 16 ++ .../src/{{cookiecutter.module_name}}/main.py | 19 ++- .../{{cookiecutter.module_name}}/models.py | 27 ++++ .../workers/__init__.py | 0 .../workers/uws.py | 20 +++ .../workers/{{cookiecutter.module_name}}.py | 99 ++++++++++++ .../{{cookiecutter.name}}/tests/conftest.py | 51 ++++++- .../{{cookiecutter.name}}/tox.ini | 31 ++++ .../example/ruff-shared.toml | 1 + .../{{cookiecutter.name}}/ruff-shared.toml | 1 + .../technote_md/testn-000/technote.toml | 2 +- .../technote_rst/testn-000/technote.toml | 2 +- 72 files changed, 2213 insertions(+), 8 deletions(-) create mode 100644 project_templates/fastapi_safir_app/example-uws/.dockerignore create mode 100644 project_templates/fastapi_safir_app/example-uws/.github/dependabot.yml create mode 100644 project_templates/fastapi_safir_app/example-uws/.github/workflows/ci.yaml create mode 100644 project_templates/fastapi_safir_app/example-uws/.github/workflows/periodic-ci.yaml create mode 100644 project_templates/fastapi_safir_app/example-uws/.gitignore create mode 100644 project_templates/fastapi_safir_app/example-uws/.pre-commit-config.yaml create mode 100644 project_templates/fastapi_safir_app/example-uws/CHANGELOG.md create mode 100644 project_templates/fastapi_safir_app/example-uws/Dockerfile create mode 100644 project_templates/fastapi_safir_app/example-uws/Dockerfile.worker create mode 100644 project_templates/fastapi_safir_app/example-uws/LICENSE create mode 100644 project_templates/fastapi_safir_app/example-uws/MANIFEST.in create mode 100644 project_templates/fastapi_safir_app/example-uws/Makefile create mode 100644 project_templates/fastapi_safir_app/example-uws/README.md create mode 100644 project_templates/fastapi_safir_app/example-uws/changelog.d/_template.md.jinja create mode 100644 project_templates/fastapi_safir_app/example-uws/pyproject.toml create mode 100644 project_templates/fastapi_safir_app/example-uws/requirements/dev.in create mode 100644 project_templates/fastapi_safir_app/example-uws/requirements/main.in create mode 100644 project_templates/fastapi_safir_app/example-uws/requirements/tox.in create mode 100644 project_templates/fastapi_safir_app/example-uws/ruff-shared.toml create mode 100755 project_templates/fastapi_safir_app/example-uws/scripts/install-base-packages.sh create mode 100755 project_templates/fastapi_safir_app/example-uws/scripts/install-dependency-packages.sh create mode 100644 project_templates/fastapi_safir_app/example-uws/scripts/install-worker-packages.sh create mode 100644 project_templates/fastapi_safir_app/example-uws/scripts/install-worker.sh create mode 100644 project_templates/fastapi_safir_app/example-uws/scripts/start-worker.sh create mode 100644 project_templates/fastapi_safir_app/example-uws/src/exampleuws/__init__.py create mode 100644 project_templates/fastapi_safir_app/example-uws/src/exampleuws/cli.py create mode 100644 project_templates/fastapi_safir_app/example-uws/src/exampleuws/config.py create mode 100644 project_templates/fastapi_safir_app/example-uws/src/exampleuws/dependencies.py create mode 100644 project_templates/fastapi_safir_app/example-uws/src/exampleuws/domain.py create mode 100644 project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/__init__.py create mode 100644 project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/external.py create mode 100644 project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/internal.py create mode 100644 project_templates/fastapi_safir_app/example-uws/src/exampleuws/main.py create mode 100644 project_templates/fastapi_safir_app/example-uws/src/exampleuws/models.py create mode 100644 project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/__init__.py create mode 100644 project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/exampleuws.py create mode 100644 project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/uws.py create mode 100644 project_templates/fastapi_safir_app/example-uws/tests/__init__.py create mode 100644 project_templates/fastapi_safir_app/example-uws/tests/conftest.py create mode 100644 project_templates/fastapi_safir_app/example-uws/tests/handlers/__init__.py create mode 100644 project_templates/fastapi_safir_app/example-uws/tests/handlers/external_test.py create mode 100644 project_templates/fastapi_safir_app/example-uws/tests/handlers/internal_test.py create mode 100644 project_templates/fastapi_safir_app/example-uws/tox.ini create mode 100644 project_templates/fastapi_safir_app/{{cookiecutter.name}}/Dockerfile.worker create mode 100644 project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker-packages.sh create mode 100644 project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh create mode 100644 project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/start-worker.sh create mode 100644 project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/cli.py create mode 100644 project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/dependencies.py create mode 100644 project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/domain.py create mode 100644 project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/__init__.py create mode 100644 project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/uws.py create mode 100644 project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/{{cookiecutter.module_name}}.py diff --git a/project_templates/fastapi_safir_app/SConscript b/project_templates/fastapi_safir_app/SConscript index 318c4952..2d1592cb 100644 --- a/project_templates/fastapi_safir_app/SConscript +++ b/project_templates/fastapi_safir_app/SConscript @@ -6,3 +6,14 @@ from templatekit.builder import cookiecutter_project_builder env = Environment(BUILDERS={'Cookiecutter': cookiecutter_project_builder}) env.Cookiecutter(AlwaysBuild(Dir('example')), 'cookiecutter.json') + +# Run cookiecutter to generate the 'example-uws' package +# This represents an IVOA UWS service that does work via an arq queue +# The explicit target directory, with AlwaysBuild, is needed for Scons to +# differentiate this build from the example build above. We can't use Scons' +# default build caching because we'd have to compute the cookiecutter +# template anyways. +env.Cookiecutter(AlwaysBuild(Dir('example-uws')), + 'cookiecutter.json', + cookiecutter_context={'name': 'example-uws', + 'uws_service': 'True'}) diff --git a/project_templates/fastapi_safir_app/cookiecutter.json b/project_templates/fastapi_safir_app/cookiecutter.json index 6e36597e..85143b1e 100644 --- a/project_templates/fastapi_safir_app/cookiecutter.json +++ b/project_templates/fastapi_safir_app/cookiecutter.json @@ -16,5 +16,6 @@ "lsst-dm", "lsst-sqre-testing" ], + "uws_service": ["False", "True"], "_extensions": ["jinja2_time.TimeExtension", "templatekit.TemplatekitExtension"] } diff --git a/project_templates/fastapi_safir_app/example-uws/.dockerignore b/project_templates/fastapi_safir_app/example-uws/.dockerignore new file mode 100644 index 00000000..840aa286 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/.dockerignore @@ -0,0 +1,142 @@ +# VSCode +.vscode/ + +# Everything below this point is a copy of .gitignore. + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# macOS +.DS_Store diff --git a/project_templates/fastapi_safir_app/example-uws/.github/dependabot.yml b/project_templates/fastapi_safir_app/example-uws/.github/dependabot.yml new file mode 100644 index 00000000..dfb90b72 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" diff --git a/project_templates/fastapi_safir_app/example-uws/.github/workflows/ci.yaml b/project_templates/fastapi_safir_app/example-uws/.github/workflows/ci.yaml new file mode 100644 index 00000000..ab34a2db --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/.github/workflows/ci.yaml @@ -0,0 +1,87 @@ +name: CI + +env: + # Current supported Python version. For applications, there is generally no + # reason to support multiple Python versions, so all actions are run with + # this version. Quote the version to avoid interpretation as a floating + # point number. + PYTHON_VERSION: "3.12" + +"on": + merge_group: {} + pull_request: {} + push: + branches-ignore: + # These should always correspond to pull requests, so ignore them for + # the push trigger and let them be triggered by the pull_request + # trigger, avoiding running the workflow twice. This is a minor + # optimization so there's no need to ensure this is comprehensive. + - "dependabot/**" + - "gh-readonly-queue/**" + - "renovate/**" + - "tickets/**" + - "u/**" + tags: + - "*" + +jobs: + lint: + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 + + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v4 + + - name: Run tox + uses: lsst-sqre/run-tox@v1 + with: + python-version: ${{ env.PYTHON_VERSION }} + tox-envs: "py,coverage-report,typing" + tox-requirements: requirements/tox.txt + + build: + runs-on: ubuntu-latest + needs: [lint, test] + timeout-minutes: 10 + + # Only do Docker builds of tagged releases and pull requests from ticket + # branches. This will still trigger on pull requests from untrusted + # repositories whose branch names match our tickets/* branch convention, + # but in this case the build will fail with an error since the secret + # won't be set. + if: > + github.event_name != 'merge_group' + && (startsWith(github.ref, 'refs/tags/') + || startsWith(github.head_ref, 'tickets/')) + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: lsst-sqre/build-and-push-to-ghcr@v1 + id: build + with: + image: ${{ github.repository }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - uses: lsst-sqre/build-and-push-to-ghcr@v1 + id: build-worker + with: + image: ${{ github.repository }}-worker + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/project_templates/fastapi_safir_app/example-uws/.github/workflows/periodic-ci.yaml b/project_templates/fastapi_safir_app/example-uws/.github/workflows/periodic-ci.yaml new file mode 100644 index 00000000..65dd8b4a --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/.github/workflows/periodic-ci.yaml @@ -0,0 +1,57 @@ +# This is a separate run of the Python test suite that runs from a schedule, +# doesn't cache the tox environment, and updates pinned dependencies first. +# The purpose is to test compatibility with the latest versions of +# dependencies. + +name: Periodic CI + +env: + # Current supported Python version. For applications, there is generally no + # reason to support multiple Python versions, so all actions are run with + # this version. Quote the version to avoid interpretation as a floating + # point number. + PYTHON_VERSION: "3.12" + +"on": + schedule: + - cron: "0 12 * * 1" + workflow_dispatch: {} + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Update dependencies + run: | + pip install --upgrade uv + uv venv + source .venv/bin/activate + make update-deps + shell: bash + + - name: Run tests in tox + uses: lsst-sqre/run-tox@v1 + with: + python-version: ${{ env.PYTHON_VERSION }} + tox-envs: "lint,typing,py" + tox-requirements: requirements/tox.txt + use-cache: false + + - name: Report status + if: failure() + uses: ravsamhq/notify-slack-action@v2 + with: + status: ${{ job.status }} + notify_when: "failure" + notification_title: "Periodic test for {repo} failed" + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }} diff --git a/project_templates/fastapi_safir_app/example-uws/.gitignore b/project_templates/fastapi_safir_app/example-uws/.gitignore new file mode 100644 index 00000000..7af665ba --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/.gitignore @@ -0,0 +1,137 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# macOS +.DS_Store diff --git a/project_templates/fastapi_safir_app/example-uws/.pre-commit-config.yaml b/project_templates/fastapi_safir_app/example-uws/.pre-commit-config.yaml new file mode 100644 index 00000000..d302149e --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/.pre-commit-config.yaml @@ -0,0 +1,14 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-toml + - id: check-yaml + - id: trailing-whitespace + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.5 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format diff --git a/project_templates/fastapi_safir_app/example-uws/CHANGELOG.md b/project_templates/fastapi_safir_app/example-uws/CHANGELOG.md new file mode 100644 index 00000000..0490886f --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/CHANGELOG.md @@ -0,0 +1,8 @@ +# Change log + +example-uws is versioned with [semver](https://semver.org/). +Dependencies are updated to the latest available version during each release, and aren't noted here. + +Find changes for the upcoming release in the project's [changelog.d directory](https://github.com/lsst-sqre/example-uws/tree/main/changelog.d/). + + diff --git a/project_templates/fastapi_safir_app/example-uws/Dockerfile b/project_templates/fastapi_safir_app/example-uws/Dockerfile new file mode 100644 index 00000000..48366262 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/Dockerfile @@ -0,0 +1,68 @@ +# This Dockerfile has four stages: +# +# base-image +# Updates the base Python image with security patches and common system +# packages. This image becomes the base of all other images. +# dependencies-image +# Installs third-party dependencies (requirements/main.txt) into a virtual +# environment. This virtual environment is ideal for copying across build +# stages. +# install-image +# Installs the app into the virtual environment. +# runtime-image +# - Copies the virtual environment into place. +# - Runs a non-root user. +# - Sets up the entrypoint and port. + +FROM python:3.12.2-slim-bookworm as base-image + +# Update system packages +COPY scripts/install-base-packages.sh . +RUN ./install-base-packages.sh && rm ./install-base-packages.sh + +FROM base-image AS dependencies-image + +# Install system packages only needed for building dependencies. +COPY scripts/install-dependency-packages.sh . +RUN ./install-dependency-packages.sh + +# Create a Python virtual environment +ENV VIRTUAL_ENV=/opt/venv +RUN python -m venv $VIRTUAL_ENV +# Make sure we use the virtualenv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +# Put the latest pip and setuptools in the virtualenv +RUN pip install --upgrade --no-cache-dir pip setuptools wheel + +# Install the app's Python runtime dependencies +COPY requirements/main.txt ./requirements.txt +RUN pip install --quiet --no-cache-dir -r requirements.txt + +FROM dependencies-image AS install-image + +# Use the virtualenv +ENV PATH="/opt/venv/bin:$PATH" + +COPY . /workdir +WORKDIR /workdir +RUN pip install --no-cache-dir . + +FROM base-image AS runtime-image + +# Create a non-root user +RUN useradd --create-home appuser + +# Copy the virtualenv +COPY --from=install-image /opt/venv /opt/venv + +# Make sure we use the virtualenv +ENV PATH="/opt/venv/bin:$PATH" + +# Switch to the non-root user. +USER appuser + +# Expose the port. +EXPOSE 8080 + +# Run the application. +CMD ["uvicorn", "exampleuws.main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/project_templates/fastapi_safir_app/example-uws/Dockerfile.worker b/project_templates/fastapi_safir_app/example-uws/Dockerfile.worker new file mode 100644 index 00000000..2edae678 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/Dockerfile.worker @@ -0,0 +1,28 @@ + +# This Dockerfile constructs the image for backend workers. These images +# are based on stack containers and install any required supporting code +# for the backend, arq, and the backend worker definition. + +FROM lsstsqre/centos:7-stack-lsst_distrib-w_2024_27 + +# Reset the user to root since we need to do system install tasks. +USER root + +# Upgrade the system packages. +COPY scripts/install-worker-packages.sh . +RUN ./install-worker-packages.sh && rm ./install-worker-packages.sh + +# Install the necessary prerequisites and the vo-cutouts code. +COPY . /workdir +RUN /workdir/scripts/install-worker.sh /workdir && rm -r /workdir +COPY scripts/start-worker.sh / + +# Create a non-root user +RUN useradd --create-home appuser + +# Switch to the non-root user. +USER appuser + +# Start the arq worker. +WORKDIR / +CMD ["/start-worker.sh"] diff --git a/project_templates/fastapi_safir_app/example-uws/LICENSE b/project_templates/fastapi_safir_app/example-uws/LICENSE new file mode 100644 index 00000000..b7acc140 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Association of Universities for Research in Astronomy, Inc. (AURA) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/project_templates/fastapi_safir_app/example-uws/MANIFEST.in b/project_templates/fastapi_safir_app/example-uws/MANIFEST.in new file mode 100644 index 00000000..e69de29b diff --git a/project_templates/fastapi_safir_app/example-uws/Makefile b/project_templates/fastapi_safir_app/example-uws/Makefile new file mode 100644 index 00000000..dbcb745c --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/Makefile @@ -0,0 +1,48 @@ +.PHONY: help +help: + @echo "Make targets for example-uws" + @echo "make init - Set up dev environment" + @echo "make run - Start a local development instance" + @echo "make update - Update pinned dependencies and run make init" + @echo "make update-deps - Update pinned dependencies" + @echo "make update-deps-no-hashes - Pin dependencies without hashes" + +.PHONY: init +init: + pip install --upgrade uv + uv pip install -r requirements/main.txt -r requirements/dev.txt \ + -r requirements/tox.txt + uv pip install --editable . + rm -rf .tox + uv pip install --upgrade pre-commit + pre-commit install + +.PHONY: run +run: + tox run -e run + +.PHONY: update +update: update-deps init + +.PHONY: update-deps +update-deps: + pip install --upgrade uv + uv pip install --upgrade pre-commit + pre-commit autoupdate + uv pip compile --upgrade --generate-hashes \ + --output-file requirements/main.txt requirements/main.in + uv pip compile --upgrade --generate-hashes \ + --output-file requirements/dev.txt requirements/dev.in + uv pip compile --upgrade --generate-hashes \ + --output-file requirements/tox.txt requirements/tox.in + +# Useful for testing against a Git version of Safir. +.PHONY: update-deps-no-hashes +update-deps-no-hashes: + pip install --upgrade uv + uv pip compile --upgrade \ + --output-file requirements/main.txt requirements/main.in + uv pip compile --upgrade \ + --output-file requirements/dev.txt requirements/dev.in + uv pip compile --upgrade \ + --output-file requirements/tox.txt requirements/tox.in diff --git a/project_templates/fastapi_safir_app/example-uws/README.md b/project_templates/fastapi_safir_app/example-uws/README.md new file mode 100644 index 00000000..8f514585 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/README.md @@ -0,0 +1,6 @@ +# example-uws + +Short one-sentence summary of the app +Learn more at https://example-uws.lsst.io + +example-uws is developed with [FastAPI](https://fastapi.tiangolo.com) and [Safir](https://safir.lsst.io). diff --git a/project_templates/fastapi_safir_app/example-uws/changelog.d/_template.md.jinja b/project_templates/fastapi_safir_app/example-uws/changelog.d/_template.md.jinja new file mode 100644 index 00000000..6e644b85 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/changelog.d/_template.md.jinja @@ -0,0 +1,7 @@ + +{%- for cat in config.categories %} + +### {{ cat }} + +- +{%- endfor %} diff --git a/project_templates/fastapi_safir_app/example-uws/pyproject.toml b/project_templates/fastapi_safir_app/example-uws/pyproject.toml new file mode 100644 index 00000000..52b5f14d --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/pyproject.toml @@ -0,0 +1,112 @@ +[project] +# https://packaging.python.org/en/latest/specifications/declaring-project-metadata/ +name = "example-uws" +description = "Short one-sentence summary of the app" +license = { file = "LICENSE" } +readme = "README.md" +keywords = ["rubin", "lsst"] +# https://pypi.org/classifiers/ +classifiers = [ + "Development Status :: 4 - Beta", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Intended Audience :: Developers", + "Natural Language :: English", + "Operating System :: POSIX", + "Typing :: Typed", +] +requires-python = ">=3.12" +# Use requirements/main.in for runtime dependencies instead. +dependencies = [] +dynamic = ["version"] + +[project.scripts] +example-uws = "exampleuws.cli:main" + +[project.urls] +Homepage = "https://example-uws.lsst.io" +Source = "https://github.com/lsst-sqre/example-uws" + +[build-system] +requires = ["setuptools>=61", "wheel", "setuptools_scm[toml]>=6.2"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] + +[tool.coverage.run] +parallel = true +branch = true +source = ["exampleuws"] + +[tool.coverage.paths] +source = ["src", ".tox/*/site-packages"] + +[tool.coverage.report] +show_missing = true +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "if self.debug:", + "if settings.DEBUG", + "raise AssertionError", + "raise NotImplementedError", + "if 0:", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.mypy] +disallow_untyped_defs = true +disallow_incomplete_defs = true +ignore_missing_imports = true +local_partial_types = true +plugins = ["pydantic.mypy"] +no_implicit_reexport = true +show_error_codes = true +strict_equality = true +warn_redundant_casts = true +warn_unreachable = true +warn_unused_ignores = true + +[tool.pydantic-mypy] +init_forbid_extra = true +init_typed = true +warn_required_dynamic_aliases = true +warn_untyped_fields = true + +[tool.pytest.ini_options] +asyncio_mode = "strict" +# The python_files setting is not for test detection (pytest will pick up any +# test files named *_test.py without this setting) but to enable special +# assert processing in any non-test supporting files under tests. We +# conventionally put test support functions under tests.support and may +# sometimes use assert in test fixtures in conftest.py, and pytest only +# enables magical assert processing (showing a full diff on assert failures +# with complex data structures rather than only the assert message) in files +# listed in python_files. +python_files = ["tests/*.py", "tests/*/*.py"] + +# Use the generic Ruff configuration in ruff.toml and extend it with only +# project-specific settings. Add a [tool.ruff.lint.extend-per-file-ignores] +# section for project-specific ignore rules. +[tool.ruff] +extend = "ruff-shared.toml" + +[tool.ruff.lint.isort] +known-first-party = ["exampleuws", "tests"] +split-on-trailing-comma = false + +[tool.scriv] +categories = [ + "Backwards-incompatible changes", + "New features", + "Bug fixes", + "Other changes", +] +entry_title_template = "{{ version }} ({{ date.strftime('%Y-%m-%d') }})" +format = "md" +md_header_level = "2" +new_fragment_template = "file:changelog.d/_template.md.jinja" +skip_fragments = "_template.md.jinja" diff --git a/project_templates/fastapi_safir_app/example-uws/requirements/dev.in b/project_templates/fastapi_safir_app/example-uws/requirements/dev.in new file mode 100644 index 00000000..5d8619b7 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/requirements/dev.in @@ -0,0 +1,23 @@ +# -*- conf -*- +# +# Editable development dependencies +# Add direct development, test, and documentation dependencies here, as well +# as implicit dev dependencies with constrained versions. +# +# After editing, update requirements/dev.txt by running: +# make update-deps + +-c main.txt + +# Testing +asgi-lifespan +coverage[toml] +httpx +mypy +pydantic +pytest +pytest-asyncio +pytest-cov + +# Documentation +scriv diff --git a/project_templates/fastapi_safir_app/example-uws/requirements/main.in b/project_templates/fastapi_safir_app/example-uws/requirements/main.in new file mode 100644 index 00000000..667f7e8f --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/requirements/main.in @@ -0,0 +1,18 @@ +# -*- conf -*- +# +# Editable runtime dependencies (equivalent to install_requires) +# Add direct runtime dependencies here, as well as implicit dependencies +# with constrained versions. +# +# After editing, update requirements/main.txt by running: +# make update-deps + +# These dependencies are for fastapi including some optional features. +fastapi +starlette +uvicorn[standard] + +# Other dependencies. +pydantic +pydantic-settings +safir[uws]>=7 diff --git a/project_templates/fastapi_safir_app/example-uws/requirements/tox.in b/project_templates/fastapi_safir_app/example-uws/requirements/tox.in new file mode 100644 index 00000000..c525b25b --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/requirements/tox.in @@ -0,0 +1,15 @@ +# -*- conf -*- +# +# Editable tox dependencies +# Add tox and its plugins here. These will be installed in the user's venv for +# local development and by CI when running tox actions. +# +# After editing, update requirements/dev.txt by running: +# make update-deps + +-c main.txt +-c dev.txt + +tox +tox-docker +tox-uv diff --git a/project_templates/fastapi_safir_app/example-uws/ruff-shared.toml b/project_templates/fastapi_safir_app/example-uws/ruff-shared.toml new file mode 100644 index 00000000..da644fb3 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/ruff-shared.toml @@ -0,0 +1,126 @@ +# Generic shared Ruff configuration file. It should be possible to use this +# file unmodified in different packages provided that one likes the style that +# it enforces. +# +# This file should be used from pyproject.toml as follows: +# +# [tool.ruff] +# extend = "ruff-shared.toml" +# +# It can then be extended with project-specific rules. A common additional +# setting in pyproject.toml is tool.ruff.lint.extend-per-file-ignores, to add +# additional project-specific ignore rules for specific paths. +# +# The rule used with Ruff configuration is to disable every non-deprecated +# lint rule that has legitimate exceptions that are not dodgy code, rather +# than cluttering code with noqa markers. This is therefore a reiatively +# relaxed configuration that errs on the side of disabling legitimate rules. +# +# Reference for settings: https://docs.astral.sh/ruff/settings/ +# Reference for rules: https://docs.astral.sh/ruff/rules/ +exclude = ["docs/**"] +line-length = 79 +target-version = "py312" + +[format] +docstring-code-format = true + +[lint] +ignore = [ + "ANN401", # sometimes Any is the right type + "ARG001", # unused function arguments are often legitimate + "ARG002", # unused method arguments are often legitimate + "ARG003", # unused class method arguments are often legitimate + "ARG005", # unused lambda arguments are often legitimate + "BLE001", # we want to catch and report Exception in background tasks + "C414", # nested sorted is how you sort by multiple keys with reverse + "D102", # sometimes we use docstring inheritence + "D104", # don't see the point of documenting every package + "D105", # our style doesn't require docstrings for magic methods + "D106", # Pydantic uses a nested Config class that doesn't warrant docs + "D205", # our documentation style allows a folded first line + "EM101", # justification (duplicate string in traceback) is silly + "EM102", # justification (duplicate string in traceback) is silly + "FBT003", # positional booleans are normal for Pydantic field defaults + "FIX002", # point of a TODO comment is that we're not ready to fix it + "G004", # forbidding logging f-strings is appealing, but not our style + "RET505", # disagree that omitting else always makes code more readable + "PLR0911", # often many returns is clearer and simpler style + "PLR0913", # factory pattern uses constructors with many arguments + "PLR2004", # too aggressive about magic values + "PLW0603", # yes global is discouraged but if needed, it's needed + "S105", # good idea but too many false positives on non-passwords + "S106", # good idea but too many false positives on non-passwords + "S107", # good idea but too many false positives on non-passwords + "S603", # not going to manually mark every subprocess call as reviewed + "S607", # using PATH is not a security vulnerability + "SIM102", # sometimes the formatting of nested if statements is clearer + "SIM117", # sometimes nested with contexts are clearer + "TCH001", # we decided to not maintain separate TYPE_CHECKING blocks + "TCH002", # we decided to not maintain separate TYPE_CHECKING blocks + "TCH003", # we decided to not maintain separate TYPE_CHECKING blocks + "TD003", # we don't require issues be created for TODOs + "TID252", # if we're going to use relative imports, use them always + "TRY003", # good general advice but lint is way too aggressive + "TRY301", # sometimes raising exceptions inside try is the best flow + "UP040", # PEP 695 type aliases not yet supported by mypy + + # The following settings should be disabled when using ruff format + # per https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules + "W191", + "E111", + "E114", + "E117", + "D206", + "D300", + "Q000", + "Q001", + "Q002", + "Q003", + "COM812", + "COM819", + "ISC001", + "ISC002", + + # Temporary bug workarounds. + "S113", # https://github.com/astral-sh/ruff/issues/12210 +] +select = ["ALL"] + +[lint.per-file-ignores] +"src/*/handlers/**" = [ + "D103", # FastAPI handlers should not have docstrings +] +"tests/**" = [ + "C901", # tests are allowed to be complex, sometimes that's convenient + "D101", # tests don't need docstrings + "D103", # tests don't need docstrings + "PLR0915", # tests are allowed to be long, sometimes that's convenient + "PT012", # way too aggressive about limiting pytest.raises blocks + "S101", # tests should use assert + "S106", # tests are allowed to hard-code dummy passwords + "S301", # allow tests for whether code can be pickled + "SLF001", # tests are allowed to access private members +] + +# These are too useful as attributes or methods to allow the conflict with the +# built-in to rule out their use. +[lint.flake8-builtins] +builtins-ignorelist = [ + "all", + "any", + "help", + "id", + "list", + "type", +] + +[lint.flake8-pytest-style] +fixture-parentheses = false +mark-parentheses = false + +[lint.mccabe] +max-complexity = 11 + +[lint.pydocstyle] +convention = "numpy" diff --git a/project_templates/fastapi_safir_app/example-uws/scripts/install-base-packages.sh b/project_templates/fastapi_safir_app/example-uws/scripts/install-base-packages.sh new file mode 100755 index 00000000..620781c1 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/scripts/install-base-packages.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# This script updates packages in the base Docker image that's used by both the +# build and runtime images, and gives us a place to install additional +# system-level packages with apt-get. +# +# Based on the blog post: +# https://pythonspeed.com/articles/system-packages-docker/ + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for +# details. +set -euo pipefail + +# Display each command as it's run. +set -x + +# Tell apt-get we're never going to be able to give manual +# feedback: +export DEBIAN_FRONTEND=noninteractive + +# Update the package listing, so we know what packages exist: +apt-get update + +# Install security updates: +apt-get -y upgrade + +# Example of installing a new package, without unnecessary packages: +apt-get -y install --no-install-recommends git + +# Delete cached files we don't need anymore: +apt-get clean +rm -rf /var/lib/apt/lists/* diff --git a/project_templates/fastapi_safir_app/example-uws/scripts/install-dependency-packages.sh b/project_templates/fastapi_safir_app/example-uws/scripts/install-dependency-packages.sh new file mode 100755 index 00000000..f63ef751 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/scripts/install-dependency-packages.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# This script installs additional packages used by the dependency image but +# not needed by the runtime image, such as additional packages required to +# build Python dependencies. +# +# Since the base image wipes all the apt caches to clean up the image that +# will be reused by the runtime image, we unfortunately have to do another +# apt-get update here, which wastes some time and network. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for +# details. +set -euo pipefail + +# Display each command as it's run. +set -x + +# Tell apt-get we're never going to be able to give manual +# feedback: +export DEBIAN_FRONTEND=noninteractive + +# Update the package listing, so we know what packages exist: +apt-get update + +# Install build-essential because sometimes Python dependencies need to build +# C modules, particularly when upgrading to newer Python versions. libffi-dev +# is sometimes needed to build cffi (a cryptography dependency). +apt-get -y install --no-install-recommends build-essential libffi-dev + +# Delete cached files we don't need anymore: +apt-get clean +rm -rf /var/lib/apt/lists/* diff --git a/project_templates/fastapi_safir_app/example-uws/scripts/install-worker-packages.sh b/project_templates/fastapi_safir_app/example-uws/scripts/install-worker-packages.sh new file mode 100644 index 00000000..2cec6fb0 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/scripts/install-worker-packages.sh @@ -0,0 +1,23 @@ + +#!/bin/bash + +# Install or upgrade any operating system packages needed on worker images. +# This is done in a separate script to create a separate cached Docker image, +# which will help with iteration speed on the more interesting setup actions +# taken later in the build. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for details. +set -euo pipefail + +# Display each command as it's run. +set -x + +# Upgrade the Red Hat packages. +# +# TODO(rra): Disabled for now because the version of CentOS used by the image +# is so old that the package repositories no longer exist. This will in theory +# soon be fixed by basing the image on AlmaLinux. +#yum -y upgrade +#yum clean all diff --git a/project_templates/fastapi_safir_app/example-uws/scripts/install-worker.sh b/project_templates/fastapi_safir_app/example-uws/scripts/install-worker.sh new file mode 100644 index 00000000..4109fd34 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/scripts/install-worker.sh @@ -0,0 +1,25 @@ + +#!/bin/bash + +# This script updates and installs the necessary prerequisites for a backend +# worker starting with a stack container. It takes one parameter, the +# directory in which to do the installation. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for details. +# set -u is omitted because the setup bash function does not support it. +set -eo pipefail + +# Enable the stack. This should be done before set -x because it will +# otherwise spew a bunch of nonsense no one cares about into the logs. +source /opt/lsst/software/stack/loadLSST.bash +setup lsst_distrib + +# Display each command as it's run. +set -x + +# Install Python dependencies and the example-uws code. +cd "$1" +pip install --no-cache-dir google-cloud-storage safir-arq +pip install --no-cache-dir --no-deps . diff --git a/project_templates/fastapi_safir_app/example-uws/scripts/start-worker.sh b/project_templates/fastapi_safir_app/example-uws/scripts/start-worker.sh new file mode 100644 index 00000000..14e44bd5 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/scripts/start-worker.sh @@ -0,0 +1,18 @@ + +#!/bin/bash + +# This script is installed in the worker image and starts the backend worker +# using arq. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for details. +# set -u is omitted because the setup bash function does not support it. +set -eo pipefail + +# Initialize the environment. +source /opt/lsst/software/stack/loadLSST.bash +setup lsst_distrib + +# Start arq with the worker. +arq exampleuws.workers.exampleuws.WorkerSettings diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/__init__.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/__init__.py new file mode 100644 index 00000000..922936c8 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/__init__.py @@ -0,0 +1,14 @@ +"""The example-uws service.""" + +__all__ = ["__version__"] + +from importlib.metadata import PackageNotFoundError, version + +__version__: str +"""The application version string (PEP 440 / SemVer compatible).""" + +try: + __version__ = version("example-uws") +except PackageNotFoundError: + # package is not installed + __version__ = "0.0.0" diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/cli.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/cli.py new file mode 100644 index 00000000..5c95ea76 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/cli.py @@ -0,0 +1,42 @@ + +"""Administrative command-line interface.""" + +from __future__ import annotations + +import click +import structlog +from safir.asyncio import run_with_asyncio +from safir.click import display_help + +from .config import uws + +__all__ = [ + "help", + "init", + "main", +] + + +@click.group(context_settings={"help_option_names": ["-h", "--help"]}) +@click.version_option(message="%(version)s") +def main() -> None: + """Administrative command-line interface for example-uws.""" + + +@main.command() +@click.argument("topic", default=None, required=False, nargs=1) +@click.pass_context +def help(ctx: click.Context, topic: str | None) -> None: + """Show help for any command.""" + display_help(main, ctx, topic) + + +@main.command() +@click.option( + "--reset", is_flag=True, help="Delete all existing database data." +) +@run_with_asyncio +async def init(*, reset: bool) -> None: + """Initialize the database storage.""" + logger = structlog.get_logger("exampleuws") + await uws.initialize_uws_database(logger, reset=reset) diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/config.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/config.py new file mode 100644 index 00000000..10726c01 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/config.py @@ -0,0 +1,55 @@ +"""Configuration definition.""" + +from __future__ import annotations + +from pydantic import Field +from pydantic_settings import SettingsConfigDict +from safir.logging import LogLevel, Profile +from safir.uws import UWSApplication, UWSAppSettings, UWSConfig, UWSRoute + +from .dependencies import post_params_dependency +from .models import ExampleuwsParameters + +__all__ = ["Config", "config"] + + +class Config(UWSAppSettings): + """Configuration for example-uws.""" + + name: str = Field("example-uws", title="Name of application") + + path_prefix: str = Field( + "/example-uws", title="URL prefix for application" + ) + + profile: Profile = Field( + Profile.development, title="Application logging profile" + ) + + log_level: LogLevel = Field( + LogLevel.INFO, title="Log level of the application's logger" + ) + + model_config = SettingsConfigDict( + env_prefix="EXAMPLE_UWS_", case_sensitive=False + ) + + @property + def uws_config(self) -> UWSConfig: + """Corresponding configuration for the UWS subsystem.""" + return self.build_uws_config( + parameters_type=ExampleuwsParameters, + worker="example_uws", + async_post_route=UWSRoute( + dependency=post_params_dependency, + summary="Create async example-uws job", + description="Create a new UWS job for example-uws", + ), + ) + + +config = Config() +"""Configuration for example-uws.""" + +uws = UWSApplication(config.uws_config) +"""The UWS application for this service.""" diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/dependencies.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/dependencies.py new file mode 100644 index 00000000..35bbf034 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/dependencies.py @@ -0,0 +1,28 @@ + +"""Job parameter dependencies.""" + +from typing import Annotated + +from fastapi import Depends, Request +from safir.uws import UWSJobParameter, uws_post_params_dependency + +__all__ = [ + "post_params_dependency", +] + + +async def post_params_dependency( + *, + # Add POST parameters here. All of them should be Form() parameters. + # Use str | None for single-valued attributes and str | list[str] | None + # for parameters that can be given more than one time. + params: Annotated[ + list[UWSJobParameter], Depends(uws_post_params_dependency) + ], +) -> list[UWSJobParameter]: + """Parse POST parameters into job parameters.""" + return [ + p + for p in params + if p.parameter_id in set() # Replace with set of parameter names + ] diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/domain.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/domain.py new file mode 100644 index 00000000..5c17e073 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/domain.py @@ -0,0 +1,15 @@ + +"""Domain models for example-uws.""" + +from __future__ import annotations + +from pydantic import BaseModel + +__all__ = ["WorkerExampleuwsModel"] + + +class WorkerExampleuwsModel(BaseModel): + """Parameter model for backend workers. + + Add fields here for all the parameters passed to backend jobs. + """ diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/__init__.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/external.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/external.py new file mode 100644 index 00000000..33b57523 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/external.py @@ -0,0 +1,52 @@ +"""Handlers for the app's external root, ``/example-uws/``.""" + +from typing import Annotated + +from fastapi import APIRouter, Depends +from safir.dependencies.logger import logger_dependency +from safir.metadata import get_metadata +from structlog.stdlib import BoundLogger + +from ..config import config +from ..models import Index + +__all__ = ["get_index", "external_router"] + +external_router = APIRouter() +"""FastAPI router for all external handlers.""" + + +@external_router.get( + "/", + description=( + "Document the top-level API here. By default it only returns metadata" + " about the application." + ), + response_model=Index, + response_model_exclude_none=True, + summary="Application metadata", +) +async def get_index( + logger: Annotated[BoundLogger, Depends(logger_dependency)], +) -> Index: + """GET ``/example-uws/`` (the app's external root). + + Customize this handler to return whatever the top-level resource of your + application should return. For example, consider listing key API URLs. + When doing so, also change or customize the response model in + `exampleuws.models.Index`. + + By convention, the root of the external API includes a field called + ``metadata`` that provides the same Safir-generated metadata as the + internal root endpoint. + """ + # There is no need to log simple requests since uvicorn will do this + # automatically, but this is included as an example of how to use the + # logger for more complex logging. + logger.info("Request for application metadata") + + metadata = get_metadata( + package_name="example-uws", + application_name=config.name, + ) + return Index(metadata=metadata) diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/internal.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/internal.py new file mode 100644 index 00000000..ae1f5b19 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/internal.py @@ -0,0 +1,42 @@ +"""Internal HTTP handlers that serve relative to the root path, ``/``. + +These handlers aren't externally visible since the app is available at a path, +``/example-uws``. See `exampleuws.handlers.external` for +the external endpoint handlers. + +These handlers should be used for monitoring, health checks, internal status, +or other information that should not be visible outside the Kubernetes cluster. +""" + +from fastapi import APIRouter +from safir.metadata import Metadata, get_metadata + +from ..config import config + +__all__ = ["get_index", "internal_router"] + +internal_router = APIRouter() +"""FastAPI router for all internal handlers.""" + + +@internal_router.get( + "/", + description=( + "Return metadata about the running application. Can also be used as" + " a health check. This route is not exposed outside the cluster and" + " therefore cannot be used by external clients." + ), + include_in_schema=False, + response_model=Metadata, + response_model_exclude_none=True, + summary="Application metadata", +) +async def get_index() -> Metadata: + """GET ``/`` (the app's internal root). + + By convention, this endpoint returns only the application's metadata. + """ + return get_metadata( + package_name="example-uws", + application_name=config.name, + ) diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/main.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/main.py new file mode 100644 index 00000000..f7bbcf16 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/main.py @@ -0,0 +1,67 @@ +"""The main application factory for the example-uws service. + +Notes +----- +Be aware that, following the normal pattern for FastAPI services, the app is +constructed when this module is loaded and is not deferred until a function is +called. +""" + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from importlib.metadata import metadata, version + +from fastapi import FastAPI +from safir.dependencies.http_client import http_client_dependency +from safir.logging import configure_logging, configure_uvicorn_logging +from safir.middleware.x_forwarded import XForwardedMiddleware + +from .config import config, uws +from .handlers.external import external_router +from .handlers.internal import internal_router + +__all__ = ["app"] + + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncIterator[None]: + """Set up and tear down the application.""" + # Any code here will be run when the application starts up. + await uws.initialize_fastapi() + + yield + + # Any code here will be run when the application shuts down. + await uws.shutdown_fastapi() + await http_client_dependency.aclose() + + +configure_logging( + profile=config.profile, + log_level=config.log_level, + name="exampleuws", +) +configure_uvicorn_logging(config.log_level) + +app = FastAPI( + title="example-uws", + description=metadata("example-uws")["Summary"], + version=version("example-uws"), + openapi_url=f"{config.path_prefix}/openapi.json", + docs_url=f"{config.path_prefix}/docs", + redoc_url=f"{config.path_prefix}/redoc", + lifespan=lifespan, +) +"""The main FastAPI application for example-uws.""" + +# Attach the routers. +app.include_router(internal_router) +uws.install_handlers(external_router) +app.include_router(external_router, prefix=f"{config.path_prefix}") + +# Add middleware. +app.add_middleware(XForwardedMiddleware) +uws.install_middleware(app) + +# Install error handlers. +uws.install_error_handlers(app) diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/models.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/models.py new file mode 100644 index 00000000..df4b159a --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/models.py @@ -0,0 +1,41 @@ +"""Models for example-uws.""" + +from typing import Self + +from pydantic import BaseModel, Field +from safir.metadata import Metadata as SafirMetadata +from safir.uws import ParametersModel, UWSJobParameter + +from .domain import WorkerExampleuwsModel + +__all__ = ["Index"] + + +class Index(BaseModel): + """Metadata returned by the external root URL of the application. + + Notes + ----- + As written, this is not very useful. Add additional metadata that will be + helpful for a user exploring the application, or replace this model with + some other model that makes more sense to return from the application API + root. + """ + + metadata: SafirMetadata = Field(..., title="Package metadata") + + +class ExampleuwsParameters(ParametersModel[WorkerExampleuwsModel]): + """Model for job parameters. + + Add fields here for all the input parameters to a job, and then update + ``from_job_parameters`` and ``to_worker_parameters`` to do the appropriate + conversions. + """ + + @classmethod + def from_job_parameters(cls, params: list[UWSJobParameter]) -> Self: + return cls() + + def to_worker_parameters(self) -> WorkerExampleuwsModel: + return WorkerExampleuwsModel() diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/__init__.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/exampleuws.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/exampleuws.py new file mode 100644 index 00000000..ff8f0d26 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/exampleuws.py @@ -0,0 +1,97 @@ +"""Worker for example-uws. + +This is a standalone file intended to be injected into a stack container as +the arq worker definition. Only this module is allowed to use stack packages. +""" + +from __future__ import annotations + +import os +from datetime import timedelta + +import structlog +from safir.arq import ArqMode +from safir.arq.uws import ( + WorkerConfig, + WorkerFatalError, + WorkerJobInfo, + WorkerResult, + build_worker, +) +from structlog.stdlib import BoundLogger + +from ..domain import WorkerExampleuwsModel + +__all__ = ["WorkerSettings"] + + +def example_uws( + params: WorkerExampleuwsModel, info: WorkerJobInfo, logger: BoundLogger +) -> list[WorkerResult]: + """Perform the work. + + This is a queue worker for the example-uws service. It takes a serialized + request, converts it into a suitable in-memory format, and then dispatches + it to the scientific code that performs the cutout. The results are stored + in a GCS bucket, and the details of the output are returned as the result + of the worker. + + Parameters + ---------- + params + Cutout parameters. + info + Information about the UWS job we're executing. + logger + Logger to use for logging. + + Returns + ------- + list of WorkerResult + Results of the job. + + Raises + ------ + WorkerFatalError + Raised if the cutout failed for unknown reasons, or due to internal + errors. This is the normal failure exception, since we usually do not + know why the backend code failed and make the pessimistic assumption + that the failure is not transient. + WorkerUsageError + Raised if the cutout failed due to deficiencies in the parameters + submitted by the user that could not be detected by the frontend + service. + """ + logger.info("Starting request") + try: + # Replace this with a call to the function that does the work. + result_url = "https://example.com/" + except Exception as e: + raise WorkerFatalError(f"{type(e).__name__}: {e!s}") from e + logger.info("Request successful") + + # Change the result ID to something reasonable, set the MIME type to an + # appropriate value, and ideally also set the result size if that's + # available. + return [ + WorkerResult( + result_id="main", url=result_url, mime_type="application/fits" + ) + ] + + +WorkerSettings = build_worker( + example_uws, + WorkerConfig( + arq_mode=ArqMode.production, + arq_queue_url=os.environ["EXAMPLE_UWS_ARQ_QUEUE_URL"], + arq_queue_password=os.getenv("EXAMPLE_UWS_ARQ_QUEUE_PASSWORD"), + grace_period=timedelta( + seconds=int(os.environ["EXAMPLE_UWS_GRACE_PERIOD"]) + ), + parameters_class=WorkerExampleuwsModel, + timeout=timedelta(seconds=int(os.environ["EXAMPLE_UWS_TIMEOUT"])), + ), + structlog.get_logger("exampleuws"), +) +"""arq configuration for the example-uws worker.""" diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/uws.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/uws.py new file mode 100644 index 00000000..910eddc1 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/uws.py @@ -0,0 +1,19 @@ + +"""Worker for UWS database updates.""" + +from __future__ import annotations + +import structlog +from safir.logging import configure_logging + +from ..config import config, uws + +__all__ = ["WorkerSettings"] + + +configure_logging( + name="exampleuws", profile=config.profile, log_level=config.log_level +) + +WorkerSettings = uws.build_worker(structlog.get_logger("exampleuws")) +"""arq configuration for the UWS database worker.""" diff --git a/project_templates/fastapi_safir_app/example-uws/tests/__init__.py b/project_templates/fastapi_safir_app/example-uws/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/project_templates/fastapi_safir_app/example-uws/tests/conftest.py b/project_templates/fastapi_safir_app/example-uws/tests/conftest.py new file mode 100644 index 00000000..e53d9049 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/tests/conftest.py @@ -0,0 +1,63 @@ +"""Test fixtures for example-uws tests.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator, Iterator +from datetime import timedelta + +import pytest +import pytest_asyncio +import structlog +from asgi_lifespan import LifespanManager +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient +from safir.arq import MockArqQueue +from safir.testing.gcs import MockStorageClient, patch_google_storage +from safir.testing.uws import MockUWSJobRunner + +from exampleuws import main +from exampleuws.config import config, uws + + +@pytest_asyncio.fixture +async def app(arq_queue: MockArqQueue) -> AsyncIterator[FastAPI]: + """Return a configured test application. + + Wraps the application in a lifespan manager so that startup and shutdown + events are sent during test execution. + """ + logger = structlog.get_logger("exampleuws") + await uws.initialize_uws_database(logger, reset=True) + uws.override_arq_queue(arq_queue) + async with LifespanManager(main.app): + yield main.app + + +@pytest.fixture +def arq_queue() -> MockArqQueue: + return MockArqQueue() + + +@pytest_asyncio.fixture +async def client(app: FastAPI) -> AsyncIterator[AsyncClient]: + """Return an ``httpx.AsyncClient`` configured to talk to the test app.""" + async with AsyncClient( + transport=ASGITransport(app=app), # type: ignore[arg-type] + base_url="https://example.com/", + ) as client: + yield client + + +@pytest.fixture(autouse=True) +def mock_google_storage() -> Iterator[MockStorageClient]: + yield from patch_google_storage( + expected_expiration=timedelta(minutes=15), bucket_name="some-bucket" + ) + + +@pytest_asyncio.fixture +async def runner( + arq_queue: MockArqQueue, +) -> AsyncIterator[MockUWSJobRunner]: + async with MockUWSJobRunner(config.uws_config, arq_queue) as runner: + yield runner diff --git a/project_templates/fastapi_safir_app/example-uws/tests/handlers/__init__.py b/project_templates/fastapi_safir_app/example-uws/tests/handlers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/project_templates/fastapi_safir_app/example-uws/tests/handlers/external_test.py b/project_templates/fastapi_safir_app/example-uws/tests/handlers/external_test.py new file mode 100644 index 00000000..af1b2d26 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/tests/handlers/external_test.py @@ -0,0 +1,22 @@ +"""Tests for the exampleuws.handlers.external module and routes.""" + +from __future__ import annotations + +import pytest +from httpx import AsyncClient + +from exampleuws.config import config + + +@pytest.mark.asyncio +async def test_get_index(client: AsyncClient) -> None: + """Test ``GET /example-uws/``.""" + response = await client.get("/example-uws/") + assert response.status_code == 200 + data = response.json() + metadata = data["metadata"] + assert metadata["name"] == config.name + assert isinstance(metadata["version"], str) + assert isinstance(metadata["description"], str) + assert isinstance(metadata["repository_url"], str) + assert isinstance(metadata["documentation_url"], str) diff --git a/project_templates/fastapi_safir_app/example-uws/tests/handlers/internal_test.py b/project_templates/fastapi_safir_app/example-uws/tests/handlers/internal_test.py new file mode 100644 index 00000000..85bfd89d --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/tests/handlers/internal_test.py @@ -0,0 +1,21 @@ +"""Tests for the exampleuws.handlers.internal module and routes.""" + +from __future__ import annotations + +import pytest +from httpx import AsyncClient + +from exampleuws.config import config + + +@pytest.mark.asyncio +async def test_get_index(client: AsyncClient) -> None: + """Test ``GET /``.""" + response = await client.get("/") + assert response.status_code == 200 + data = response.json() + assert data["name"] == config.name + assert isinstance(data["version"], str) + assert isinstance(data["description"], str) + assert isinstance(data["repository_url"], str) + assert isinstance(data["documentation_url"], str) diff --git a/project_templates/fastapi_safir_app/example-uws/tox.ini b/project_templates/fastapi_safir_app/example-uws/tox.ini new file mode 100644 index 00000000..db6ecd57 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/tox.ini @@ -0,0 +1,63 @@ +[tox] +envlist = py,coverage-report,typing,lint +isolated_build = True + +[docker:postgres] +image = postgres:latest +environment = + POSTGRES_PASSWORD=INSECURE-PASSWORD + POSTGRES_USER=example-uws + POSTGRES_DB=example-uws +# The healthcheck ensures that tox-docker won't run tests until the +# container is up and the command finishes with exit code 0 (success) +healthcheck_cmd = PGPASSWORD=$POSTGRES_PASSWORD psql \ + --user=$POSTGRES_USER --dbname=$POSTGRES_DB \ + --host=127.0.0.1 --quiet --no-align --tuples-only \ + -1 --command="SELECT 1" +healthcheck_timeout = 1 +healthcheck_retries = 30 +healthcheck_interval = 1 +healthcheck_start_period = 1 + +[testenv] +description = Run pytest against {envname}. +deps = + -r{toxinidir}/requirements/main.txt + -r{toxinidir}/requirements/dev.txt +commands = + pytest --cov=exampleuws --cov-branch --cov-report= {posargs} + +[testenv:coverage-report] +description = Compile coverage from each test run. +skip_install = true +deps = coverage[toml]>=5.0.2 +depends = + py +commands = coverage report + +[testenv:py] +docker = + postgres +setenv = + EXAMPLE_UWS_DATABASE_URL = postgresql://example-uws@localhost/example-uws + EXAMPLE_UWS_DATABASE_PASSWORD = INSECURE-PASSWORD + EXAMPLE_UWS_ARQ_QUEUE_URL = redis://localhost/0 + EXAMPLE_UWS_SERVICE_ACCOUNT = example-uws@example.com + EXAMPLE_UWS_STORAGE_URL = gs://some-bucket + +[testenv:typing] +description = Run mypy. +commands = + mypy src/exampleuws tests + +[testenv:lint] +description = Lint codebase by running pre-commit +skip_install = true +deps = + pre-commit +commands = pre-commit run --all-files + +[testenv:run] +description = Run the development server with auto-reload for code changes. +usedevelop = true +commands = uvicorn exampleuws.main:app --reload diff --git a/project_templates/fastapi_safir_app/example/ruff-shared.toml b/project_templates/fastapi_safir_app/example/ruff-shared.toml index 823693a4..da644fb3 100644 --- a/project_templates/fastapi_safir_app/example/ruff-shared.toml +++ b/project_templates/fastapi_safir_app/example/ruff-shared.toml @@ -30,6 +30,7 @@ ignore = [ "ANN401", # sometimes Any is the right type "ARG001", # unused function arguments are often legitimate "ARG002", # unused method arguments are often legitimate + "ARG003", # unused class method arguments are often legitimate "ARG005", # unused lambda arguments are often legitimate "BLE001", # we want to catch and report Exception in background tasks "C414", # nested sorted is how you sort by multiple keys with reverse diff --git a/project_templates/fastapi_safir_app/example/src/example/main.py b/project_templates/fastapi_safir_app/example/src/example/main.py index 7a97bd2b..e649f64b 100644 --- a/project_templates/fastapi_safir_app/example/src/example/main.py +++ b/project_templates/fastapi_safir_app/example/src/example/main.py @@ -20,7 +20,7 @@ from .handlers.external import external_router from .handlers.internal import internal_router -__all__ = ["app", "config"] +__all__ = ["app"] @asynccontextmanager diff --git a/project_templates/fastapi_safir_app/hooks/post_gen_project.py b/project_templates/fastapi_safir_app/hooks/post_gen_project.py index 8ae16a76..0252ce8e 100644 --- a/project_templates/fastapi_safir_app/hooks/post_gen_project.py +++ b/project_templates/fastapi_safir_app/hooks/post_gen_project.py @@ -9,4 +9,18 @@ import shutil # These variables are interpolated by cookiecutter before this hook is run +uws_service = True if '{{ cookiecutter.uws_service }}' == 'True' else False +module_name = '{{ cookiecutter.module_name }}' github_org = '{{ cookiecutter.github_org }}' + +# Remove unused files if the application will not be using UWS. +if not uws_service: + print(f"(post-gen hook) Removing unused UWS support files") + shutil.rmtree(f"src/{module_name}/workers") + os.remove("Dockerfile.worker") + os.remove("scripts/install-worker.sh") + os.remove("scripts/install-worker-packages.sh") + os.remove("scripts/start-worker.sh") + os.remove(f"src/{module_name}/cli.py") + os.remove(f"src/{module_name}/dependencies.py") + os.remove(f"src/{module_name}/domain.py") diff --git a/project_templates/fastapi_safir_app/templatekit.yaml b/project_templates/fastapi_safir_app/templatekit.yaml index a33c4034..24b7ee9d 100644 --- a/project_templates/fastapi_safir_app/templatekit.yaml +++ b/project_templates/fastapi_safir_app/templatekit.yaml @@ -17,3 +17,7 @@ dialog_fields: - label: "Initial copyright holder" key: "copyright_holder" component: "select" + - label: "UWS service" + key: "uws_service" + hint: "Select True if this is an IVOA UWS service with a separate worker." + component: "select" diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/.github/workflows/ci.yaml b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/.github/workflows/ci.yaml index 8e9d4f73..c187b137 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/.github/workflows/ci.yaml +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/.github/workflows/ci.yaml @@ -79,3 +79,11 @@ jobs: with: image: {{ "${{ github.repository }}" }} github_token: {{ "${{ secrets.GITHUB_TOKEN }}" }} +{%- if cookiecutter.uws_service == "True" %} + + - uses: lsst-sqre/build-and-push-to-ghcr@v1 + id: build-worker + with: + image: {{ "${{ github.repository }}-worker" }} + github_token: {{ "${{ secrets.GITHUB_TOKEN }}" }} +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/Dockerfile.worker b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/Dockerfile.worker new file mode 100644 index 00000000..c8c8d703 --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/Dockerfile.worker @@ -0,0 +1,29 @@ +{%- if cookiecutter.uws_service == "True" %} +# This Dockerfile constructs the image for backend workers. These images +# are based on stack containers and install any required supporting code +# for the backend, arq, and the backend worker definition. + +FROM lsstsqre/centos:7-stack-lsst_distrib-w_2024_27 + +# Reset the user to root since we need to do system install tasks. +USER root + +# Upgrade the system packages. +COPY scripts/install-worker-packages.sh . +RUN ./install-worker-packages.sh && rm ./install-worker-packages.sh + +# Install the necessary prerequisites and the vo-cutouts code. +COPY . /workdir +RUN /workdir/scripts/install-worker.sh /workdir && rm -r /workdir +COPY scripts/start-worker.sh / + +# Create a non-root user +RUN useradd --create-home appuser + +# Switch to the non-root user. +USER appuser + +# Start the arq worker. +WORKDIR / +CMD ["/start-worker.sh"] +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in index 985d2fe1..32661f34 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in @@ -15,4 +15,8 @@ uvicorn[standard] # Other dependencies. pydantic pydantic-settings +{%- if cookiecutter.uws_service == "True" %} +safir[uws]>=7 +{%- else %} safir>=5 +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/tox.in b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/tox.in index fde253f3..c815cd88 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/tox.in +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/tox.in @@ -11,4 +11,7 @@ -c dev.txt tox +{%- if cookiecutter.uws_service == "True" %} +tox-docker +{%- endif %} tox-uv diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/ruff-shared.toml b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/ruff-shared.toml index 823693a4..da644fb3 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/ruff-shared.toml +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/ruff-shared.toml @@ -30,6 +30,7 @@ ignore = [ "ANN401", # sometimes Any is the right type "ARG001", # unused function arguments are often legitimate "ARG002", # unused method arguments are often legitimate + "ARG003", # unused class method arguments are often legitimate "ARG005", # unused lambda arguments are often legitimate "BLE001", # we want to catch and report Exception in background tasks "C414", # nested sorted is how you sort by multiple keys with reverse diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker-packages.sh b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker-packages.sh new file mode 100644 index 00000000..5c14b855 --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker-packages.sh @@ -0,0 +1,24 @@ +{%- if cookiecutter.uws_service == "True" %} +#!/bin/bash + +# Install or upgrade any operating system packages needed on worker images. +# This is done in a separate script to create a separate cached Docker image, +# which will help with iteration speed on the more interesting setup actions +# taken later in the build. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for details. +set -euo pipefail + +# Display each command as it's run. +set -x + +# Upgrade the Red Hat packages. +# +# TODO(rra): Disabled for now because the version of CentOS used by the image +# is so old that the package repositories no longer exist. This will in theory +# soon be fixed by basing the image on AlmaLinux. +#yum -y upgrade +#yum clean all +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh new file mode 100644 index 00000000..ae885bd8 --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh @@ -0,0 +1,26 @@ +{%- if cookiecutter.uws_service == "True" %} +#!/bin/bash + +# This script updates and installs the necessary prerequisites for a backend +# worker starting with a stack container. It takes one parameter, the +# directory in which to do the installation. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for details. +# set -u is omitted because the setup bash function does not support it. +set -eo pipefail + +# Enable the stack. This should be done before set -x because it will +# otherwise spew a bunch of nonsense no one cares about into the logs. +source /opt/lsst/software/stack/loadLSST.bash +setup lsst_distrib + +# Display each command as it's run. +set -x + +# Install Python dependencies and the {{ cookiecutter.name }} code. +cd "$1" +pip install --no-cache-dir google-cloud-storage safir-arq +pip install --no-cache-dir --no-deps . +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/start-worker.sh b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/start-worker.sh new file mode 100644 index 00000000..c70bb34f --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/start-worker.sh @@ -0,0 +1,19 @@ +{%- if cookiecutter.uws_service == "True" %} +#!/bin/bash + +# This script is installed in the worker image and starts the backend worker +# using arq. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for details. +# set -u is omitted because the setup bash function does not support it. +set -eo pipefail + +# Initialize the environment. +source /opt/lsst/software/stack/loadLSST.bash +setup lsst_distrib + +# Start arq with the worker. +arq {{ cookiecutter.module_name }}.workers.{{ cookiecutter.module_name }}.WorkerSettings +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/cli.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/cli.py new file mode 100644 index 00000000..b3b17746 --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/cli.py @@ -0,0 +1,43 @@ +{%- if cookiecutter.uws_service == "True" %} +"""Administrative command-line interface.""" + +from __future__ import annotations + +import click +import structlog +from safir.asyncio import run_with_asyncio +from safir.click import display_help + +from .config import uws + +__all__ = [ + "help", + "init", + "main", +] + + +@click.group(context_settings={"help_option_names": ["-h", "--help"]}) +@click.version_option(message="%(version)s") +def main() -> None: + """Administrative command-line interface for {{ cookiecutter.name }}.""" + + +@main.command() +@click.argument("topic", default=None, required=False, nargs=1) +@click.pass_context +def help(ctx: click.Context, topic: str | None) -> None: + """Show help for any command.""" + display_help(main, ctx, topic) + + +@main.command() +@click.option( + "--reset", is_flag=True, help="Delete all existing database data." +) +@run_with_asyncio +async def init(*, reset: bool) -> None: + """Initialize the database storage.""" + logger = structlog.get_logger("{{ cookiecutter.module_name }}") + await uws.initialize_uws_database(logger, reset=reset) +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/config.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/config.py index dbe10d1e..5be025f5 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/config.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/config.py @@ -3,13 +3,19 @@ from __future__ import annotations from pydantic import Field -from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic_settings import {% if cookiecutter.uws_service != 'True' %}BaseSettings, {% endif %}SettingsConfigDict from safir.logging import LogLevel, Profile +{%- if cookiecutter.uws_service == "True" %} +from safir.uws import UWSApplication, UWSAppSettings, UWSConfig, UWSRoute + +from .dependencies import post_params_dependency +from .models import {{ cookiecutter.module_name | capitalize }}Parameters +{%- endif %} __all__ = ["Config", "config"] -class Config(BaseSettings): +class Config({% if cookiecutter.uws_service == 'True' %}UWSAppSettings{% else %}BaseSettings{% endif %}): """Configuration for {{ cookiecutter.name }}.""" name: str = Field("{{ cookiecutter.name }}", title="Name of application") @@ -29,7 +35,27 @@ class Config(BaseSettings): model_config = SettingsConfigDict( env_prefix="{{ cookiecutter.name | upper | replace('-', '_') }}_", case_sensitive=False ) +{%- if cookiecutter.uws_service == "True" %} + + @property + def uws_config(self) -> UWSConfig: + """Corresponding configuration for the UWS subsystem.""" + return self.build_uws_config( + parameters_type={{ cookiecutter.module_name | capitalize }}Parameters, + worker="{{ cookiecutter.name | replace('-', '_') }}", + async_post_route=UWSRoute( + dependency=post_params_dependency, + summary="Create async {{ cookiecutter.name }} job", + description="Create a new UWS job for {{ cookiecutter.name }}", + ), + ) +{%- endif %} config = Config() """Configuration for {{ cookiecutter.name }}.""" +{%- if cookiecutter.uws_service == "True" %} + +uws = UWSApplication(config.uws_config) +"""The UWS application for this service.""" +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/dependencies.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/dependencies.py new file mode 100644 index 00000000..f7ebf7aa --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/dependencies.py @@ -0,0 +1,29 @@ +{%- if cookiecutter.uws_service == "True" %} +"""Job parameter dependencies.""" + +from typing import Annotated + +from fastapi import Depends, Request +from safir.uws import UWSJobParameter, uws_post_params_dependency + +__all__ = [ + "post_params_dependency", +] + + +async def post_params_dependency( + *, + # Add POST parameters here. All of them should be Form() parameters. + # Use str | None for single-valued attributes and str | list[str] | None + # for parameters that can be given more than one time. + params: Annotated[ + list[UWSJobParameter], Depends(uws_post_params_dependency) + ], +) -> list[UWSJobParameter]: + """Parse POST parameters into job parameters.""" + return [ + p + for p in params + if p.parameter_id in set() # Replace with set of parameter names + ] +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/domain.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/domain.py new file mode 100644 index 00000000..9d9a2e9e --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/domain.py @@ -0,0 +1,16 @@ +{%- if cookiecutter.uws_service == "True" %} +"""Domain models for {{ cookiecutter.name }}.""" + +from __future__ import annotations + +from pydantic import BaseModel + +__all__ = ["Worker{{ cookiecutter.module_name | capitalize }}Model"] + + +class Worker{{ cookiecutter.module_name | capitalize }}Model(BaseModel): + """Parameter model for backend workers. + + Add fields here for all the parameters passed to backend jobs. + """ +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/main.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/main.py index 1539ce02..d096e9ba 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/main.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/main.py @@ -16,21 +16,27 @@ from safir.logging import configure_logging, configure_uvicorn_logging from safir.middleware.x_forwarded import XForwardedMiddleware -from .config import config +from .config import config{% if cookiecutter.uws_service == "True" %}, uws{% endif %} from .handlers.external import external_router from .handlers.internal import internal_router -__all__ = ["app", "config"] +__all__ = ["app"] @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: """Set up and tear down the application.""" # Any code here will be run when the application starts up. + {%- if cookiecutter.uws_service == "True" %} + await uws.initialize_fastapi() + {%- endif %} yield # Any code here will be run when the application shuts down. + {%- if cookiecutter.uws_service == "True" %} + await uws.shutdown_fastapi() + {%- endif %} await http_client_dependency.aclose() @@ -54,7 +60,16 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # Attach the routers. app.include_router(internal_router) +{%- if cookiecutter.uws_service == "True" %} +uws.install_handlers(external_router) +{%- endif %} app.include_router(external_router, prefix=f"{config.path_prefix}") # Add middleware. app.add_middleware(XForwardedMiddleware) +{%- if cookiecutter.uws_service == "True" %} +uws.install_middleware(app) + +# Install error handlers. +uws.install_error_handlers(app) +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/models.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/models.py index 8a4c6b6b..b9a51133 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/models.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/models.py @@ -1,7 +1,16 @@ """Models for {{ cookiecutter.name }}.""" +{% if cookiecutter.uws_service == "True" -%} +from typing import Self + +{% endif -%} from pydantic import BaseModel, Field from safir.metadata import Metadata as SafirMetadata +{%- if cookiecutter.uws_service == "True" %} +from safir.uws import ParametersModel, UWSJobParameter + +from .domain import Worker{{ cookiecutter.module_name | capitalize }}Model +{%- endif %} __all__ = ["Index"] @@ -18,3 +27,21 @@ class Index(BaseModel): """ metadata: SafirMetadata = Field(..., title="Package metadata") +{%- if cookiecutter.uws_service == "True" %} + + +class {{ cookiecutter.module_name | capitalize }}Parameters(ParametersModel[Worker{{ cookiecutter.module_name | capitalize }}Model]): + """Model for job parameters. + + Add fields here for all the input parameters to a job, and then update + ``from_job_parameters`` and ``to_worker_parameters`` to do the appropriate + conversions. + """ + + @classmethod + def from_job_parameters(cls, params: list[UWSJobParameter]) -> Self: + return cls() + + def to_worker_parameters(self) -> Worker{{ cookiecutter.module_name | capitalize }}Model: + return Worker{{ cookiecutter.module_name | capitalize }}Model() +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/__init__.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/uws.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/uws.py new file mode 100644 index 00000000..12826354 --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/uws.py @@ -0,0 +1,20 @@ +{%- if cookiecutter.uws_service == "True" %} +"""Worker for UWS database updates.""" + +from __future__ import annotations + +import structlog +from safir.logging import configure_logging + +from ..config import config, uws + +__all__ = ["WorkerSettings"] + + +configure_logging( + name="{{ cookiecutter.module_name }}", profile=config.profile, log_level=config.log_level +) + +WorkerSettings = uws.build_worker(structlog.get_logger("{{ cookiecutter.module_name }}")) +"""arq configuration for the UWS database worker.""" +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/{{cookiecutter.module_name}}.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/{{cookiecutter.module_name}}.py new file mode 100644 index 00000000..7dc534ad --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/{{cookiecutter.module_name}}.py @@ -0,0 +1,99 @@ +{%- if cookiecutter.uws_service == "True" -%} +"""Worker for {{ cookiecutter.name }}. + +This is a standalone file intended to be injected into a stack container as +the arq worker definition. Only this module is allowed to use stack packages. +""" + +from __future__ import annotations + +import os +from datetime import timedelta + +import structlog +from safir.arq import ArqMode +from safir.arq.uws import ( + WorkerConfig, + WorkerFatalError, + WorkerJobInfo, + WorkerResult, + build_worker, +) +from structlog.stdlib import BoundLogger + +from ..domain import Worker{{ cookiecutter.module_name | capitalize }}Model + +__all__ = ["WorkerSettings"] + + +def {{ cookiecutter.name | replace('-', '_') }}( + params: Worker{{ cookiecutter.module_name | capitalize }}Model, info: WorkerJobInfo, logger: BoundLogger +) -> list[WorkerResult]: + """Perform the work. + + This is a queue worker for the {{ cookiecutter.name }} service. It takes a serialized + request, converts it into a suitable in-memory format, and then dispatches + it to the scientific code that performs the cutout. The results are stored + in a GCS bucket, and the details of the output are returned as the result + of the worker. + + Parameters + ---------- + params + Cutout parameters. + info + Information about the UWS job we're executing. + logger + Logger to use for logging. + + Returns + ------- + list of WorkerResult + Results of the job. + + Raises + ------ + WorkerFatalError + Raised if the cutout failed for unknown reasons, or due to internal + errors. This is the normal failure exception, since we usually do not + know why the backend code failed and make the pessimistic assumption + that the failure is not transient. + WorkerUsageError + Raised if the cutout failed due to deficiencies in the parameters + submitted by the user that could not be detected by the frontend + service. + """ + logger.info("Starting request") + try: + # Replace this with a call to the function that does the work. + result_url = "https://example.com/" + except Exception as e: + raise WorkerFatalError(f"{type(e).__name__}: {e!s}") from e + logger.info("Request successful") + + # Change the result ID to something reasonable, set the MIME type to an + # appropriate value, and ideally also set the result size if that's + # available. + return [ + WorkerResult( + result_id="main", url=result_url, mime_type="application/fits" + ) + ] + + +WorkerSettings = build_worker( + {{ cookiecutter.name | replace('-', '_') }}, + WorkerConfig( + arq_mode=ArqMode.production, + arq_queue_url=os.environ["{{ cookiecutter.name | upper | replace('-', '_') }}_ARQ_QUEUE_URL"], + arq_queue_password=os.getenv("{{ cookiecutter.name | upper | replace('-', '_') }}_ARQ_QUEUE_PASSWORD"), + grace_period=timedelta( + seconds=int(os.environ["{{ cookiecutter.name | upper | replace('-', '_') }}_GRACE_PERIOD"]) + ), + parameters_class=Worker{{ cookiecutter.module_name | capitalize }}Model, + timeout=timedelta(seconds=int(os.environ["{{ cookiecutter.name | upper | replace('-', '_') }}_TIMEOUT"])), + ), + structlog.get_logger("{{ cookiecutter.module_name }}"), +) +"""arq configuration for the {{ cookiecutter.name }} worker.""" +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tests/conftest.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tests/conftest.py index ea59f213..4144aeec 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tests/conftest.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tests/conftest.py @@ -2,25 +2,57 @@ from __future__ import annotations -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator{% if cookiecutter.uws_service == "True" %}, Iterator +from datetime import timedelta +{%- endif %} +{% if cookiecutter.uws_service == "True" -%} +import pytest +{% endif -%} import pytest_asyncio +{%- if cookiecutter.uws_service == "True" %} +import structlog +{%- endif %} from asgi_lifespan import LifespanManager from fastapi import FastAPI from httpx import ASGITransport, AsyncClient +{%- if cookiecutter.uws_service == "True" %} +from safir.arq import MockArqQueue +from safir.testing.gcs import MockStorageClient, patch_google_storage +from safir.testing.uws import MockUWSJobRunner +{%- endif %} from {{ cookiecutter.module_name }} import main +{%- if cookiecutter.uws_service == "True" %} +from {{ cookiecutter.module_name }}.config import config, uws +{%- endif %} @pytest_asyncio.fixture +{%- if cookiecutter.uws_service == "True" %} +async def app(arq_queue: MockArqQueue) -> AsyncIterator[FastAPI]: +{%- else %} async def app() -> AsyncIterator[FastAPI]: +{%- endif %} """Return a configured test application. Wraps the application in a lifespan manager so that startup and shutdown events are sent during test execution. """ + {%- if cookiecutter.uws_service == "True" %} + logger = structlog.get_logger("{{ cookiecutter.module_name }}") + await uws.initialize_uws_database(logger, reset=True) + uws.override_arq_queue(arq_queue) + {%- endif %} async with LifespanManager(main.app): yield main.app +{%- if cookiecutter.uws_service == "True" %} + + +@pytest.fixture +def arq_queue() -> MockArqQueue: + return MockArqQueue() +{%- endif %} @pytest_asyncio.fixture @@ -31,3 +63,20 @@ async def client(app: FastAPI) -> AsyncIterator[AsyncClient]: base_url="https://example.com/", ) as client: yield client +{%- if cookiecutter.uws_service == "True" %} + + +@pytest.fixture(autouse=True) +def mock_google_storage() -> Iterator[MockStorageClient]: + yield from patch_google_storage( + expected_expiration=timedelta(minutes=15), bucket_name="some-bucket" + ) + + +@pytest_asyncio.fixture +async def runner( + arq_queue: MockArqQueue, +) -> AsyncIterator[MockUWSJobRunner]: + async with MockUWSJobRunner(config.uws_config, arq_queue) as runner: + yield runner +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tox.ini b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tox.ini index 2a358aa6..f602d40c 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tox.ini +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tox.ini @@ -1,6 +1,25 @@ [tox] envlist = py,coverage-report,typing,lint isolated_build = True +{%- if cookiecutter.uws_service == "True" %} + +[docker:postgres] +image = postgres:latest +environment = + POSTGRES_PASSWORD=INSECURE-PASSWORD + POSTGRES_USER={{ cookiecutter.name }} + POSTGRES_DB={{ cookiecutter.name }} +# The healthcheck ensures that tox-docker won't run tests until the +# container is up and the command finishes with exit code 0 (success) +healthcheck_cmd = PGPASSWORD=$POSTGRES_PASSWORD psql \ + --user=$POSTGRES_USER --dbname=$POSTGRES_DB \ + --host=127.0.0.1 --quiet --no-align --tuples-only \ + -1 --command="SELECT 1" +healthcheck_timeout = 1 +healthcheck_retries = 30 +healthcheck_interval = 1 +healthcheck_start_period = 1 +{%- endif %} [testenv] description = Run pytest against {envname}. @@ -17,6 +36,18 @@ deps = coverage[toml]>=5.0.2 depends = py commands = coverage report +{%- if cookiecutter.uws_service == "True" %} + +[testenv:py] +docker = + postgres +setenv = + {{ cookiecutter.name | upper | replace('-', '_') }}_DATABASE_URL = postgresql://{{cookiecutter.name}}@localhost/{{cookiecutter.name}} + {{ cookiecutter.name | upper | replace('-', '_') }}_DATABASE_PASSWORD = INSECURE-PASSWORD + {{ cookiecutter.name | upper | replace('-', '_') }}_ARQ_QUEUE_URL = redis://localhost/0 + {{ cookiecutter.name | upper | replace('-', '_') }}_SERVICE_ACCOUNT = {{cookiecutter.name}}@example.com + {{ cookiecutter.name | upper | replace('-', '_') }}_STORAGE_URL = gs://some-bucket +{%- endif %} [testenv:typing] description = Run mypy. diff --git a/project_templates/square_pypi_package/example/ruff-shared.toml b/project_templates/square_pypi_package/example/ruff-shared.toml index 823693a4..da644fb3 100644 --- a/project_templates/square_pypi_package/example/ruff-shared.toml +++ b/project_templates/square_pypi_package/example/ruff-shared.toml @@ -30,6 +30,7 @@ ignore = [ "ANN401", # sometimes Any is the right type "ARG001", # unused function arguments are often legitimate "ARG002", # unused method arguments are often legitimate + "ARG003", # unused class method arguments are often legitimate "ARG005", # unused lambda arguments are often legitimate "BLE001", # we want to catch and report Exception in background tasks "C414", # nested sorted is how you sort by multiple keys with reverse diff --git a/project_templates/square_pypi_package/{{cookiecutter.name}}/ruff-shared.toml b/project_templates/square_pypi_package/{{cookiecutter.name}}/ruff-shared.toml index 823693a4..da644fb3 100644 --- a/project_templates/square_pypi_package/{{cookiecutter.name}}/ruff-shared.toml +++ b/project_templates/square_pypi_package/{{cookiecutter.name}}/ruff-shared.toml @@ -30,6 +30,7 @@ ignore = [ "ANN401", # sometimes Any is the right type "ARG001", # unused function arguments are often legitimate "ARG002", # unused method arguments are often legitimate + "ARG003", # unused class method arguments are often legitimate "ARG005", # unused lambda arguments are often legitimate "BLE001", # we want to catch and report Exception in background tasks "C414", # nested sorted is how you sort by multiple keys with reverse diff --git a/project_templates/technote_md/testn-000/technote.toml b/project_templates/technote_md/testn-000/technote.toml index d81928a8..2b29dafe 100644 --- a/project_templates/technote_md/testn-000/technote.toml +++ b/project_templates/technote_md/testn-000/technote.toml @@ -4,7 +4,7 @@ series_id = "TESTN" canonical_url = "https://testn-000.lsst.io" github_url = "https://github.com/lsst/testn-000" github_default_branch = "main" -date_created = 2024-07-30T17:55:32Z +date_created = 2024-07-30T22:02:34Z organization.name = "Vera C. Rubin Observatory" organization.ror = "https://ror.org/048g3cy84" license.id = "CC-BY-4.0" diff --git a/project_templates/technote_rst/testn-000/technote.toml b/project_templates/technote_rst/testn-000/technote.toml index 05580a74..db919c6f 100644 --- a/project_templates/technote_rst/testn-000/technote.toml +++ b/project_templates/technote_rst/testn-000/technote.toml @@ -4,7 +4,7 @@ series_id = "TESTN" canonical_url = "https://testn-000.lsst.io" github_url = "https://github.com/lsst/testn-000" github_default_branch = "main" -date_created = 2024-07-30T17:55:32Z +date_created = 2024-07-30T22:02:34Z organization.name = "Vera C. Rubin Observatory" organization.ror = "https://ror.org/048g3cy84" license.id = "CC-BY-4.0" From 3ec9f416f57c6fd023d33186166391807000c270 Mon Sep 17 00:00:00 2001 From: Russ Allbery Date: Tue, 30 Jul 2024 15:23:22 -0700 Subject: [PATCH 2/4] Add a flavor setting to FastAPI Safir app template Rather than a boolean saying whether or not the service in question is a UWS service, instead add a new flavor key that currently takes only two values: Default and UWS. Change all the conditionals to instead check that the flavor is UWS to enable the UWS code. --- project_templates/fastapi_safir_app/SConscript | 2 +- .../fastapi_safir_app/cookiecutter.json | 5 ++++- .../hooks/post_gen_project.py | 2 +- .../fastapi_safir_app/templatekit.yaml | 8 ++++---- .../.github/workflows/ci.yaml | 2 +- .../{{cookiecutter.name}}/Dockerfile.worker | 2 +- .../{{cookiecutter.name}}/requirements/main.in | 2 +- .../{{cookiecutter.name}}/requirements/tox.in | 2 +- .../scripts/install-worker-packages.sh | 2 +- .../scripts/install-worker.sh | 2 +- .../scripts/start-worker.sh | 2 +- .../src/{{cookiecutter.module_name}}/cli.py | 2 +- .../src/{{cookiecutter.module_name}}/config.py | 10 +++++----- .../dependencies.py | 2 +- .../src/{{cookiecutter.module_name}}/domain.py | 2 +- .../src/{{cookiecutter.module_name}}/main.py | 10 +++++----- .../src/{{cookiecutter.module_name}}/models.py | 6 +++--- .../workers/uws.py | 2 +- .../workers/{{cookiecutter.module_name}}.py | 2 +- .../{{cookiecutter.name}}/tests/conftest.py | 18 +++++++++--------- .../{{cookiecutter.name}}/tox.ini | 4 ++-- .../technote_md/testn-000/technote.toml | 2 +- .../technote_rst/testn-000/technote.toml | 2 +- 23 files changed, 48 insertions(+), 45 deletions(-) diff --git a/project_templates/fastapi_safir_app/SConscript b/project_templates/fastapi_safir_app/SConscript index 2d1592cb..1905f159 100644 --- a/project_templates/fastapi_safir_app/SConscript +++ b/project_templates/fastapi_safir_app/SConscript @@ -16,4 +16,4 @@ env.Cookiecutter(AlwaysBuild(Dir('example')), env.Cookiecutter(AlwaysBuild(Dir('example-uws')), 'cookiecutter.json', cookiecutter_context={'name': 'example-uws', - 'uws_service': 'True'}) + 'flavor': 'UWS'}) diff --git a/project_templates/fastapi_safir_app/cookiecutter.json b/project_templates/fastapi_safir_app/cookiecutter.json index 85143b1e..a7946c8c 100644 --- a/project_templates/fastapi_safir_app/cookiecutter.json +++ b/project_templates/fastapi_safir_app/cookiecutter.json @@ -16,6 +16,9 @@ "lsst-dm", "lsst-sqre-testing" ], - "uws_service": ["False", "True"], + "flavor": [ + "Default", + "UWS" + ], "_extensions": ["jinja2_time.TimeExtension", "templatekit.TemplatekitExtension"] } diff --git a/project_templates/fastapi_safir_app/hooks/post_gen_project.py b/project_templates/fastapi_safir_app/hooks/post_gen_project.py index 0252ce8e..fa3ea129 100644 --- a/project_templates/fastapi_safir_app/hooks/post_gen_project.py +++ b/project_templates/fastapi_safir_app/hooks/post_gen_project.py @@ -9,7 +9,7 @@ import shutil # These variables are interpolated by cookiecutter before this hook is run -uws_service = True if '{{ cookiecutter.uws_service }}' == 'True' else False +uws_service = True if '{{ cookiecutter.flavor }}' == 'UWS' else False module_name = '{{ cookiecutter.module_name }}' github_org = '{{ cookiecutter.github_org }}' diff --git a/project_templates/fastapi_safir_app/templatekit.yaml b/project_templates/fastapi_safir_app/templatekit.yaml index 24b7ee9d..de2bab37 100644 --- a/project_templates/fastapi_safir_app/templatekit.yaml +++ b/project_templates/fastapi_safir_app/templatekit.yaml @@ -14,10 +14,10 @@ dialog_fields: label: "GitHub organization" hint: "The package will be created in this GitHub organization." component: "select" + - key: "flavor" + label: "Flavor" + hint: "Flavor of FastAPI service to create." + component: "select" - label: "Initial copyright holder" key: "copyright_holder" component: "select" - - label: "UWS service" - key: "uws_service" - hint: "Select True if this is an IVOA UWS service with a separate worker." - component: "select" diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/.github/workflows/ci.yaml b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/.github/workflows/ci.yaml index c187b137..1b156dae 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/.github/workflows/ci.yaml +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/.github/workflows/ci.yaml @@ -79,7 +79,7 @@ jobs: with: image: {{ "${{ github.repository }}" }} github_token: {{ "${{ secrets.GITHUB_TOKEN }}" }} -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} - uses: lsst-sqre/build-and-push-to-ghcr@v1 id: build-worker diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/Dockerfile.worker b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/Dockerfile.worker index c8c8d703..5d181a91 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/Dockerfile.worker +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/Dockerfile.worker @@ -1,4 +1,4 @@ -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} # This Dockerfile constructs the image for backend workers. These images # are based on stack containers and install any required supporting code # for the backend, arq, and the backend worker definition. diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in index 32661f34..e8f3d62d 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in @@ -15,7 +15,7 @@ uvicorn[standard] # Other dependencies. pydantic pydantic-settings -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} safir[uws]>=7 {%- else %} safir>=5 diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/tox.in b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/tox.in index c815cd88..bf382363 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/tox.in +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/tox.in @@ -11,7 +11,7 @@ -c dev.txt tox -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} tox-docker {%- endif %} tox-uv diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker-packages.sh b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker-packages.sh index 5c14b855..d2f52d5c 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker-packages.sh +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker-packages.sh @@ -1,4 +1,4 @@ -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} #!/bin/bash # Install or upgrade any operating system packages needed on worker images. diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh index ae885bd8..efaf3f16 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh @@ -1,4 +1,4 @@ -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} #!/bin/bash # This script updates and installs the necessary prerequisites for a backend diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/start-worker.sh b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/start-worker.sh index c70bb34f..6b411458 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/start-worker.sh +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/start-worker.sh @@ -1,4 +1,4 @@ -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} #!/bin/bash # This script is installed in the worker image and starts the backend worker diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/cli.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/cli.py index b3b17746..b2463b45 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/cli.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/cli.py @@ -1,4 +1,4 @@ -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} """Administrative command-line interface.""" from __future__ import annotations diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/config.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/config.py index 5be025f5..0c9b96fd 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/config.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/config.py @@ -3,9 +3,9 @@ from __future__ import annotations from pydantic import Field -from pydantic_settings import {% if cookiecutter.uws_service != 'True' %}BaseSettings, {% endif %}SettingsConfigDict +from pydantic_settings import {% if cookiecutter.flavor != "UWS" %}BaseSettings, {% endif %}SettingsConfigDict from safir.logging import LogLevel, Profile -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} from safir.uws import UWSApplication, UWSAppSettings, UWSConfig, UWSRoute from .dependencies import post_params_dependency @@ -15,7 +15,7 @@ __all__ = ["Config", "config"] -class Config({% if cookiecutter.uws_service == 'True' %}UWSAppSettings{% else %}BaseSettings{% endif %}): +class Config({% if cookiecutter.flavor == "UWS" %}UWSAppSettings{% else %}BaseSettings{% endif %}): """Configuration for {{ cookiecutter.name }}.""" name: str = Field("{{ cookiecutter.name }}", title="Name of application") @@ -35,7 +35,7 @@ class Config({% if cookiecutter.uws_service == 'True' %}UWSAppSettings{% else %} model_config = SettingsConfigDict( env_prefix="{{ cookiecutter.name | upper | replace('-', '_') }}_", case_sensitive=False ) -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} @property def uws_config(self) -> UWSConfig: @@ -54,7 +54,7 @@ def uws_config(self) -> UWSConfig: config = Config() """Configuration for {{ cookiecutter.name }}.""" -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} uws = UWSApplication(config.uws_config) """The UWS application for this service.""" diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/dependencies.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/dependencies.py index f7ebf7aa..5ce0eccd 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/dependencies.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/dependencies.py @@ -1,4 +1,4 @@ -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} """Job parameter dependencies.""" from typing import Annotated diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/domain.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/domain.py index 9d9a2e9e..dbd6a5ed 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/domain.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/domain.py @@ -1,4 +1,4 @@ -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} """Domain models for {{ cookiecutter.name }}.""" from __future__ import annotations diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/main.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/main.py index d096e9ba..a0c62010 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/main.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/main.py @@ -16,7 +16,7 @@ from safir.logging import configure_logging, configure_uvicorn_logging from safir.middleware.x_forwarded import XForwardedMiddleware -from .config import config{% if cookiecutter.uws_service == "True" %}, uws{% endif %} +from .config import config{% if cookiecutter.flavor == "UWS" %}, uws{% endif %} from .handlers.external import external_router from .handlers.internal import internal_router @@ -27,14 +27,14 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: """Set up and tear down the application.""" # Any code here will be run when the application starts up. - {%- if cookiecutter.uws_service == "True" %} + {%- if cookiecutter.flavor == "UWS" %} await uws.initialize_fastapi() {%- endif %} yield # Any code here will be run when the application shuts down. - {%- if cookiecutter.uws_service == "True" %} + {%- if cookiecutter.flavor == "UWS" %} await uws.shutdown_fastapi() {%- endif %} await http_client_dependency.aclose() @@ -60,14 +60,14 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # Attach the routers. app.include_router(internal_router) -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} uws.install_handlers(external_router) {%- endif %} app.include_router(external_router, prefix=f"{config.path_prefix}") # Add middleware. app.add_middleware(XForwardedMiddleware) -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} uws.install_middleware(app) # Install error handlers. diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/models.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/models.py index b9a51133..ca591f59 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/models.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/models.py @@ -1,12 +1,12 @@ """Models for {{ cookiecutter.name }}.""" -{% if cookiecutter.uws_service == "True" -%} +{% if cookiecutter.flavor == "UWS" -%} from typing import Self {% endif -%} from pydantic import BaseModel, Field from safir.metadata import Metadata as SafirMetadata -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} from safir.uws import ParametersModel, UWSJobParameter from .domain import Worker{{ cookiecutter.module_name | capitalize }}Model @@ -27,7 +27,7 @@ class Index(BaseModel): """ metadata: SafirMetadata = Field(..., title="Package metadata") -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} class {{ cookiecutter.module_name | capitalize }}Parameters(ParametersModel[Worker{{ cookiecutter.module_name | capitalize }}Model]): diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/uws.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/uws.py index 12826354..14b4ba93 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/uws.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/uws.py @@ -1,4 +1,4 @@ -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} """Worker for UWS database updates.""" from __future__ import annotations diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/{{cookiecutter.module_name}}.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/{{cookiecutter.module_name}}.py index 7dc534ad..0538c74e 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/{{cookiecutter.module_name}}.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/{{cookiecutter.module_name}}.py @@ -1,4 +1,4 @@ -{%- if cookiecutter.uws_service == "True" -%} +{%- if cookiecutter.flavor == "UWS" -%} """Worker for {{ cookiecutter.name }}. This is a standalone file intended to be injected into a stack container as diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tests/conftest.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tests/conftest.py index 4144aeec..377e7327 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tests/conftest.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tests/conftest.py @@ -2,34 +2,34 @@ from __future__ import annotations -from collections.abc import AsyncIterator{% if cookiecutter.uws_service == "True" %}, Iterator +from collections.abc import AsyncIterator{% if cookiecutter.flavor == "UWS" %}, Iterator from datetime import timedelta {%- endif %} -{% if cookiecutter.uws_service == "True" -%} +{% if cookiecutter.flavor == "UWS" -%} import pytest {% endif -%} import pytest_asyncio -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} import structlog {%- endif %} from asgi_lifespan import LifespanManager from fastapi import FastAPI from httpx import ASGITransport, AsyncClient -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} from safir.arq import MockArqQueue from safir.testing.gcs import MockStorageClient, patch_google_storage from safir.testing.uws import MockUWSJobRunner {%- endif %} from {{ cookiecutter.module_name }} import main -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} from {{ cookiecutter.module_name }}.config import config, uws {%- endif %} @pytest_asyncio.fixture -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} async def app(arq_queue: MockArqQueue) -> AsyncIterator[FastAPI]: {%- else %} async def app() -> AsyncIterator[FastAPI]: @@ -39,14 +39,14 @@ async def app() -> AsyncIterator[FastAPI]: Wraps the application in a lifespan manager so that startup and shutdown events are sent during test execution. """ - {%- if cookiecutter.uws_service == "True" %} + {%- if cookiecutter.flavor == "UWS" %} logger = structlog.get_logger("{{ cookiecutter.module_name }}") await uws.initialize_uws_database(logger, reset=True) uws.override_arq_queue(arq_queue) {%- endif %} async with LifespanManager(main.app): yield main.app -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} @pytest.fixture @@ -63,7 +63,7 @@ async def client(app: FastAPI) -> AsyncIterator[AsyncClient]: base_url="https://example.com/", ) as client: yield client -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} @pytest.fixture(autouse=True) diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tox.ini b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tox.ini index f602d40c..58fab517 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tox.ini +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = py,coverage-report,typing,lint isolated_build = True -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} [docker:postgres] image = postgres:latest @@ -36,7 +36,7 @@ deps = coverage[toml]>=5.0.2 depends = py commands = coverage report -{%- if cookiecutter.uws_service == "True" %} +{%- if cookiecutter.flavor == "UWS" %} [testenv:py] docker = diff --git a/project_templates/technote_md/testn-000/technote.toml b/project_templates/technote_md/testn-000/technote.toml index 2b29dafe..2b0cc259 100644 --- a/project_templates/technote_md/testn-000/technote.toml +++ b/project_templates/technote_md/testn-000/technote.toml @@ -4,7 +4,7 @@ series_id = "TESTN" canonical_url = "https://testn-000.lsst.io" github_url = "https://github.com/lsst/testn-000" github_default_branch = "main" -date_created = 2024-07-30T22:02:34Z +date_created = 2024-07-30T22:22:53Z organization.name = "Vera C. Rubin Observatory" organization.ror = "https://ror.org/048g3cy84" license.id = "CC-BY-4.0" diff --git a/project_templates/technote_rst/testn-000/technote.toml b/project_templates/technote_rst/testn-000/technote.toml index db919c6f..83db5f90 100644 --- a/project_templates/technote_rst/testn-000/technote.toml +++ b/project_templates/technote_rst/testn-000/technote.toml @@ -4,7 +4,7 @@ series_id = "TESTN" canonical_url = "https://testn-000.lsst.io" github_url = "https://github.com/lsst/testn-000" github_default_branch = "main" -date_created = 2024-07-30T22:02:34Z +date_created = 2024-07-30T22:22:53Z organization.name = "Vera C. Rubin Observatory" organization.ror = "https://ror.org/048g3cy84" license.id = "CC-BY-4.0" From 149b56bff04057f7df76bccb42108a6f25cd94e0 Mon Sep 17 00:00:00 2001 From: Russ Allbery Date: Wed, 31 Jul 2024 16:45:33 -0700 Subject: [PATCH 3/4] Add more dependencies to UWS worker Docker image The Butler client requires a pip-installed httpx, and the log configuration requires structlog. --- .../fastapi_safir_app/example-uws/scripts/install-worker.sh | 6 ++++-- .../{{cookiecutter.name}}/scripts/install-worker.sh | 6 ++++-- project_templates/technote_md/testn-000/technote.toml | 2 +- project_templates/technote_rst/testn-000/technote.toml | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/project_templates/fastapi_safir_app/example-uws/scripts/install-worker.sh b/project_templates/fastapi_safir_app/example-uws/scripts/install-worker.sh index 4109fd34..825b9a4c 100644 --- a/project_templates/fastapi_safir_app/example-uws/scripts/install-worker.sh +++ b/project_templates/fastapi_safir_app/example-uws/scripts/install-worker.sh @@ -19,7 +19,9 @@ setup lsst_distrib # Display each command as it's run. set -x -# Install Python dependencies and the example-uws code. +# Install Python dependencies and the example-uws code. httpx is +# required by the remote Butler client, and google-cloud-storage is used to +# store cutouts when the bucket has a gs URL. cd "$1" -pip install --no-cache-dir google-cloud-storage safir-arq +pip install --no-cache-dir google-cloud-storage httpx safir-arq structlog pip install --no-cache-dir --no-deps . diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh index efaf3f16..9796d81b 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh @@ -19,8 +19,10 @@ setup lsst_distrib # Display each command as it's run. set -x -# Install Python dependencies and the {{ cookiecutter.name }} code. +# Install Python dependencies and the {{ cookiecutter.name }} code. httpx is +# required by the remote Butler client, and google-cloud-storage is used to +# store cutouts when the bucket has a gs URL. cd "$1" -pip install --no-cache-dir google-cloud-storage safir-arq +pip install --no-cache-dir google-cloud-storage httpx safir-arq structlog pip install --no-cache-dir --no-deps . {%- endif %} diff --git a/project_templates/technote_md/testn-000/technote.toml b/project_templates/technote_md/testn-000/technote.toml index 2b0cc259..c2f270f5 100644 --- a/project_templates/technote_md/testn-000/technote.toml +++ b/project_templates/technote_md/testn-000/technote.toml @@ -4,7 +4,7 @@ series_id = "TESTN" canonical_url = "https://testn-000.lsst.io" github_url = "https://github.com/lsst/testn-000" github_default_branch = "main" -date_created = 2024-07-30T22:22:53Z +date_created = 2024-07-31T23:45:27Z organization.name = "Vera C. Rubin Observatory" organization.ror = "https://ror.org/048g3cy84" license.id = "CC-BY-4.0" diff --git a/project_templates/technote_rst/testn-000/technote.toml b/project_templates/technote_rst/testn-000/technote.toml index 83db5f90..516e6e1b 100644 --- a/project_templates/technote_rst/testn-000/technote.toml +++ b/project_templates/technote_rst/testn-000/technote.toml @@ -4,7 +4,7 @@ series_id = "TESTN" canonical_url = "https://testn-000.lsst.io" github_url = "https://github.com/lsst/testn-000" github_default_branch = "main" -date_created = 2024-07-30T22:22:53Z +date_created = 2024-07-31T23:45:27Z organization.name = "Vera C. Rubin Observatory" organization.ror = "https://ror.org/048g3cy84" license.id = "CC-BY-4.0" From e1a14ace98b5808dfcf52f7a9284df27987bcd6c Mon Sep 17 00:00:00 2001 From: Russ Allbery Date: Fri, 2 Aug 2024 11:55:33 -0700 Subject: [PATCH 4/4] Adjust FastAPI Safir app dependencies Make the UWS flavor depend on Safir 6.2.0, not 7, since the UWS support was just a feature release. --- .../fastapi_safir_app/example-uws/requirements/main.in | 2 +- .../{{cookiecutter.name}}/requirements/main.in | 2 +- project_templates/technote_md/testn-000/technote.toml | 2 +- project_templates/technote_rst/testn-000/technote.toml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/project_templates/fastapi_safir_app/example-uws/requirements/main.in b/project_templates/fastapi_safir_app/example-uws/requirements/main.in index 667f7e8f..9a2f9b13 100644 --- a/project_templates/fastapi_safir_app/example-uws/requirements/main.in +++ b/project_templates/fastapi_safir_app/example-uws/requirements/main.in @@ -15,4 +15,4 @@ uvicorn[standard] # Other dependencies. pydantic pydantic-settings -safir[uws]>=7 +safir[uws]>=6.2.0 diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in index e8f3d62d..51c89c88 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in @@ -16,7 +16,7 @@ uvicorn[standard] pydantic pydantic-settings {%- if cookiecutter.flavor == "UWS" %} -safir[uws]>=7 +safir[uws]>=6.2.0 {%- else %} safir>=5 {%- endif %} diff --git a/project_templates/technote_md/testn-000/technote.toml b/project_templates/technote_md/testn-000/technote.toml index c2f270f5..db06c7fe 100644 --- a/project_templates/technote_md/testn-000/technote.toml +++ b/project_templates/technote_md/testn-000/technote.toml @@ -4,7 +4,7 @@ series_id = "TESTN" canonical_url = "https://testn-000.lsst.io" github_url = "https://github.com/lsst/testn-000" github_default_branch = "main" -date_created = 2024-07-31T23:45:27Z +date_created = 2024-08-02T18:56:37Z organization.name = "Vera C. Rubin Observatory" organization.ror = "https://ror.org/048g3cy84" license.id = "CC-BY-4.0" diff --git a/project_templates/technote_rst/testn-000/technote.toml b/project_templates/technote_rst/testn-000/technote.toml index 516e6e1b..6f52362b 100644 --- a/project_templates/technote_rst/testn-000/technote.toml +++ b/project_templates/technote_rst/testn-000/technote.toml @@ -4,7 +4,7 @@ series_id = "TESTN" canonical_url = "https://testn-000.lsst.io" github_url = "https://github.com/lsst/testn-000" github_default_branch = "main" -date_created = 2024-07-31T23:45:27Z +date_created = 2024-08-02T18:56:37Z organization.name = "Vera C. Rubin Observatory" organization.ror = "https://ror.org/048g3cy84" license.id = "CC-BY-4.0"