diff --git a/project_templates/fastapi_safir_app/SConscript b/project_templates/fastapi_safir_app/SConscript index 318c4952..1905f159 100644 --- a/project_templates/fastapi_safir_app/SConscript +++ b/project_templates/fastapi_safir_app/SConscript @@ -6,3 +6,14 @@ from templatekit.builder import cookiecutter_project_builder env = Environment(BUILDERS={'Cookiecutter': cookiecutter_project_builder}) env.Cookiecutter(AlwaysBuild(Dir('example')), 'cookiecutter.json') + +# Run cookiecutter to generate the 'example-uws' package +# This represents an IVOA UWS service that does work via an arq queue +# The explicit target directory, with AlwaysBuild, is needed for Scons to +# differentiate this build from the example build above. We can't use Scons' +# default build caching because we'd have to compute the cookiecutter +# template anyways. +env.Cookiecutter(AlwaysBuild(Dir('example-uws')), + 'cookiecutter.json', + cookiecutter_context={'name': 'example-uws', + 'flavor': 'UWS'}) diff --git a/project_templates/fastapi_safir_app/cookiecutter.json b/project_templates/fastapi_safir_app/cookiecutter.json index 6e36597e..a7946c8c 100644 --- a/project_templates/fastapi_safir_app/cookiecutter.json +++ b/project_templates/fastapi_safir_app/cookiecutter.json @@ -16,5 +16,9 @@ "lsst-dm", "lsst-sqre-testing" ], + "flavor": [ + "Default", + "UWS" + ], "_extensions": ["jinja2_time.TimeExtension", "templatekit.TemplatekitExtension"] } diff --git a/project_templates/fastapi_safir_app/example-uws/.dockerignore b/project_templates/fastapi_safir_app/example-uws/.dockerignore new file mode 100644 index 00000000..840aa286 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/.dockerignore @@ -0,0 +1,142 @@ +# VSCode +.vscode/ + +# Everything below this point is a copy of .gitignore. + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# macOS +.DS_Store diff --git a/project_templates/fastapi_safir_app/example-uws/.github/dependabot.yml b/project_templates/fastapi_safir_app/example-uws/.github/dependabot.yml new file mode 100644 index 00000000..dfb90b72 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" diff --git a/project_templates/fastapi_safir_app/example-uws/.github/workflows/ci.yaml b/project_templates/fastapi_safir_app/example-uws/.github/workflows/ci.yaml new file mode 100644 index 00000000..ab34a2db --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/.github/workflows/ci.yaml @@ -0,0 +1,87 @@ +name: CI + +env: + # Current supported Python version. For applications, there is generally no + # reason to support multiple Python versions, so all actions are run with + # this version. Quote the version to avoid interpretation as a floating + # point number. + PYTHON_VERSION: "3.12" + +"on": + merge_group: {} + pull_request: {} + push: + branches-ignore: + # These should always correspond to pull requests, so ignore them for + # the push trigger and let them be triggered by the pull_request + # trigger, avoiding running the workflow twice. This is a minor + # optimization so there's no need to ensure this is comprehensive. + - "dependabot/**" + - "gh-readonly-queue/**" + - "renovate/**" + - "tickets/**" + - "u/**" + tags: + - "*" + +jobs: + lint: + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 + + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v4 + + - name: Run tox + uses: lsst-sqre/run-tox@v1 + with: + python-version: ${{ env.PYTHON_VERSION }} + tox-envs: "py,coverage-report,typing" + tox-requirements: requirements/tox.txt + + build: + runs-on: ubuntu-latest + needs: [lint, test] + timeout-minutes: 10 + + # Only do Docker builds of tagged releases and pull requests from ticket + # branches. This will still trigger on pull requests from untrusted + # repositories whose branch names match our tickets/* branch convention, + # but in this case the build will fail with an error since the secret + # won't be set. + if: > + github.event_name != 'merge_group' + && (startsWith(github.ref, 'refs/tags/') + || startsWith(github.head_ref, 'tickets/')) + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: lsst-sqre/build-and-push-to-ghcr@v1 + id: build + with: + image: ${{ github.repository }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - uses: lsst-sqre/build-and-push-to-ghcr@v1 + id: build-worker + with: + image: ${{ github.repository }}-worker + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/project_templates/fastapi_safir_app/example-uws/.github/workflows/periodic-ci.yaml b/project_templates/fastapi_safir_app/example-uws/.github/workflows/periodic-ci.yaml new file mode 100644 index 00000000..65dd8b4a --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/.github/workflows/periodic-ci.yaml @@ -0,0 +1,57 @@ +# This is a separate run of the Python test suite that runs from a schedule, +# doesn't cache the tox environment, and updates pinned dependencies first. +# The purpose is to test compatibility with the latest versions of +# dependencies. + +name: Periodic CI + +env: + # Current supported Python version. For applications, there is generally no + # reason to support multiple Python versions, so all actions are run with + # this version. Quote the version to avoid interpretation as a floating + # point number. + PYTHON_VERSION: "3.12" + +"on": + schedule: + - cron: "0 12 * * 1" + workflow_dispatch: {} + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Update dependencies + run: | + pip install --upgrade uv + uv venv + source .venv/bin/activate + make update-deps + shell: bash + + - name: Run tests in tox + uses: lsst-sqre/run-tox@v1 + with: + python-version: ${{ env.PYTHON_VERSION }} + tox-envs: "lint,typing,py" + tox-requirements: requirements/tox.txt + use-cache: false + + - name: Report status + if: failure() + uses: ravsamhq/notify-slack-action@v2 + with: + status: ${{ job.status }} + notify_when: "failure" + notification_title: "Periodic test for {repo} failed" + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }} diff --git a/project_templates/fastapi_safir_app/example-uws/.gitignore b/project_templates/fastapi_safir_app/example-uws/.gitignore new file mode 100644 index 00000000..7af665ba --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/.gitignore @@ -0,0 +1,137 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# macOS +.DS_Store diff --git a/project_templates/fastapi_safir_app/example-uws/.pre-commit-config.yaml b/project_templates/fastapi_safir_app/example-uws/.pre-commit-config.yaml new file mode 100644 index 00000000..d302149e --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/.pre-commit-config.yaml @@ -0,0 +1,14 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-toml + - id: check-yaml + - id: trailing-whitespace + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.5 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format diff --git a/project_templates/fastapi_safir_app/example-uws/CHANGELOG.md b/project_templates/fastapi_safir_app/example-uws/CHANGELOG.md new file mode 100644 index 00000000..0490886f --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/CHANGELOG.md @@ -0,0 +1,8 @@ +# Change log + +example-uws is versioned with [semver](https://semver.org/). +Dependencies are updated to the latest available version during each release, and aren't noted here. + +Find changes for the upcoming release in the project's [changelog.d directory](https://github.com/lsst-sqre/example-uws/tree/main/changelog.d/). + + diff --git a/project_templates/fastapi_safir_app/example-uws/Dockerfile b/project_templates/fastapi_safir_app/example-uws/Dockerfile new file mode 100644 index 00000000..48366262 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/Dockerfile @@ -0,0 +1,68 @@ +# This Dockerfile has four stages: +# +# base-image +# Updates the base Python image with security patches and common system +# packages. This image becomes the base of all other images. +# dependencies-image +# Installs third-party dependencies (requirements/main.txt) into a virtual +# environment. This virtual environment is ideal for copying across build +# stages. +# install-image +# Installs the app into the virtual environment. +# runtime-image +# - Copies the virtual environment into place. +# - Runs a non-root user. +# - Sets up the entrypoint and port. + +FROM python:3.12.2-slim-bookworm as base-image + +# Update system packages +COPY scripts/install-base-packages.sh . +RUN ./install-base-packages.sh && rm ./install-base-packages.sh + +FROM base-image AS dependencies-image + +# Install system packages only needed for building dependencies. +COPY scripts/install-dependency-packages.sh . +RUN ./install-dependency-packages.sh + +# Create a Python virtual environment +ENV VIRTUAL_ENV=/opt/venv +RUN python -m venv $VIRTUAL_ENV +# Make sure we use the virtualenv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +# Put the latest pip and setuptools in the virtualenv +RUN pip install --upgrade --no-cache-dir pip setuptools wheel + +# Install the app's Python runtime dependencies +COPY requirements/main.txt ./requirements.txt +RUN pip install --quiet --no-cache-dir -r requirements.txt + +FROM dependencies-image AS install-image + +# Use the virtualenv +ENV PATH="/opt/venv/bin:$PATH" + +COPY . /workdir +WORKDIR /workdir +RUN pip install --no-cache-dir . + +FROM base-image AS runtime-image + +# Create a non-root user +RUN useradd --create-home appuser + +# Copy the virtualenv +COPY --from=install-image /opt/venv /opt/venv + +# Make sure we use the virtualenv +ENV PATH="/opt/venv/bin:$PATH" + +# Switch to the non-root user. +USER appuser + +# Expose the port. +EXPOSE 8080 + +# Run the application. +CMD ["uvicorn", "exampleuws.main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/project_templates/fastapi_safir_app/example-uws/Dockerfile.worker b/project_templates/fastapi_safir_app/example-uws/Dockerfile.worker new file mode 100644 index 00000000..2edae678 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/Dockerfile.worker @@ -0,0 +1,28 @@ + +# This Dockerfile constructs the image for backend workers. These images +# are based on stack containers and install any required supporting code +# for the backend, arq, and the backend worker definition. + +FROM lsstsqre/centos:7-stack-lsst_distrib-w_2024_27 + +# Reset the user to root since we need to do system install tasks. +USER root + +# Upgrade the system packages. +COPY scripts/install-worker-packages.sh . +RUN ./install-worker-packages.sh && rm ./install-worker-packages.sh + +# Install the necessary prerequisites and the vo-cutouts code. +COPY . /workdir +RUN /workdir/scripts/install-worker.sh /workdir && rm -r /workdir +COPY scripts/start-worker.sh / + +# Create a non-root user +RUN useradd --create-home appuser + +# Switch to the non-root user. +USER appuser + +# Start the arq worker. +WORKDIR / +CMD ["/start-worker.sh"] diff --git a/project_templates/fastapi_safir_app/example-uws/LICENSE b/project_templates/fastapi_safir_app/example-uws/LICENSE new file mode 100644 index 00000000..b7acc140 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Association of Universities for Research in Astronomy, Inc. (AURA) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/project_templates/fastapi_safir_app/example-uws/MANIFEST.in b/project_templates/fastapi_safir_app/example-uws/MANIFEST.in new file mode 100644 index 00000000..e69de29b diff --git a/project_templates/fastapi_safir_app/example-uws/Makefile b/project_templates/fastapi_safir_app/example-uws/Makefile new file mode 100644 index 00000000..dbcb745c --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/Makefile @@ -0,0 +1,48 @@ +.PHONY: help +help: + @echo "Make targets for example-uws" + @echo "make init - Set up dev environment" + @echo "make run - Start a local development instance" + @echo "make update - Update pinned dependencies and run make init" + @echo "make update-deps - Update pinned dependencies" + @echo "make update-deps-no-hashes - Pin dependencies without hashes" + +.PHONY: init +init: + pip install --upgrade uv + uv pip install -r requirements/main.txt -r requirements/dev.txt \ + -r requirements/tox.txt + uv pip install --editable . + rm -rf .tox + uv pip install --upgrade pre-commit + pre-commit install + +.PHONY: run +run: + tox run -e run + +.PHONY: update +update: update-deps init + +.PHONY: update-deps +update-deps: + pip install --upgrade uv + uv pip install --upgrade pre-commit + pre-commit autoupdate + uv pip compile --upgrade --generate-hashes \ + --output-file requirements/main.txt requirements/main.in + uv pip compile --upgrade --generate-hashes \ + --output-file requirements/dev.txt requirements/dev.in + uv pip compile --upgrade --generate-hashes \ + --output-file requirements/tox.txt requirements/tox.in + +# Useful for testing against a Git version of Safir. +.PHONY: update-deps-no-hashes +update-deps-no-hashes: + pip install --upgrade uv + uv pip compile --upgrade \ + --output-file requirements/main.txt requirements/main.in + uv pip compile --upgrade \ + --output-file requirements/dev.txt requirements/dev.in + uv pip compile --upgrade \ + --output-file requirements/tox.txt requirements/tox.in diff --git a/project_templates/fastapi_safir_app/example-uws/README.md b/project_templates/fastapi_safir_app/example-uws/README.md new file mode 100644 index 00000000..8f514585 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/README.md @@ -0,0 +1,6 @@ +# example-uws + +Short one-sentence summary of the app +Learn more at https://example-uws.lsst.io + +example-uws is developed with [FastAPI](https://fastapi.tiangolo.com) and [Safir](https://safir.lsst.io). diff --git a/project_templates/fastapi_safir_app/example-uws/changelog.d/_template.md.jinja b/project_templates/fastapi_safir_app/example-uws/changelog.d/_template.md.jinja new file mode 100644 index 00000000..6e644b85 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/changelog.d/_template.md.jinja @@ -0,0 +1,7 @@ + +{%- for cat in config.categories %} + +### {{ cat }} + +- +{%- endfor %} diff --git a/project_templates/fastapi_safir_app/example-uws/pyproject.toml b/project_templates/fastapi_safir_app/example-uws/pyproject.toml new file mode 100644 index 00000000..52b5f14d --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/pyproject.toml @@ -0,0 +1,112 @@ +[project] +# https://packaging.python.org/en/latest/specifications/declaring-project-metadata/ +name = "example-uws" +description = "Short one-sentence summary of the app" +license = { file = "LICENSE" } +readme = "README.md" +keywords = ["rubin", "lsst"] +# https://pypi.org/classifiers/ +classifiers = [ + "Development Status :: 4 - Beta", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Intended Audience :: Developers", + "Natural Language :: English", + "Operating System :: POSIX", + "Typing :: Typed", +] +requires-python = ">=3.12" +# Use requirements/main.in for runtime dependencies instead. +dependencies = [] +dynamic = ["version"] + +[project.scripts] +example-uws = "exampleuws.cli:main" + +[project.urls] +Homepage = "https://example-uws.lsst.io" +Source = "https://github.com/lsst-sqre/example-uws" + +[build-system] +requires = ["setuptools>=61", "wheel", "setuptools_scm[toml]>=6.2"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] + +[tool.coverage.run] +parallel = true +branch = true +source = ["exampleuws"] + +[tool.coverage.paths] +source = ["src", ".tox/*/site-packages"] + +[tool.coverage.report] +show_missing = true +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "if self.debug:", + "if settings.DEBUG", + "raise AssertionError", + "raise NotImplementedError", + "if 0:", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.mypy] +disallow_untyped_defs = true +disallow_incomplete_defs = true +ignore_missing_imports = true +local_partial_types = true +plugins = ["pydantic.mypy"] +no_implicit_reexport = true +show_error_codes = true +strict_equality = true +warn_redundant_casts = true +warn_unreachable = true +warn_unused_ignores = true + +[tool.pydantic-mypy] +init_forbid_extra = true +init_typed = true +warn_required_dynamic_aliases = true +warn_untyped_fields = true + +[tool.pytest.ini_options] +asyncio_mode = "strict" +# The python_files setting is not for test detection (pytest will pick up any +# test files named *_test.py without this setting) but to enable special +# assert processing in any non-test supporting files under tests. We +# conventionally put test support functions under tests.support and may +# sometimes use assert in test fixtures in conftest.py, and pytest only +# enables magical assert processing (showing a full diff on assert failures +# with complex data structures rather than only the assert message) in files +# listed in python_files. +python_files = ["tests/*.py", "tests/*/*.py"] + +# Use the generic Ruff configuration in ruff.toml and extend it with only +# project-specific settings. Add a [tool.ruff.lint.extend-per-file-ignores] +# section for project-specific ignore rules. +[tool.ruff] +extend = "ruff-shared.toml" + +[tool.ruff.lint.isort] +known-first-party = ["exampleuws", "tests"] +split-on-trailing-comma = false + +[tool.scriv] +categories = [ + "Backwards-incompatible changes", + "New features", + "Bug fixes", + "Other changes", +] +entry_title_template = "{{ version }} ({{ date.strftime('%Y-%m-%d') }})" +format = "md" +md_header_level = "2" +new_fragment_template = "file:changelog.d/_template.md.jinja" +skip_fragments = "_template.md.jinja" diff --git a/project_templates/fastapi_safir_app/example-uws/requirements/dev.in b/project_templates/fastapi_safir_app/example-uws/requirements/dev.in new file mode 100644 index 00000000..5d8619b7 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/requirements/dev.in @@ -0,0 +1,23 @@ +# -*- conf -*- +# +# Editable development dependencies +# Add direct development, test, and documentation dependencies here, as well +# as implicit dev dependencies with constrained versions. +# +# After editing, update requirements/dev.txt by running: +# make update-deps + +-c main.txt + +# Testing +asgi-lifespan +coverage[toml] +httpx +mypy +pydantic +pytest +pytest-asyncio +pytest-cov + +# Documentation +scriv diff --git a/project_templates/fastapi_safir_app/example-uws/requirements/main.in b/project_templates/fastapi_safir_app/example-uws/requirements/main.in new file mode 100644 index 00000000..9a2f9b13 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/requirements/main.in @@ -0,0 +1,18 @@ +# -*- conf -*- +# +# Editable runtime dependencies (equivalent to install_requires) +# Add direct runtime dependencies here, as well as implicit dependencies +# with constrained versions. +# +# After editing, update requirements/main.txt by running: +# make update-deps + +# These dependencies are for fastapi including some optional features. +fastapi +starlette +uvicorn[standard] + +# Other dependencies. +pydantic +pydantic-settings +safir[uws]>=6.2.0 diff --git a/project_templates/fastapi_safir_app/example-uws/requirements/tox.in b/project_templates/fastapi_safir_app/example-uws/requirements/tox.in new file mode 100644 index 00000000..c525b25b --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/requirements/tox.in @@ -0,0 +1,15 @@ +# -*- conf -*- +# +# Editable tox dependencies +# Add tox and its plugins here. These will be installed in the user's venv for +# local development and by CI when running tox actions. +# +# After editing, update requirements/dev.txt by running: +# make update-deps + +-c main.txt +-c dev.txt + +tox +tox-docker +tox-uv diff --git a/project_templates/fastapi_safir_app/example-uws/ruff-shared.toml b/project_templates/fastapi_safir_app/example-uws/ruff-shared.toml new file mode 100644 index 00000000..da644fb3 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/ruff-shared.toml @@ -0,0 +1,126 @@ +# Generic shared Ruff configuration file. It should be possible to use this +# file unmodified in different packages provided that one likes the style that +# it enforces. +# +# This file should be used from pyproject.toml as follows: +# +# [tool.ruff] +# extend = "ruff-shared.toml" +# +# It can then be extended with project-specific rules. A common additional +# setting in pyproject.toml is tool.ruff.lint.extend-per-file-ignores, to add +# additional project-specific ignore rules for specific paths. +# +# The rule used with Ruff configuration is to disable every non-deprecated +# lint rule that has legitimate exceptions that are not dodgy code, rather +# than cluttering code with noqa markers. This is therefore a reiatively +# relaxed configuration that errs on the side of disabling legitimate rules. +# +# Reference for settings: https://docs.astral.sh/ruff/settings/ +# Reference for rules: https://docs.astral.sh/ruff/rules/ +exclude = ["docs/**"] +line-length = 79 +target-version = "py312" + +[format] +docstring-code-format = true + +[lint] +ignore = [ + "ANN401", # sometimes Any is the right type + "ARG001", # unused function arguments are often legitimate + "ARG002", # unused method arguments are often legitimate + "ARG003", # unused class method arguments are often legitimate + "ARG005", # unused lambda arguments are often legitimate + "BLE001", # we want to catch and report Exception in background tasks + "C414", # nested sorted is how you sort by multiple keys with reverse + "D102", # sometimes we use docstring inheritence + "D104", # don't see the point of documenting every package + "D105", # our style doesn't require docstrings for magic methods + "D106", # Pydantic uses a nested Config class that doesn't warrant docs + "D205", # our documentation style allows a folded first line + "EM101", # justification (duplicate string in traceback) is silly + "EM102", # justification (duplicate string in traceback) is silly + "FBT003", # positional booleans are normal for Pydantic field defaults + "FIX002", # point of a TODO comment is that we're not ready to fix it + "G004", # forbidding logging f-strings is appealing, but not our style + "RET505", # disagree that omitting else always makes code more readable + "PLR0911", # often many returns is clearer and simpler style + "PLR0913", # factory pattern uses constructors with many arguments + "PLR2004", # too aggressive about magic values + "PLW0603", # yes global is discouraged but if needed, it's needed + "S105", # good idea but too many false positives on non-passwords + "S106", # good idea but too many false positives on non-passwords + "S107", # good idea but too many false positives on non-passwords + "S603", # not going to manually mark every subprocess call as reviewed + "S607", # using PATH is not a security vulnerability + "SIM102", # sometimes the formatting of nested if statements is clearer + "SIM117", # sometimes nested with contexts are clearer + "TCH001", # we decided to not maintain separate TYPE_CHECKING blocks + "TCH002", # we decided to not maintain separate TYPE_CHECKING blocks + "TCH003", # we decided to not maintain separate TYPE_CHECKING blocks + "TD003", # we don't require issues be created for TODOs + "TID252", # if we're going to use relative imports, use them always + "TRY003", # good general advice but lint is way too aggressive + "TRY301", # sometimes raising exceptions inside try is the best flow + "UP040", # PEP 695 type aliases not yet supported by mypy + + # The following settings should be disabled when using ruff format + # per https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules + "W191", + "E111", + "E114", + "E117", + "D206", + "D300", + "Q000", + "Q001", + "Q002", + "Q003", + "COM812", + "COM819", + "ISC001", + "ISC002", + + # Temporary bug workarounds. + "S113", # https://github.com/astral-sh/ruff/issues/12210 +] +select = ["ALL"] + +[lint.per-file-ignores] +"src/*/handlers/**" = [ + "D103", # FastAPI handlers should not have docstrings +] +"tests/**" = [ + "C901", # tests are allowed to be complex, sometimes that's convenient + "D101", # tests don't need docstrings + "D103", # tests don't need docstrings + "PLR0915", # tests are allowed to be long, sometimes that's convenient + "PT012", # way too aggressive about limiting pytest.raises blocks + "S101", # tests should use assert + "S106", # tests are allowed to hard-code dummy passwords + "S301", # allow tests for whether code can be pickled + "SLF001", # tests are allowed to access private members +] + +# These are too useful as attributes or methods to allow the conflict with the +# built-in to rule out their use. +[lint.flake8-builtins] +builtins-ignorelist = [ + "all", + "any", + "help", + "id", + "list", + "type", +] + +[lint.flake8-pytest-style] +fixture-parentheses = false +mark-parentheses = false + +[lint.mccabe] +max-complexity = 11 + +[lint.pydocstyle] +convention = "numpy" diff --git a/project_templates/fastapi_safir_app/example-uws/scripts/install-base-packages.sh b/project_templates/fastapi_safir_app/example-uws/scripts/install-base-packages.sh new file mode 100755 index 00000000..620781c1 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/scripts/install-base-packages.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# This script updates packages in the base Docker image that's used by both the +# build and runtime images, and gives us a place to install additional +# system-level packages with apt-get. +# +# Based on the blog post: +# https://pythonspeed.com/articles/system-packages-docker/ + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for +# details. +set -euo pipefail + +# Display each command as it's run. +set -x + +# Tell apt-get we're never going to be able to give manual +# feedback: +export DEBIAN_FRONTEND=noninteractive + +# Update the package listing, so we know what packages exist: +apt-get update + +# Install security updates: +apt-get -y upgrade + +# Example of installing a new package, without unnecessary packages: +apt-get -y install --no-install-recommends git + +# Delete cached files we don't need anymore: +apt-get clean +rm -rf /var/lib/apt/lists/* diff --git a/project_templates/fastapi_safir_app/example-uws/scripts/install-dependency-packages.sh b/project_templates/fastapi_safir_app/example-uws/scripts/install-dependency-packages.sh new file mode 100755 index 00000000..f63ef751 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/scripts/install-dependency-packages.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# This script installs additional packages used by the dependency image but +# not needed by the runtime image, such as additional packages required to +# build Python dependencies. +# +# Since the base image wipes all the apt caches to clean up the image that +# will be reused by the runtime image, we unfortunately have to do another +# apt-get update here, which wastes some time and network. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for +# details. +set -euo pipefail + +# Display each command as it's run. +set -x + +# Tell apt-get we're never going to be able to give manual +# feedback: +export DEBIAN_FRONTEND=noninteractive + +# Update the package listing, so we know what packages exist: +apt-get update + +# Install build-essential because sometimes Python dependencies need to build +# C modules, particularly when upgrading to newer Python versions. libffi-dev +# is sometimes needed to build cffi (a cryptography dependency). +apt-get -y install --no-install-recommends build-essential libffi-dev + +# Delete cached files we don't need anymore: +apt-get clean +rm -rf /var/lib/apt/lists/* diff --git a/project_templates/fastapi_safir_app/example-uws/scripts/install-worker-packages.sh b/project_templates/fastapi_safir_app/example-uws/scripts/install-worker-packages.sh new file mode 100644 index 00000000..2cec6fb0 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/scripts/install-worker-packages.sh @@ -0,0 +1,23 @@ + +#!/bin/bash + +# Install or upgrade any operating system packages needed on worker images. +# This is done in a separate script to create a separate cached Docker image, +# which will help with iteration speed on the more interesting setup actions +# taken later in the build. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for details. +set -euo pipefail + +# Display each command as it's run. +set -x + +# Upgrade the Red Hat packages. +# +# TODO(rra): Disabled for now because the version of CentOS used by the image +# is so old that the package repositories no longer exist. This will in theory +# soon be fixed by basing the image on AlmaLinux. +#yum -y upgrade +#yum clean all diff --git a/project_templates/fastapi_safir_app/example-uws/scripts/install-worker.sh b/project_templates/fastapi_safir_app/example-uws/scripts/install-worker.sh new file mode 100644 index 00000000..825b9a4c --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/scripts/install-worker.sh @@ -0,0 +1,27 @@ + +#!/bin/bash + +# This script updates and installs the necessary prerequisites for a backend +# worker starting with a stack container. It takes one parameter, the +# directory in which to do the installation. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for details. +# set -u is omitted because the setup bash function does not support it. +set -eo pipefail + +# Enable the stack. This should be done before set -x because it will +# otherwise spew a bunch of nonsense no one cares about into the logs. +source /opt/lsst/software/stack/loadLSST.bash +setup lsst_distrib + +# Display each command as it's run. +set -x + +# Install Python dependencies and the example-uws code. httpx is +# required by the remote Butler client, and google-cloud-storage is used to +# store cutouts when the bucket has a gs URL. +cd "$1" +pip install --no-cache-dir google-cloud-storage httpx safir-arq structlog +pip install --no-cache-dir --no-deps . diff --git a/project_templates/fastapi_safir_app/example-uws/scripts/start-worker.sh b/project_templates/fastapi_safir_app/example-uws/scripts/start-worker.sh new file mode 100644 index 00000000..14e44bd5 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/scripts/start-worker.sh @@ -0,0 +1,18 @@ + +#!/bin/bash + +# This script is installed in the worker image and starts the backend worker +# using arq. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for details. +# set -u is omitted because the setup bash function does not support it. +set -eo pipefail + +# Initialize the environment. +source /opt/lsst/software/stack/loadLSST.bash +setup lsst_distrib + +# Start arq with the worker. +arq exampleuws.workers.exampleuws.WorkerSettings diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/__init__.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/__init__.py new file mode 100644 index 00000000..922936c8 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/__init__.py @@ -0,0 +1,14 @@ +"""The example-uws service.""" + +__all__ = ["__version__"] + +from importlib.metadata import PackageNotFoundError, version + +__version__: str +"""The application version string (PEP 440 / SemVer compatible).""" + +try: + __version__ = version("example-uws") +except PackageNotFoundError: + # package is not installed + __version__ = "0.0.0" diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/cli.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/cli.py new file mode 100644 index 00000000..5c95ea76 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/cli.py @@ -0,0 +1,42 @@ + +"""Administrative command-line interface.""" + +from __future__ import annotations + +import click +import structlog +from safir.asyncio import run_with_asyncio +from safir.click import display_help + +from .config import uws + +__all__ = [ + "help", + "init", + "main", +] + + +@click.group(context_settings={"help_option_names": ["-h", "--help"]}) +@click.version_option(message="%(version)s") +def main() -> None: + """Administrative command-line interface for example-uws.""" + + +@main.command() +@click.argument("topic", default=None, required=False, nargs=1) +@click.pass_context +def help(ctx: click.Context, topic: str | None) -> None: + """Show help for any command.""" + display_help(main, ctx, topic) + + +@main.command() +@click.option( + "--reset", is_flag=True, help="Delete all existing database data." +) +@run_with_asyncio +async def init(*, reset: bool) -> None: + """Initialize the database storage.""" + logger = structlog.get_logger("exampleuws") + await uws.initialize_uws_database(logger, reset=reset) diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/config.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/config.py new file mode 100644 index 00000000..10726c01 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/config.py @@ -0,0 +1,55 @@ +"""Configuration definition.""" + +from __future__ import annotations + +from pydantic import Field +from pydantic_settings import SettingsConfigDict +from safir.logging import LogLevel, Profile +from safir.uws import UWSApplication, UWSAppSettings, UWSConfig, UWSRoute + +from .dependencies import post_params_dependency +from .models import ExampleuwsParameters + +__all__ = ["Config", "config"] + + +class Config(UWSAppSettings): + """Configuration for example-uws.""" + + name: str = Field("example-uws", title="Name of application") + + path_prefix: str = Field( + "/example-uws", title="URL prefix for application" + ) + + profile: Profile = Field( + Profile.development, title="Application logging profile" + ) + + log_level: LogLevel = Field( + LogLevel.INFO, title="Log level of the application's logger" + ) + + model_config = SettingsConfigDict( + env_prefix="EXAMPLE_UWS_", case_sensitive=False + ) + + @property + def uws_config(self) -> UWSConfig: + """Corresponding configuration for the UWS subsystem.""" + return self.build_uws_config( + parameters_type=ExampleuwsParameters, + worker="example_uws", + async_post_route=UWSRoute( + dependency=post_params_dependency, + summary="Create async example-uws job", + description="Create a new UWS job for example-uws", + ), + ) + + +config = Config() +"""Configuration for example-uws.""" + +uws = UWSApplication(config.uws_config) +"""The UWS application for this service.""" diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/dependencies.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/dependencies.py new file mode 100644 index 00000000..35bbf034 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/dependencies.py @@ -0,0 +1,28 @@ + +"""Job parameter dependencies.""" + +from typing import Annotated + +from fastapi import Depends, Request +from safir.uws import UWSJobParameter, uws_post_params_dependency + +__all__ = [ + "post_params_dependency", +] + + +async def post_params_dependency( + *, + # Add POST parameters here. All of them should be Form() parameters. + # Use str | None for single-valued attributes and str | list[str] | None + # for parameters that can be given more than one time. + params: Annotated[ + list[UWSJobParameter], Depends(uws_post_params_dependency) + ], +) -> list[UWSJobParameter]: + """Parse POST parameters into job parameters.""" + return [ + p + for p in params + if p.parameter_id in set() # Replace with set of parameter names + ] diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/domain.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/domain.py new file mode 100644 index 00000000..5c17e073 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/domain.py @@ -0,0 +1,15 @@ + +"""Domain models for example-uws.""" + +from __future__ import annotations + +from pydantic import BaseModel + +__all__ = ["WorkerExampleuwsModel"] + + +class WorkerExampleuwsModel(BaseModel): + """Parameter model for backend workers. + + Add fields here for all the parameters passed to backend jobs. + """ diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/__init__.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/external.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/external.py new file mode 100644 index 00000000..33b57523 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/external.py @@ -0,0 +1,52 @@ +"""Handlers for the app's external root, ``/example-uws/``.""" + +from typing import Annotated + +from fastapi import APIRouter, Depends +from safir.dependencies.logger import logger_dependency +from safir.metadata import get_metadata +from structlog.stdlib import BoundLogger + +from ..config import config +from ..models import Index + +__all__ = ["get_index", "external_router"] + +external_router = APIRouter() +"""FastAPI router for all external handlers.""" + + +@external_router.get( + "/", + description=( + "Document the top-level API here. By default it only returns metadata" + " about the application." + ), + response_model=Index, + response_model_exclude_none=True, + summary="Application metadata", +) +async def get_index( + logger: Annotated[BoundLogger, Depends(logger_dependency)], +) -> Index: + """GET ``/example-uws/`` (the app's external root). + + Customize this handler to return whatever the top-level resource of your + application should return. For example, consider listing key API URLs. + When doing so, also change or customize the response model in + `exampleuws.models.Index`. + + By convention, the root of the external API includes a field called + ``metadata`` that provides the same Safir-generated metadata as the + internal root endpoint. + """ + # There is no need to log simple requests since uvicorn will do this + # automatically, but this is included as an example of how to use the + # logger for more complex logging. + logger.info("Request for application metadata") + + metadata = get_metadata( + package_name="example-uws", + application_name=config.name, + ) + return Index(metadata=metadata) diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/internal.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/internal.py new file mode 100644 index 00000000..ae1f5b19 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/handlers/internal.py @@ -0,0 +1,42 @@ +"""Internal HTTP handlers that serve relative to the root path, ``/``. + +These handlers aren't externally visible since the app is available at a path, +``/example-uws``. See `exampleuws.handlers.external` for +the external endpoint handlers. + +These handlers should be used for monitoring, health checks, internal status, +or other information that should not be visible outside the Kubernetes cluster. +""" + +from fastapi import APIRouter +from safir.metadata import Metadata, get_metadata + +from ..config import config + +__all__ = ["get_index", "internal_router"] + +internal_router = APIRouter() +"""FastAPI router for all internal handlers.""" + + +@internal_router.get( + "/", + description=( + "Return metadata about the running application. Can also be used as" + " a health check. This route is not exposed outside the cluster and" + " therefore cannot be used by external clients." + ), + include_in_schema=False, + response_model=Metadata, + response_model_exclude_none=True, + summary="Application metadata", +) +async def get_index() -> Metadata: + """GET ``/`` (the app's internal root). + + By convention, this endpoint returns only the application's metadata. + """ + return get_metadata( + package_name="example-uws", + application_name=config.name, + ) diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/main.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/main.py new file mode 100644 index 00000000..f7bbcf16 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/main.py @@ -0,0 +1,67 @@ +"""The main application factory for the example-uws service. + +Notes +----- +Be aware that, following the normal pattern for FastAPI services, the app is +constructed when this module is loaded and is not deferred until a function is +called. +""" + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from importlib.metadata import metadata, version + +from fastapi import FastAPI +from safir.dependencies.http_client import http_client_dependency +from safir.logging import configure_logging, configure_uvicorn_logging +from safir.middleware.x_forwarded import XForwardedMiddleware + +from .config import config, uws +from .handlers.external import external_router +from .handlers.internal import internal_router + +__all__ = ["app"] + + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncIterator[None]: + """Set up and tear down the application.""" + # Any code here will be run when the application starts up. + await uws.initialize_fastapi() + + yield + + # Any code here will be run when the application shuts down. + await uws.shutdown_fastapi() + await http_client_dependency.aclose() + + +configure_logging( + profile=config.profile, + log_level=config.log_level, + name="exampleuws", +) +configure_uvicorn_logging(config.log_level) + +app = FastAPI( + title="example-uws", + description=metadata("example-uws")["Summary"], + version=version("example-uws"), + openapi_url=f"{config.path_prefix}/openapi.json", + docs_url=f"{config.path_prefix}/docs", + redoc_url=f"{config.path_prefix}/redoc", + lifespan=lifespan, +) +"""The main FastAPI application for example-uws.""" + +# Attach the routers. +app.include_router(internal_router) +uws.install_handlers(external_router) +app.include_router(external_router, prefix=f"{config.path_prefix}") + +# Add middleware. +app.add_middleware(XForwardedMiddleware) +uws.install_middleware(app) + +# Install error handlers. +uws.install_error_handlers(app) diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/models.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/models.py new file mode 100644 index 00000000..df4b159a --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/models.py @@ -0,0 +1,41 @@ +"""Models for example-uws.""" + +from typing import Self + +from pydantic import BaseModel, Field +from safir.metadata import Metadata as SafirMetadata +from safir.uws import ParametersModel, UWSJobParameter + +from .domain import WorkerExampleuwsModel + +__all__ = ["Index"] + + +class Index(BaseModel): + """Metadata returned by the external root URL of the application. + + Notes + ----- + As written, this is not very useful. Add additional metadata that will be + helpful for a user exploring the application, or replace this model with + some other model that makes more sense to return from the application API + root. + """ + + metadata: SafirMetadata = Field(..., title="Package metadata") + + +class ExampleuwsParameters(ParametersModel[WorkerExampleuwsModel]): + """Model for job parameters. + + Add fields here for all the input parameters to a job, and then update + ``from_job_parameters`` and ``to_worker_parameters`` to do the appropriate + conversions. + """ + + @classmethod + def from_job_parameters(cls, params: list[UWSJobParameter]) -> Self: + return cls() + + def to_worker_parameters(self) -> WorkerExampleuwsModel: + return WorkerExampleuwsModel() diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/__init__.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/exampleuws.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/exampleuws.py new file mode 100644 index 00000000..ff8f0d26 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/exampleuws.py @@ -0,0 +1,97 @@ +"""Worker for example-uws. + +This is a standalone file intended to be injected into a stack container as +the arq worker definition. Only this module is allowed to use stack packages. +""" + +from __future__ import annotations + +import os +from datetime import timedelta + +import structlog +from safir.arq import ArqMode +from safir.arq.uws import ( + WorkerConfig, + WorkerFatalError, + WorkerJobInfo, + WorkerResult, + build_worker, +) +from structlog.stdlib import BoundLogger + +from ..domain import WorkerExampleuwsModel + +__all__ = ["WorkerSettings"] + + +def example_uws( + params: WorkerExampleuwsModel, info: WorkerJobInfo, logger: BoundLogger +) -> list[WorkerResult]: + """Perform the work. + + This is a queue worker for the example-uws service. It takes a serialized + request, converts it into a suitable in-memory format, and then dispatches + it to the scientific code that performs the cutout. The results are stored + in a GCS bucket, and the details of the output are returned as the result + of the worker. + + Parameters + ---------- + params + Cutout parameters. + info + Information about the UWS job we're executing. + logger + Logger to use for logging. + + Returns + ------- + list of WorkerResult + Results of the job. + + Raises + ------ + WorkerFatalError + Raised if the cutout failed for unknown reasons, or due to internal + errors. This is the normal failure exception, since we usually do not + know why the backend code failed and make the pessimistic assumption + that the failure is not transient. + WorkerUsageError + Raised if the cutout failed due to deficiencies in the parameters + submitted by the user that could not be detected by the frontend + service. + """ + logger.info("Starting request") + try: + # Replace this with a call to the function that does the work. + result_url = "https://example.com/" + except Exception as e: + raise WorkerFatalError(f"{type(e).__name__}: {e!s}") from e + logger.info("Request successful") + + # Change the result ID to something reasonable, set the MIME type to an + # appropriate value, and ideally also set the result size if that's + # available. + return [ + WorkerResult( + result_id="main", url=result_url, mime_type="application/fits" + ) + ] + + +WorkerSettings = build_worker( + example_uws, + WorkerConfig( + arq_mode=ArqMode.production, + arq_queue_url=os.environ["EXAMPLE_UWS_ARQ_QUEUE_URL"], + arq_queue_password=os.getenv("EXAMPLE_UWS_ARQ_QUEUE_PASSWORD"), + grace_period=timedelta( + seconds=int(os.environ["EXAMPLE_UWS_GRACE_PERIOD"]) + ), + parameters_class=WorkerExampleuwsModel, + timeout=timedelta(seconds=int(os.environ["EXAMPLE_UWS_TIMEOUT"])), + ), + structlog.get_logger("exampleuws"), +) +"""arq configuration for the example-uws worker.""" diff --git a/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/uws.py b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/uws.py new file mode 100644 index 00000000..910eddc1 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/src/exampleuws/workers/uws.py @@ -0,0 +1,19 @@ + +"""Worker for UWS database updates.""" + +from __future__ import annotations + +import structlog +from safir.logging import configure_logging + +from ..config import config, uws + +__all__ = ["WorkerSettings"] + + +configure_logging( + name="exampleuws", profile=config.profile, log_level=config.log_level +) + +WorkerSettings = uws.build_worker(structlog.get_logger("exampleuws")) +"""arq configuration for the UWS database worker.""" diff --git a/project_templates/fastapi_safir_app/example-uws/tests/__init__.py b/project_templates/fastapi_safir_app/example-uws/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/project_templates/fastapi_safir_app/example-uws/tests/conftest.py b/project_templates/fastapi_safir_app/example-uws/tests/conftest.py new file mode 100644 index 00000000..e53d9049 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/tests/conftest.py @@ -0,0 +1,63 @@ +"""Test fixtures for example-uws tests.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator, Iterator +from datetime import timedelta + +import pytest +import pytest_asyncio +import structlog +from asgi_lifespan import LifespanManager +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient +from safir.arq import MockArqQueue +from safir.testing.gcs import MockStorageClient, patch_google_storage +from safir.testing.uws import MockUWSJobRunner + +from exampleuws import main +from exampleuws.config import config, uws + + +@pytest_asyncio.fixture +async def app(arq_queue: MockArqQueue) -> AsyncIterator[FastAPI]: + """Return a configured test application. + + Wraps the application in a lifespan manager so that startup and shutdown + events are sent during test execution. + """ + logger = structlog.get_logger("exampleuws") + await uws.initialize_uws_database(logger, reset=True) + uws.override_arq_queue(arq_queue) + async with LifespanManager(main.app): + yield main.app + + +@pytest.fixture +def arq_queue() -> MockArqQueue: + return MockArqQueue() + + +@pytest_asyncio.fixture +async def client(app: FastAPI) -> AsyncIterator[AsyncClient]: + """Return an ``httpx.AsyncClient`` configured to talk to the test app.""" + async with AsyncClient( + transport=ASGITransport(app=app), # type: ignore[arg-type] + base_url="https://example.com/", + ) as client: + yield client + + +@pytest.fixture(autouse=True) +def mock_google_storage() -> Iterator[MockStorageClient]: + yield from patch_google_storage( + expected_expiration=timedelta(minutes=15), bucket_name="some-bucket" + ) + + +@pytest_asyncio.fixture +async def runner( + arq_queue: MockArqQueue, +) -> AsyncIterator[MockUWSJobRunner]: + async with MockUWSJobRunner(config.uws_config, arq_queue) as runner: + yield runner diff --git a/project_templates/fastapi_safir_app/example-uws/tests/handlers/__init__.py b/project_templates/fastapi_safir_app/example-uws/tests/handlers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/project_templates/fastapi_safir_app/example-uws/tests/handlers/external_test.py b/project_templates/fastapi_safir_app/example-uws/tests/handlers/external_test.py new file mode 100644 index 00000000..af1b2d26 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/tests/handlers/external_test.py @@ -0,0 +1,22 @@ +"""Tests for the exampleuws.handlers.external module and routes.""" + +from __future__ import annotations + +import pytest +from httpx import AsyncClient + +from exampleuws.config import config + + +@pytest.mark.asyncio +async def test_get_index(client: AsyncClient) -> None: + """Test ``GET /example-uws/``.""" + response = await client.get("/example-uws/") + assert response.status_code == 200 + data = response.json() + metadata = data["metadata"] + assert metadata["name"] == config.name + assert isinstance(metadata["version"], str) + assert isinstance(metadata["description"], str) + assert isinstance(metadata["repository_url"], str) + assert isinstance(metadata["documentation_url"], str) diff --git a/project_templates/fastapi_safir_app/example-uws/tests/handlers/internal_test.py b/project_templates/fastapi_safir_app/example-uws/tests/handlers/internal_test.py new file mode 100644 index 00000000..85bfd89d --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/tests/handlers/internal_test.py @@ -0,0 +1,21 @@ +"""Tests for the exampleuws.handlers.internal module and routes.""" + +from __future__ import annotations + +import pytest +from httpx import AsyncClient + +from exampleuws.config import config + + +@pytest.mark.asyncio +async def test_get_index(client: AsyncClient) -> None: + """Test ``GET /``.""" + response = await client.get("/") + assert response.status_code == 200 + data = response.json() + assert data["name"] == config.name + assert isinstance(data["version"], str) + assert isinstance(data["description"], str) + assert isinstance(data["repository_url"], str) + assert isinstance(data["documentation_url"], str) diff --git a/project_templates/fastapi_safir_app/example-uws/tox.ini b/project_templates/fastapi_safir_app/example-uws/tox.ini new file mode 100644 index 00000000..db6ecd57 --- /dev/null +++ b/project_templates/fastapi_safir_app/example-uws/tox.ini @@ -0,0 +1,63 @@ +[tox] +envlist = py,coverage-report,typing,lint +isolated_build = True + +[docker:postgres] +image = postgres:latest +environment = + POSTGRES_PASSWORD=INSECURE-PASSWORD + POSTGRES_USER=example-uws + POSTGRES_DB=example-uws +# The healthcheck ensures that tox-docker won't run tests until the +# container is up and the command finishes with exit code 0 (success) +healthcheck_cmd = PGPASSWORD=$POSTGRES_PASSWORD psql \ + --user=$POSTGRES_USER --dbname=$POSTGRES_DB \ + --host=127.0.0.1 --quiet --no-align --tuples-only \ + -1 --command="SELECT 1" +healthcheck_timeout = 1 +healthcheck_retries = 30 +healthcheck_interval = 1 +healthcheck_start_period = 1 + +[testenv] +description = Run pytest against {envname}. +deps = + -r{toxinidir}/requirements/main.txt + -r{toxinidir}/requirements/dev.txt +commands = + pytest --cov=exampleuws --cov-branch --cov-report= {posargs} + +[testenv:coverage-report] +description = Compile coverage from each test run. +skip_install = true +deps = coverage[toml]>=5.0.2 +depends = + py +commands = coverage report + +[testenv:py] +docker = + postgres +setenv = + EXAMPLE_UWS_DATABASE_URL = postgresql://example-uws@localhost/example-uws + EXAMPLE_UWS_DATABASE_PASSWORD = INSECURE-PASSWORD + EXAMPLE_UWS_ARQ_QUEUE_URL = redis://localhost/0 + EXAMPLE_UWS_SERVICE_ACCOUNT = example-uws@example.com + EXAMPLE_UWS_STORAGE_URL = gs://some-bucket + +[testenv:typing] +description = Run mypy. +commands = + mypy src/exampleuws tests + +[testenv:lint] +description = Lint codebase by running pre-commit +skip_install = true +deps = + pre-commit +commands = pre-commit run --all-files + +[testenv:run] +description = Run the development server with auto-reload for code changes. +usedevelop = true +commands = uvicorn exampleuws.main:app --reload diff --git a/project_templates/fastapi_safir_app/example/ruff-shared.toml b/project_templates/fastapi_safir_app/example/ruff-shared.toml index 823693a4..da644fb3 100644 --- a/project_templates/fastapi_safir_app/example/ruff-shared.toml +++ b/project_templates/fastapi_safir_app/example/ruff-shared.toml @@ -30,6 +30,7 @@ ignore = [ "ANN401", # sometimes Any is the right type "ARG001", # unused function arguments are often legitimate "ARG002", # unused method arguments are often legitimate + "ARG003", # unused class method arguments are often legitimate "ARG005", # unused lambda arguments are often legitimate "BLE001", # we want to catch and report Exception in background tasks "C414", # nested sorted is how you sort by multiple keys with reverse diff --git a/project_templates/fastapi_safir_app/example/src/example/main.py b/project_templates/fastapi_safir_app/example/src/example/main.py index 7a97bd2b..e649f64b 100644 --- a/project_templates/fastapi_safir_app/example/src/example/main.py +++ b/project_templates/fastapi_safir_app/example/src/example/main.py @@ -20,7 +20,7 @@ from .handlers.external import external_router from .handlers.internal import internal_router -__all__ = ["app", "config"] +__all__ = ["app"] @asynccontextmanager diff --git a/project_templates/fastapi_safir_app/hooks/post_gen_project.py b/project_templates/fastapi_safir_app/hooks/post_gen_project.py index 8ae16a76..fa3ea129 100644 --- a/project_templates/fastapi_safir_app/hooks/post_gen_project.py +++ b/project_templates/fastapi_safir_app/hooks/post_gen_project.py @@ -9,4 +9,18 @@ import shutil # These variables are interpolated by cookiecutter before this hook is run +uws_service = True if '{{ cookiecutter.flavor }}' == 'UWS' else False +module_name = '{{ cookiecutter.module_name }}' github_org = '{{ cookiecutter.github_org }}' + +# Remove unused files if the application will not be using UWS. +if not uws_service: + print(f"(post-gen hook) Removing unused UWS support files") + shutil.rmtree(f"src/{module_name}/workers") + os.remove("Dockerfile.worker") + os.remove("scripts/install-worker.sh") + os.remove("scripts/install-worker-packages.sh") + os.remove("scripts/start-worker.sh") + os.remove(f"src/{module_name}/cli.py") + os.remove(f"src/{module_name}/dependencies.py") + os.remove(f"src/{module_name}/domain.py") diff --git a/project_templates/fastapi_safir_app/templatekit.yaml b/project_templates/fastapi_safir_app/templatekit.yaml index a33c4034..de2bab37 100644 --- a/project_templates/fastapi_safir_app/templatekit.yaml +++ b/project_templates/fastapi_safir_app/templatekit.yaml @@ -14,6 +14,10 @@ dialog_fields: label: "GitHub organization" hint: "The package will be created in this GitHub organization." component: "select" + - key: "flavor" + label: "Flavor" + hint: "Flavor of FastAPI service to create." + component: "select" - label: "Initial copyright holder" key: "copyright_holder" component: "select" diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/.github/workflows/ci.yaml b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/.github/workflows/ci.yaml index 8e9d4f73..1b156dae 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/.github/workflows/ci.yaml +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/.github/workflows/ci.yaml @@ -79,3 +79,11 @@ jobs: with: image: {{ "${{ github.repository }}" }} github_token: {{ "${{ secrets.GITHUB_TOKEN }}" }} +{%- if cookiecutter.flavor == "UWS" %} + + - uses: lsst-sqre/build-and-push-to-ghcr@v1 + id: build-worker + with: + image: {{ "${{ github.repository }}-worker" }} + github_token: {{ "${{ secrets.GITHUB_TOKEN }}" }} +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/Dockerfile.worker b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/Dockerfile.worker new file mode 100644 index 00000000..5d181a91 --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/Dockerfile.worker @@ -0,0 +1,29 @@ +{%- if cookiecutter.flavor == "UWS" %} +# This Dockerfile constructs the image for backend workers. These images +# are based on stack containers and install any required supporting code +# for the backend, arq, and the backend worker definition. + +FROM lsstsqre/centos:7-stack-lsst_distrib-w_2024_27 + +# Reset the user to root since we need to do system install tasks. +USER root + +# Upgrade the system packages. +COPY scripts/install-worker-packages.sh . +RUN ./install-worker-packages.sh && rm ./install-worker-packages.sh + +# Install the necessary prerequisites and the vo-cutouts code. +COPY . /workdir +RUN /workdir/scripts/install-worker.sh /workdir && rm -r /workdir +COPY scripts/start-worker.sh / + +# Create a non-root user +RUN useradd --create-home appuser + +# Switch to the non-root user. +USER appuser + +# Start the arq worker. +WORKDIR / +CMD ["/start-worker.sh"] +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in index 985d2fe1..51c89c88 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/main.in @@ -15,4 +15,8 @@ uvicorn[standard] # Other dependencies. pydantic pydantic-settings +{%- if cookiecutter.flavor == "UWS" %} +safir[uws]>=6.2.0 +{%- else %} safir>=5 +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/tox.in b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/tox.in index fde253f3..bf382363 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/tox.in +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/requirements/tox.in @@ -11,4 +11,7 @@ -c dev.txt tox +{%- if cookiecutter.flavor == "UWS" %} +tox-docker +{%- endif %} tox-uv diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/ruff-shared.toml b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/ruff-shared.toml index 823693a4..da644fb3 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/ruff-shared.toml +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/ruff-shared.toml @@ -30,6 +30,7 @@ ignore = [ "ANN401", # sometimes Any is the right type "ARG001", # unused function arguments are often legitimate "ARG002", # unused method arguments are often legitimate + "ARG003", # unused class method arguments are often legitimate "ARG005", # unused lambda arguments are often legitimate "BLE001", # we want to catch and report Exception in background tasks "C414", # nested sorted is how you sort by multiple keys with reverse diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker-packages.sh b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker-packages.sh new file mode 100644 index 00000000..d2f52d5c --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker-packages.sh @@ -0,0 +1,24 @@ +{%- if cookiecutter.flavor == "UWS" %} +#!/bin/bash + +# Install or upgrade any operating system packages needed on worker images. +# This is done in a separate script to create a separate cached Docker image, +# which will help with iteration speed on the more interesting setup actions +# taken later in the build. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for details. +set -euo pipefail + +# Display each command as it's run. +set -x + +# Upgrade the Red Hat packages. +# +# TODO(rra): Disabled for now because the version of CentOS used by the image +# is so old that the package repositories no longer exist. This will in theory +# soon be fixed by basing the image on AlmaLinux. +#yum -y upgrade +#yum clean all +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh new file mode 100644 index 00000000..9796d81b --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/install-worker.sh @@ -0,0 +1,28 @@ +{%- if cookiecutter.flavor == "UWS" %} +#!/bin/bash + +# This script updates and installs the necessary prerequisites for a backend +# worker starting with a stack container. It takes one parameter, the +# directory in which to do the installation. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for details. +# set -u is omitted because the setup bash function does not support it. +set -eo pipefail + +# Enable the stack. This should be done before set -x because it will +# otherwise spew a bunch of nonsense no one cares about into the logs. +source /opt/lsst/software/stack/loadLSST.bash +setup lsst_distrib + +# Display each command as it's run. +set -x + +# Install Python dependencies and the {{ cookiecutter.name }} code. httpx is +# required by the remote Butler client, and google-cloud-storage is used to +# store cutouts when the bucket has a gs URL. +cd "$1" +pip install --no-cache-dir google-cloud-storage httpx safir-arq structlog +pip install --no-cache-dir --no-deps . +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/start-worker.sh b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/start-worker.sh new file mode 100644 index 00000000..6b411458 --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/scripts/start-worker.sh @@ -0,0 +1,19 @@ +{%- if cookiecutter.flavor == "UWS" %} +#!/bin/bash + +# This script is installed in the worker image and starts the backend worker +# using arq. + +# Bash "strict mode", to help catch problems and bugs in the shell +# script. Every bash script you write should include this. See +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ for details. +# set -u is omitted because the setup bash function does not support it. +set -eo pipefail + +# Initialize the environment. +source /opt/lsst/software/stack/loadLSST.bash +setup lsst_distrib + +# Start arq with the worker. +arq {{ cookiecutter.module_name }}.workers.{{ cookiecutter.module_name }}.WorkerSettings +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/cli.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/cli.py new file mode 100644 index 00000000..b2463b45 --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/cli.py @@ -0,0 +1,43 @@ +{%- if cookiecutter.flavor == "UWS" %} +"""Administrative command-line interface.""" + +from __future__ import annotations + +import click +import structlog +from safir.asyncio import run_with_asyncio +from safir.click import display_help + +from .config import uws + +__all__ = [ + "help", + "init", + "main", +] + + +@click.group(context_settings={"help_option_names": ["-h", "--help"]}) +@click.version_option(message="%(version)s") +def main() -> None: + """Administrative command-line interface for {{ cookiecutter.name }}.""" + + +@main.command() +@click.argument("topic", default=None, required=False, nargs=1) +@click.pass_context +def help(ctx: click.Context, topic: str | None) -> None: + """Show help for any command.""" + display_help(main, ctx, topic) + + +@main.command() +@click.option( + "--reset", is_flag=True, help="Delete all existing database data." +) +@run_with_asyncio +async def init(*, reset: bool) -> None: + """Initialize the database storage.""" + logger = structlog.get_logger("{{ cookiecutter.module_name }}") + await uws.initialize_uws_database(logger, reset=reset) +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/config.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/config.py index dbe10d1e..0c9b96fd 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/config.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/config.py @@ -3,13 +3,19 @@ from __future__ import annotations from pydantic import Field -from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic_settings import {% if cookiecutter.flavor != "UWS" %}BaseSettings, {% endif %}SettingsConfigDict from safir.logging import LogLevel, Profile +{%- if cookiecutter.flavor == "UWS" %} +from safir.uws import UWSApplication, UWSAppSettings, UWSConfig, UWSRoute + +from .dependencies import post_params_dependency +from .models import {{ cookiecutter.module_name | capitalize }}Parameters +{%- endif %} __all__ = ["Config", "config"] -class Config(BaseSettings): +class Config({% if cookiecutter.flavor == "UWS" %}UWSAppSettings{% else %}BaseSettings{% endif %}): """Configuration for {{ cookiecutter.name }}.""" name: str = Field("{{ cookiecutter.name }}", title="Name of application") @@ -29,7 +35,27 @@ class Config(BaseSettings): model_config = SettingsConfigDict( env_prefix="{{ cookiecutter.name | upper | replace('-', '_') }}_", case_sensitive=False ) +{%- if cookiecutter.flavor == "UWS" %} + + @property + def uws_config(self) -> UWSConfig: + """Corresponding configuration for the UWS subsystem.""" + return self.build_uws_config( + parameters_type={{ cookiecutter.module_name | capitalize }}Parameters, + worker="{{ cookiecutter.name | replace('-', '_') }}", + async_post_route=UWSRoute( + dependency=post_params_dependency, + summary="Create async {{ cookiecutter.name }} job", + description="Create a new UWS job for {{ cookiecutter.name }}", + ), + ) +{%- endif %} config = Config() """Configuration for {{ cookiecutter.name }}.""" +{%- if cookiecutter.flavor == "UWS" %} + +uws = UWSApplication(config.uws_config) +"""The UWS application for this service.""" +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/dependencies.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/dependencies.py new file mode 100644 index 00000000..5ce0eccd --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/dependencies.py @@ -0,0 +1,29 @@ +{%- if cookiecutter.flavor == "UWS" %} +"""Job parameter dependencies.""" + +from typing import Annotated + +from fastapi import Depends, Request +from safir.uws import UWSJobParameter, uws_post_params_dependency + +__all__ = [ + "post_params_dependency", +] + + +async def post_params_dependency( + *, + # Add POST parameters here. All of them should be Form() parameters. + # Use str | None for single-valued attributes and str | list[str] | None + # for parameters that can be given more than one time. + params: Annotated[ + list[UWSJobParameter], Depends(uws_post_params_dependency) + ], +) -> list[UWSJobParameter]: + """Parse POST parameters into job parameters.""" + return [ + p + for p in params + if p.parameter_id in set() # Replace with set of parameter names + ] +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/domain.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/domain.py new file mode 100644 index 00000000..dbd6a5ed --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/domain.py @@ -0,0 +1,16 @@ +{%- if cookiecutter.flavor == "UWS" %} +"""Domain models for {{ cookiecutter.name }}.""" + +from __future__ import annotations + +from pydantic import BaseModel + +__all__ = ["Worker{{ cookiecutter.module_name | capitalize }}Model"] + + +class Worker{{ cookiecutter.module_name | capitalize }}Model(BaseModel): + """Parameter model for backend workers. + + Add fields here for all the parameters passed to backend jobs. + """ +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/main.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/main.py index 1539ce02..a0c62010 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/main.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/main.py @@ -16,21 +16,27 @@ from safir.logging import configure_logging, configure_uvicorn_logging from safir.middleware.x_forwarded import XForwardedMiddleware -from .config import config +from .config import config{% if cookiecutter.flavor == "UWS" %}, uws{% endif %} from .handlers.external import external_router from .handlers.internal import internal_router -__all__ = ["app", "config"] +__all__ = ["app"] @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: """Set up and tear down the application.""" # Any code here will be run when the application starts up. + {%- if cookiecutter.flavor == "UWS" %} + await uws.initialize_fastapi() + {%- endif %} yield # Any code here will be run when the application shuts down. + {%- if cookiecutter.flavor == "UWS" %} + await uws.shutdown_fastapi() + {%- endif %} await http_client_dependency.aclose() @@ -54,7 +60,16 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: # Attach the routers. app.include_router(internal_router) +{%- if cookiecutter.flavor == "UWS" %} +uws.install_handlers(external_router) +{%- endif %} app.include_router(external_router, prefix=f"{config.path_prefix}") # Add middleware. app.add_middleware(XForwardedMiddleware) +{%- if cookiecutter.flavor == "UWS" %} +uws.install_middleware(app) + +# Install error handlers. +uws.install_error_handlers(app) +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/models.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/models.py index 8a4c6b6b..ca591f59 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/models.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/models.py @@ -1,7 +1,16 @@ """Models for {{ cookiecutter.name }}.""" +{% if cookiecutter.flavor == "UWS" -%} +from typing import Self + +{% endif -%} from pydantic import BaseModel, Field from safir.metadata import Metadata as SafirMetadata +{%- if cookiecutter.flavor == "UWS" %} +from safir.uws import ParametersModel, UWSJobParameter + +from .domain import Worker{{ cookiecutter.module_name | capitalize }}Model +{%- endif %} __all__ = ["Index"] @@ -18,3 +27,21 @@ class Index(BaseModel): """ metadata: SafirMetadata = Field(..., title="Package metadata") +{%- if cookiecutter.flavor == "UWS" %} + + +class {{ cookiecutter.module_name | capitalize }}Parameters(ParametersModel[Worker{{ cookiecutter.module_name | capitalize }}Model]): + """Model for job parameters. + + Add fields here for all the input parameters to a job, and then update + ``from_job_parameters`` and ``to_worker_parameters`` to do the appropriate + conversions. + """ + + @classmethod + def from_job_parameters(cls, params: list[UWSJobParameter]) -> Self: + return cls() + + def to_worker_parameters(self) -> Worker{{ cookiecutter.module_name | capitalize }}Model: + return Worker{{ cookiecutter.module_name | capitalize }}Model() +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/__init__.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/uws.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/uws.py new file mode 100644 index 00000000..14b4ba93 --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/uws.py @@ -0,0 +1,20 @@ +{%- if cookiecutter.flavor == "UWS" %} +"""Worker for UWS database updates.""" + +from __future__ import annotations + +import structlog +from safir.logging import configure_logging + +from ..config import config, uws + +__all__ = ["WorkerSettings"] + + +configure_logging( + name="{{ cookiecutter.module_name }}", profile=config.profile, log_level=config.log_level +) + +WorkerSettings = uws.build_worker(structlog.get_logger("{{ cookiecutter.module_name }}")) +"""arq configuration for the UWS database worker.""" +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/{{cookiecutter.module_name}}.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/{{cookiecutter.module_name}}.py new file mode 100644 index 00000000..0538c74e --- /dev/null +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/src/{{cookiecutter.module_name}}/workers/{{cookiecutter.module_name}}.py @@ -0,0 +1,99 @@ +{%- if cookiecutter.flavor == "UWS" -%} +"""Worker for {{ cookiecutter.name }}. + +This is a standalone file intended to be injected into a stack container as +the arq worker definition. Only this module is allowed to use stack packages. +""" + +from __future__ import annotations + +import os +from datetime import timedelta + +import structlog +from safir.arq import ArqMode +from safir.arq.uws import ( + WorkerConfig, + WorkerFatalError, + WorkerJobInfo, + WorkerResult, + build_worker, +) +from structlog.stdlib import BoundLogger + +from ..domain import Worker{{ cookiecutter.module_name | capitalize }}Model + +__all__ = ["WorkerSettings"] + + +def {{ cookiecutter.name | replace('-', '_') }}( + params: Worker{{ cookiecutter.module_name | capitalize }}Model, info: WorkerJobInfo, logger: BoundLogger +) -> list[WorkerResult]: + """Perform the work. + + This is a queue worker for the {{ cookiecutter.name }} service. It takes a serialized + request, converts it into a suitable in-memory format, and then dispatches + it to the scientific code that performs the cutout. The results are stored + in a GCS bucket, and the details of the output are returned as the result + of the worker. + + Parameters + ---------- + params + Cutout parameters. + info + Information about the UWS job we're executing. + logger + Logger to use for logging. + + Returns + ------- + list of WorkerResult + Results of the job. + + Raises + ------ + WorkerFatalError + Raised if the cutout failed for unknown reasons, or due to internal + errors. This is the normal failure exception, since we usually do not + know why the backend code failed and make the pessimistic assumption + that the failure is not transient. + WorkerUsageError + Raised if the cutout failed due to deficiencies in the parameters + submitted by the user that could not be detected by the frontend + service. + """ + logger.info("Starting request") + try: + # Replace this with a call to the function that does the work. + result_url = "https://example.com/" + except Exception as e: + raise WorkerFatalError(f"{type(e).__name__}: {e!s}") from e + logger.info("Request successful") + + # Change the result ID to something reasonable, set the MIME type to an + # appropriate value, and ideally also set the result size if that's + # available. + return [ + WorkerResult( + result_id="main", url=result_url, mime_type="application/fits" + ) + ] + + +WorkerSettings = build_worker( + {{ cookiecutter.name | replace('-', '_') }}, + WorkerConfig( + arq_mode=ArqMode.production, + arq_queue_url=os.environ["{{ cookiecutter.name | upper | replace('-', '_') }}_ARQ_QUEUE_URL"], + arq_queue_password=os.getenv("{{ cookiecutter.name | upper | replace('-', '_') }}_ARQ_QUEUE_PASSWORD"), + grace_period=timedelta( + seconds=int(os.environ["{{ cookiecutter.name | upper | replace('-', '_') }}_GRACE_PERIOD"]) + ), + parameters_class=Worker{{ cookiecutter.module_name | capitalize }}Model, + timeout=timedelta(seconds=int(os.environ["{{ cookiecutter.name | upper | replace('-', '_') }}_TIMEOUT"])), + ), + structlog.get_logger("{{ cookiecutter.module_name }}"), +) +"""arq configuration for the {{ cookiecutter.name }} worker.""" +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tests/conftest.py b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tests/conftest.py index ea59f213..377e7327 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tests/conftest.py +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tests/conftest.py @@ -2,25 +2,57 @@ from __future__ import annotations -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator{% if cookiecutter.flavor == "UWS" %}, Iterator +from datetime import timedelta +{%- endif %} +{% if cookiecutter.flavor == "UWS" -%} +import pytest +{% endif -%} import pytest_asyncio +{%- if cookiecutter.flavor == "UWS" %} +import structlog +{%- endif %} from asgi_lifespan import LifespanManager from fastapi import FastAPI from httpx import ASGITransport, AsyncClient +{%- if cookiecutter.flavor == "UWS" %} +from safir.arq import MockArqQueue +from safir.testing.gcs import MockStorageClient, patch_google_storage +from safir.testing.uws import MockUWSJobRunner +{%- endif %} from {{ cookiecutter.module_name }} import main +{%- if cookiecutter.flavor == "UWS" %} +from {{ cookiecutter.module_name }}.config import config, uws +{%- endif %} @pytest_asyncio.fixture +{%- if cookiecutter.flavor == "UWS" %} +async def app(arq_queue: MockArqQueue) -> AsyncIterator[FastAPI]: +{%- else %} async def app() -> AsyncIterator[FastAPI]: +{%- endif %} """Return a configured test application. Wraps the application in a lifespan manager so that startup and shutdown events are sent during test execution. """ + {%- if cookiecutter.flavor == "UWS" %} + logger = structlog.get_logger("{{ cookiecutter.module_name }}") + await uws.initialize_uws_database(logger, reset=True) + uws.override_arq_queue(arq_queue) + {%- endif %} async with LifespanManager(main.app): yield main.app +{%- if cookiecutter.flavor == "UWS" %} + + +@pytest.fixture +def arq_queue() -> MockArqQueue: + return MockArqQueue() +{%- endif %} @pytest_asyncio.fixture @@ -31,3 +63,20 @@ async def client(app: FastAPI) -> AsyncIterator[AsyncClient]: base_url="https://example.com/", ) as client: yield client +{%- if cookiecutter.flavor == "UWS" %} + + +@pytest.fixture(autouse=True) +def mock_google_storage() -> Iterator[MockStorageClient]: + yield from patch_google_storage( + expected_expiration=timedelta(minutes=15), bucket_name="some-bucket" + ) + + +@pytest_asyncio.fixture +async def runner( + arq_queue: MockArqQueue, +) -> AsyncIterator[MockUWSJobRunner]: + async with MockUWSJobRunner(config.uws_config, arq_queue) as runner: + yield runner +{%- endif %} diff --git a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tox.ini b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tox.ini index 2a358aa6..58fab517 100644 --- a/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tox.ini +++ b/project_templates/fastapi_safir_app/{{cookiecutter.name}}/tox.ini @@ -1,6 +1,25 @@ [tox] envlist = py,coverage-report,typing,lint isolated_build = True +{%- if cookiecutter.flavor == "UWS" %} + +[docker:postgres] +image = postgres:latest +environment = + POSTGRES_PASSWORD=INSECURE-PASSWORD + POSTGRES_USER={{ cookiecutter.name }} + POSTGRES_DB={{ cookiecutter.name }} +# The healthcheck ensures that tox-docker won't run tests until the +# container is up and the command finishes with exit code 0 (success) +healthcheck_cmd = PGPASSWORD=$POSTGRES_PASSWORD psql \ + --user=$POSTGRES_USER --dbname=$POSTGRES_DB \ + --host=127.0.0.1 --quiet --no-align --tuples-only \ + -1 --command="SELECT 1" +healthcheck_timeout = 1 +healthcheck_retries = 30 +healthcheck_interval = 1 +healthcheck_start_period = 1 +{%- endif %} [testenv] description = Run pytest against {envname}. @@ -17,6 +36,18 @@ deps = coverage[toml]>=5.0.2 depends = py commands = coverage report +{%- if cookiecutter.flavor == "UWS" %} + +[testenv:py] +docker = + postgres +setenv = + {{ cookiecutter.name | upper | replace('-', '_') }}_DATABASE_URL = postgresql://{{cookiecutter.name}}@localhost/{{cookiecutter.name}} + {{ cookiecutter.name | upper | replace('-', '_') }}_DATABASE_PASSWORD = INSECURE-PASSWORD + {{ cookiecutter.name | upper | replace('-', '_') }}_ARQ_QUEUE_URL = redis://localhost/0 + {{ cookiecutter.name | upper | replace('-', '_') }}_SERVICE_ACCOUNT = {{cookiecutter.name}}@example.com + {{ cookiecutter.name | upper | replace('-', '_') }}_STORAGE_URL = gs://some-bucket +{%- endif %} [testenv:typing] description = Run mypy. diff --git a/project_templates/square_pypi_package/example/ruff-shared.toml b/project_templates/square_pypi_package/example/ruff-shared.toml index 823693a4..da644fb3 100644 --- a/project_templates/square_pypi_package/example/ruff-shared.toml +++ b/project_templates/square_pypi_package/example/ruff-shared.toml @@ -30,6 +30,7 @@ ignore = [ "ANN401", # sometimes Any is the right type "ARG001", # unused function arguments are often legitimate "ARG002", # unused method arguments are often legitimate + "ARG003", # unused class method arguments are often legitimate "ARG005", # unused lambda arguments are often legitimate "BLE001", # we want to catch and report Exception in background tasks "C414", # nested sorted is how you sort by multiple keys with reverse diff --git a/project_templates/square_pypi_package/{{cookiecutter.name}}/ruff-shared.toml b/project_templates/square_pypi_package/{{cookiecutter.name}}/ruff-shared.toml index 823693a4..da644fb3 100644 --- a/project_templates/square_pypi_package/{{cookiecutter.name}}/ruff-shared.toml +++ b/project_templates/square_pypi_package/{{cookiecutter.name}}/ruff-shared.toml @@ -30,6 +30,7 @@ ignore = [ "ANN401", # sometimes Any is the right type "ARG001", # unused function arguments are often legitimate "ARG002", # unused method arguments are often legitimate + "ARG003", # unused class method arguments are often legitimate "ARG005", # unused lambda arguments are often legitimate "BLE001", # we want to catch and report Exception in background tasks "C414", # nested sorted is how you sort by multiple keys with reverse diff --git a/project_templates/technote_md/testn-000/technote.toml b/project_templates/technote_md/testn-000/technote.toml index d81928a8..db06c7fe 100644 --- a/project_templates/technote_md/testn-000/technote.toml +++ b/project_templates/technote_md/testn-000/technote.toml @@ -4,7 +4,7 @@ series_id = "TESTN" canonical_url = "https://testn-000.lsst.io" github_url = "https://github.com/lsst/testn-000" github_default_branch = "main" -date_created = 2024-07-30T17:55:32Z +date_created = 2024-08-02T18:56:37Z organization.name = "Vera C. Rubin Observatory" organization.ror = "https://ror.org/048g3cy84" license.id = "CC-BY-4.0" diff --git a/project_templates/technote_rst/testn-000/technote.toml b/project_templates/technote_rst/testn-000/technote.toml index 05580a74..6f52362b 100644 --- a/project_templates/technote_rst/testn-000/technote.toml +++ b/project_templates/technote_rst/testn-000/technote.toml @@ -4,7 +4,7 @@ series_id = "TESTN" canonical_url = "https://testn-000.lsst.io" github_url = "https://github.com/lsst/testn-000" github_default_branch = "main" -date_created = 2024-07-30T17:55:32Z +date_created = 2024-08-02T18:56:37Z organization.name = "Vera C. Rubin Observatory" organization.ror = "https://ror.org/048g3cy84" license.id = "CC-BY-4.0"