Merge pull request #280 from lsst-sqre/tickets/DM-45281

DM-45281: Separate safir-arq from safir
lsst-sqre · Jul 25, 2024 · a44029a · a44029a
2 parents f83776b + 55a5bd2
commit a44029a
Show file tree

Hide file tree

Showing 45 changed files with 108 additions and 97 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -108,13 +108,20 @@ jobs:
         with:
           fetch-depth: 0 # full history for setuptools_scm
 
-      - name: Build and publish
+      - name: Test building and publishing safir
         uses: lsst-sqre/build-and-publish-to-pypi@v2
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           upload: false
           working-directory: "safir"
 
+      - name: Test building and publishing safir-arq
+        uses: lsst-sqre/build-and-publish-to-pypi@v2
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          upload: false
+          working-directory: "safir-arq"
+
   pypi:
 
     name: Upload release to PyPI
@@ -137,3 +144,8 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           working-directory: "safir"
+
+      - uses: lsst-sqre/build-and-publish-to-pypi@v2
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          working-directory: "safir-arq"
diff --git a/Makefile b/Makefile
diff --git a/changelog.d/20240724_155805_rra_DM_45281.md b/changelog.d/20240724_155805_rra_DM_45281.md
@@ -0,0 +1,3 @@
+### New features
+
+- `safir.arq` is now available as a separate PyPI package, `safir-arq`, so that it can be installed in environments where the full Safir dependency may be too heavy-weight or conflict with other packages. The `safir[arq]` dependency will continue to work as before (by installing `safir-arq` behind the scenes).
diff --git a/noxfile.py b/noxfile.py
@@ -24,7 +24,9 @@
 def _install(session: nox.Session) -> None:
     """Install the application and all dependencies into the session."""
     session.install("--upgrade", "uv")
-    session.install("-e", "./safir[arq,db,dev,gcs,kubernetes,redis]")
+    session.install(
+        "-e", "./safir-arq", "./safir[arq,db,dev,gcs,kubernetes,redis]"
+    )
 
 
 def _install_dev(session: nox.Session, bin_prefix: str = "") -> None:
@@ -105,9 +107,13 @@ def typing(session: nox.Session) -> None:
     session.run(
         "mypy",
         *session.posargs,
+        "--namespace-packages",
+        "--explicit-package-bases",
         "noxfile.py",
         "safir/src",
+        "safir-arq/src",
         "safir/tests",
+        env={"MYPYPATH": "safir/src:safir:safir-arq/src"},
     )
 
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -51,7 +51,7 @@ extend = "ruff-shared.toml"
 "noxfile.py" = [
     "T201",    # print makes sense as output from nox rules
 ]
-"safir/src/safir/**" = [
+"*/src/safir/**" = [
     "N818",    # Exception is correct in some cases, others are part of API
 ]
 "safir/src/safir/testing/**" = [
@@ -70,6 +70,7 @@ extend = "ruff-shared.toml"
 ]
 
 [tool.ruff.lint.isort]
+detect-same-package = false
 known-first-party = ["safir", "tests"]
 split-on-trailing-comma = false
 

diff --git a/safir-arq/LICENSE b/safir-arq/LICENSE
@@ -0,0 +1 @@
+../LICENSE
diff --git a/safir-arq/README.md b/safir-arq/README.md
@@ -0,0 +1,13 @@
+# safir-arq
+
+safir-arq is a subpackage of Safir, Rubin Observatory's library for building [FastAPI](https://fastapi.tiangolo.com/) services for the [Rubin Science Platform (Phalanx)](https://github.com/lsst-sqre/phalanx) and [Roundtable](https://github.com/lsst-sqre/roundtable) Kubernetes clusters.
+It is a separate PyPI module so that it can be used as a dependency in contexts where the full Safir dependency is undesirable.
+
+safir-arq is available from [PyPI](https://pypi.org/project/safir-arq/):
+
+```sh
+pip install safir-arq
+```
+
+safir-arq is developed and tested in conjunction with Safir.
+Read more about Safir at https://safir.lsst.io.
diff --git a/safir-arq/pyproject.toml b/safir-arq/pyproject.toml
@@ -0,0 +1,51 @@
+[project]
+# https://packaging.python.org/en/latest/specifications/declaring-project-metadata/
+name = "safir-arq"
+description = "arq support for the Rubin Observatory SQuaRE framework, Safir."
+license = {file = "LICENSE"}
+readme= "README.md"
+keywords = [
+    "rubin",
+    "lsst",
+]
+# https://pypi.org/classifiers/
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Intended Audience :: Developers",
+    "Natural Language :: English",
+    "Operating System :: POSIX",
+    "Typing :: Typed",
+]
+requires-python = ">=3.11"
+dependencies = [
+    "arq>=0.23,<1",
+    "pydantic>2,<3",
+    "pydantic-core",
+]
+dynamic = ["version"]
+
+[[project.authors]]
+name = "Association of Universities for Research in Astronomy, Inc. (AURA)"
+email = "[email protected]"
+
+[project.urls]
+Homepage = "https://safir.lsst.io"
+Source = "https://github.com/lsst-sqre/safir"
+"Change log" = "https://safir.lsst.io/changelog.html"
+"Issue tracker" = "https://github.com/lsst-sqre/safir/issues"
+
+[build-system]
+requires = [
+    "setuptools>=61",
+    "wheel",
+    "setuptools_scm[toml]>=6.2"
+]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools_scm]
+root = ".."
diff --git a/safir/src/safir/arq.py → safir-arq/src/safir/arq/__init__.py b/safir/src/safir/arq.py → safir-arq/src/safir/arq/__init__.py
@@ -6,7 +6,7 @@
 import asyncio
 import uuid
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import UTC, datetime
 from enum import Enum
 from typing import Any, Self
 
@@ -17,8 +17,6 @@
 from pydantic import SecretStr
 from pydantic_core import Url
 
-from .datetime import current_datetime
-
 __all__ = [
     "ArqJobError",
     "JobNotQueued",
@@ -515,7 +513,7 @@ async def enqueue(
             name=task_name,
             args=task_args,
             kwargs=task_kwargs,
-            enqueue_time=current_datetime(microseconds=True),
+            enqueue_time=datetime.now(tz=UTC),
             status=JobStatus.queued,
             queue_name=queue_name,
         )
@@ -545,8 +543,8 @@ async def abort_job(
                 kwargs=job_metadata.kwargs,
                 status=job_metadata.status,
                 enqueue_time=job_metadata.enqueue_time,
-                start_time=current_datetime(microseconds=True),
-                finish_time=current_datetime(microseconds=True),
+                start_time=datetime.now(tz=UTC),
+                finish_time=datetime.now(tz=UTC),
                 result=asyncio.CancelledError(),
                 success=False,
                 queue_name=queue_name,
@@ -614,8 +612,8 @@ async def set_complete(
             kwargs=job_metadata.kwargs,
             status=job_metadata.status,
             enqueue_time=job_metadata.enqueue_time,
-            start_time=current_datetime(microseconds=True),
-            finish_time=current_datetime(microseconds=True),
+            start_time=datetime.now(tz=UTC),
+            finish_time=datetime.now(tz=UTC),
             result=result,
             success=success,
             queue_name=queue_name,

diff --git a/safir/src/safir/py.typed → safir-arq/src/safir/arq/py.typed b/safir/src/safir/py.typed → safir-arq/src/safir/arq/py.typed
diff --git a/safir/pyproject.toml b/safir/pyproject.toml
@@ -38,7 +38,7 @@ dynamic = ["version"]
 
 [project.optional-dependencies]
 arq = [
-    "arq>=0.23,<1"
+    "safir-arq",
 ]
 db = [
     "asyncpg<1",

diff --git a/safir/src/safir/__init__.py b/safir/src/safir/__init__.py
diff --git a/safir/src/safir/asyncio.py → safir/src/safir/asyncio/__init__.py b/safir/src/safir/asyncio.py → safir/src/safir/asyncio/__init__.py
@@ -9,7 +9,7 @@
 from types import EllipsisType
 from typing import Generic, ParamSpec, TypeVar
 
-from .datetime import current_datetime
+from safir.datetime import current_datetime
 
 #: Parameter spec for functions decorated by `run_with_asyncio`.
 P = ParamSpec("P")

diff --git a/safir/src/safir/asyncio/py.typed b/safir/src/safir/asyncio/py.typed
diff --git a/safir/src/safir/click.py → safir/src/safir/click/__init__.py b/safir/src/safir/click.py → safir/src/safir/click/__init__.py
diff --git a/safir/src/safir/click/py.typed b/safir/src/safir/click/py.typed
diff --git a/safir/src/safir/database/py.typed b/safir/src/safir/database/py.typed
diff --git a/safir/src/safir/datetime.py → safir/src/safir/datetime/__init__.py b/safir/src/safir/datetime.py → safir/src/safir/datetime/__init__.py
diff --git a/safir/src/safir/datetime/py.typed b/safir/src/safir/datetime/py.typed
diff --git a/safir/src/safir/dependencies/arq.py b/safir/src/safir/dependencies/arq.py
@@ -4,7 +4,7 @@
 
 from arq.connections import RedisSettings
 
-from ..arq import ArqMode, ArqQueue, MockArqQueue, RedisArqQueue
+from safir.arq import ArqMode, ArqQueue, MockArqQueue, RedisArqQueue
 
 __all__ = ["ArqDependency", "arq_dependency"]
 

diff --git a/safir/src/safir/dependencies/db_session.py b/safir/src/safir/dependencies/db_session.py
@@ -5,7 +5,7 @@
 from pydantic import SecretStr
 from sqlalchemy.ext.asyncio import AsyncEngine, async_scoped_session
 
-from ..database import create_async_session, create_database_engine
+from safir.database import create_async_session, create_database_engine
 
 __all__ = ["DatabaseSessionDependency", "db_session_dependency"]
 

diff --git a/safir/src/safir/dependencies/py.typed b/safir/src/safir/dependencies/py.typed
diff --git a/safir/src/safir/fastapi.py → safir/src/safir/fastapi/__init__.py b/safir/src/safir/fastapi.py → safir/src/safir/fastapi/__init__.py
@@ -7,8 +7,8 @@
 from fastapi import Request, status
 from fastapi.responses import JSONResponse
 
-from .models import ErrorLocation
-from .slack.webhook import SlackIgnoredException
+from safir.models import ErrorLocation
+from safir.slack.webhook import SlackIgnoredException
 
 __all__ = [
     "ClientRequestError",

diff --git a/safir/src/safir/fastapi/py.typed b/safir/src/safir/fastapi/py.typed
diff --git a/safir/src/safir/gcs.py → safir/src/safir/gcs/__init__.py b/safir/src/safir/gcs.py → safir/src/safir/gcs/__init__.py
diff --git a/safir/src/safir/gcs/py.typed b/safir/src/safir/gcs/py.typed
diff --git a/safir/src/safir/github/py.typed b/safir/src/safir/github/py.typed
diff --git a/safir/src/safir/kubernetes.py → safir/src/safir/kubernetes/__init__.py b/safir/src/safir/kubernetes.py → safir/src/safir/kubernetes/__init__.py
diff --git a/safir/src/safir/kubernetes/py.typed b/safir/src/safir/kubernetes/py.typed
diff --git a/safir/src/safir/logging.py → safir/src/safir/logging/__init__.py b/safir/src/safir/logging.py → safir/src/safir/logging/__init__.py
diff --git a/safir/src/safir/logging/py.typed b/safir/src/safir/logging/py.typed
diff --git a/safir/src/safir/metadata.py → safir/src/safir/metadata/__init__.py b/safir/src/safir/metadata.py → safir/src/safir/metadata/__init__.py
diff --git a/safir/src/safir/metadata/py.typed b/safir/src/safir/metadata/py.typed
diff --git a/safir/src/safir/middleware/py.typed b/safir/src/safir/middleware/py.typed
diff --git a/safir/src/safir/models.py → safir/src/safir/models/__init__.py b/safir/src/safir/models.py → safir/src/safir/models/__init__.py
diff --git a/safir/src/safir/models/py.typed b/safir/src/safir/models/py.typed
diff --git a/safir/src/safir/pydantic.py → safir/src/safir/pydantic/__init__.py b/safir/src/safir/pydantic.py → safir/src/safir/pydantic/__init__.py
@@ -16,7 +16,7 @@
 )
 from pydantic_core import Url
 
-from .datetime import parse_timedelta
+from safir.datetime import parse_timedelta
 
 P = ParamSpec("P")
 T = TypeVar("T")

diff --git a/safir/src/safir/pydantic/py.typed b/safir/src/safir/pydantic/py.typed
diff --git a/safir/src/safir/redis.py → safir/src/safir/redis/__init__.py b/safir/src/safir/redis.py → safir/src/safir/redis/__init__.py
@@ -15,7 +15,7 @@
 from cryptography.fernet import Fernet
 from pydantic import BaseModel
 
-from .slack.blockkit import (
+from safir.slack.blockkit import (
     SlackCodeBlock,
     SlackException,
     SlackMessage,

diff --git a/safir/src/safir/redis/py.typed b/safir/src/safir/redis/py.typed
diff --git a/safir/src/safir/slack/py.typed b/safir/src/safir/slack/py.typed
diff --git a/safir/src/safir/slack/webhook.py b/safir/src/safir/slack/webhook.py
@@ -12,8 +12,9 @@
 from starlette.exceptions import HTTPException as StarletteHTTPException
 from structlog.stdlib import BoundLogger
 
-from ..datetime import current_datetime, format_datetime_for_logging
-from ..dependencies.http_client import http_client_dependency
+from safir.datetime import current_datetime, format_datetime_for_logging
+from safir.dependencies.http_client import http_client_dependency
+
 from .blockkit import (
     SlackCodeBlock,
     SlackException,

diff --git a/safir/src/safir/testing/kubernetes.py b/safir/src/safir/testing/kubernetes.py
@@ -45,7 +45,7 @@
     V1Status,
 )
 
-from ..asyncio import AsyncMultiQueue
+from safir.asyncio import AsyncMultiQueue
 
 __all__ = [
     "MockKubernetesApi",

diff --git a/safir/src/safir/testing/py.typed b/safir/src/safir/testing/py.typed
diff --git a/safir/tests/safir_test.py b/safir/tests/safir_test.py
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		### New features

		- `safir.arq` is now available as a separate PyPI package, `safir-arq`, so that it can be installed in environments where the full Safir dependency may be too heavy-weight or conflict with other packages. The `safir[arq]` dependency will continue to work as before (by installing `safir-arq` behind the scenes).