Routes and dependencies for GitHub CI app integration

src/mobu/dependencies/github.py

@@ -5,6 +5,11 @@
 import yaml
 
 from ..models.github import GitHubConfig
+from ..models.user import User
+from ..services.github_ci.ci_manager import CiManager
+from .context import ContextDependency
+
+__all__ = ["GitHubConfigDependency", "CiManagerDependency"]
 
 
 class GitHubConfigDependency:
@@ -28,4 +33,59 @@ def initialize(self, path: Path) -> None:
         )
 
 
+class CiManagerDependency:
+    """A process-global object to manage background CI workers.
+
+    It is important to close this when Mobu shuts down to make sure that
+    GitHub PRs that use the mobu CI app functionality don't have stuck
+    check runs.
+    """
+
+    def __init__(self) -> None:
+        self._ci_manager: CiManager | None = None
+
+    def __call__(self) -> CiManager:
+        return self.ci_manager
+
+    async def initialize(
+        self, base_context: ContextDependency, users: list[User]
+    ) -> None:
+        self._ci_manager = CiManager(
+            users=users,
+            http_client=base_context.process_context.http_client,
+            gafaelfawr_storage=base_context.process_context.gafaelfawr,
+            logger=base_context.process_context.logger,
+        )
+
+    @property
+    def ci_manager(self) -> CiManager:
+        if not self._ci_manager:
+            raise RuntimeError("CiManagerDependency not initialized")
+        return self._ci_manager
+
+    async def aclose(self) -> None:
+        if self._ci_manager:
+            await self._ci_manager.aclose()
+        self._ci_manager = None
+
+
+class MaybeCiManagerDependency:
+    """Try to return a CiManager, but don't blow up if it's not there.
+
+    Used in external routes that return info about mobu, and may be called on
+    installations that do not have the github ci functionality enabled.
+    """
+
+    def __init__(self, dep: CiManagerDependency) -> None:
+        self.dep = dep
+
+    def __call__(self) -> CiManager | None:
+        try:
+            return self.dep.ci_manager
+        except RuntimeError:
+            return None
+
+
 github_config_dependency = GitHubConfigDependency()
+ci_manager_dependency = CiManagerDependency()
+maybe_ci_manager_dependency = MaybeCiManagerDependency(ci_manager_dependency)

src/mobu/factory.py

@@ -37,9 +37,11 @@ class ProcessContext:
 
     def __init__(self, http_client: AsyncClient) -> None:
         self.http_client = http_client
-        logger = structlog.get_logger("mobu")
-        gafaelfawr = GafaelfawrStorage(http_client, logger)
-        self.manager = FlockManager(gafaelfawr, http_client, logger)
+        self.logger = structlog.get_logger("mobu")
+        self.gafaelfawr = GafaelfawrStorage(self.http_client, self.logger)
+        self.manager = FlockManager(
+            self.gafaelfawr, self.http_client, self.logger
+        )
 
     async def aclose(self) -> None:
         """Clean up a process context.

src/mobu/handlers/github_ci_app.py

@@ -0,0 +1,150 @@
+"""Github webhook handlers for CI app."""
+
+import asyncio
+from typing import Annotated
+
+from fastapi import APIRouter, Depends, HTTPException
+from gidgethub import routing
+from gidgethub.sansio import Event
+from safir.github.webhooks import GitHubCheckRunEventModel
+from safir.slack.webhook import SlackRouteErrorHandler
+
+from ..config import config
+from ..constants import GITHUB_WEBHOOK_WAIT_SECONDS
+from ..dependencies.context import RequestContext, anonymous_context_dependency
+from ..dependencies.github import (
+    ci_manager_dependency,
+    github_config_dependency,
+)
+from ..models.github import GitHubCheckSuiteEventModel, GitHubConfig
+from ..services.github_ci.ci_manager import CiManager
+
+__all__ = ["api_router"]
+
+api_router = APIRouter(route_class=SlackRouteErrorHandler)
+"""Registers incoming HTTP GitHub webhook requests"""
+
+
+gidgethub_router = routing.Router()
+"""Registers handlers for specific GitHub webhook payloads"""
+
+
+@api_router.post(
+    "/webhook",
+    summary="GitHub CI webhooks",
+    description="Receives webhook events from the GitHub mobu CI app.",
+    status_code=202,
+)
+async def post_webhook(
+    context: Annotated[RequestContext, Depends(anonymous_context_dependency)],
+    github_config: Annotated[GitHubConfig, Depends(github_config_dependency)],
+    ci_manager: Annotated[CiManager, Depends(ci_manager_dependency)],
+) -> None:
+    """Process GitHub webhook events for the mobu CI GitHubApp.
+
+    Rejects webhooks from organizations that are not explicitly allowed via the
+    mobu config. This should be exposed via a Gafaelfawr anonymous ingress.
+    """
+    webhook_secret = config.github_ci_app.webhook_secret
+    body = await context.request.body()
+    event = Event.from_http(
+        context.request.headers, body, secret=webhook_secret
+    )
+
+    owner = event.data.get("organization", {}).get("login")
+    if owner not in github_config.accepted_github_orgs:
+        context.logger.debug(
+            "Ignoring GitHub event for unaccepted org",
+            owner=owner,
+            accepted_orgs=github_config.accepted_github_orgs,
+        )
+        raise HTTPException(
+            status_code=403,
+            detail=(
+                "Mobu is not configured to accept webhooks from this GitHub"
+                " org."
+            ),
+        )
+
+    # Bind the X-GitHub-Delivery header to the logger context; this
+    # identifies the webhook request in GitHub's API and UI for
+    # diagnostics
+    context.rebind_logger(github_app="ci", github_delivery=event.delivery_id)
+    context.logger.debug("Received GitHub webhook", payload=event.data)
+    # Give GitHub some time to reach internal consistency.
+    await asyncio.sleep(GITHUB_WEBHOOK_WAIT_SECONDS)
+    await gidgethub_router.dispatch(
+        event=event, context=context, ci_manager=ci_manager
+    )
+
+
+@gidgethub_router.register("check_suite", action="requested")
+async def handle_check_suite_requested(
+    event: Event, context: RequestContext, ci_manager: CiManager
+) -> None:
+    """Start a run for any check suite request with an associated PR."""
+    context.rebind_logger(
+        github_webhook_event_type="check_suite",
+        github_webhook_action="requested",
+    )
+    em = GitHubCheckSuiteEventModel.model_validate(event.data)
+    if not bool(em.check_suite.pull_requests):
+        context.logger.debug("Ignoring; no associated pull requests")
+        return
+
+    await ci_manager.enqueue(
+        installation_id=em.installation.id,
+        repo_name=em.repository.name,
+        repo_owner=em.repository.owner.login,
+        ref=em.check_suite.head_sha,
+    )
+
+    context.logger.info("github ci webhook handled")
+
+
+@gidgethub_router.register("check_suite", action="rerequested")
+async def handle_check_suite_rerequested(
+    event: Event, context: RequestContext, ci_manager: CiManager
+) -> None:
+    """Start a run for any check suite re-request with an associated PR."""
+    context.rebind_logger(
+        github_webhook_event_type="check_suite",
+        github_webhook_action="rerequested",
+    )
+    em = GitHubCheckSuiteEventModel.model_validate(event.data)
+    if not bool(em.check_suite.pull_requests):
+        context.logger.debug("Ignoring; no associated pull requests")
+        return
+
+    await ci_manager.enqueue(
+        installation_id=em.installation.id,
+        repo_name=em.repository.name,
+        repo_owner=em.repository.owner.login,
+        ref=em.check_suite.head_sha,
+    )
+
+    context.logger.info("github ci webhook handled")
+
+
+@gidgethub_router.register("check_run", action="rerequested")
+async def handle_check_run_rerequested(
+    event: Event, context: RequestContext, ci_manager: CiManager
+) -> None:
+    """Start a run for any check run re-request with an associated PR."""
+    context.rebind_logger(
+        github_webhook_event_type="check_run",
+        github_webhook_action="rerequested",
+    )
+    em = GitHubCheckRunEventModel.model_validate(event.data)
+    if not bool(em.check_run.pull_requests):
+        context.logger.debug("Ignoring; no associated pull requests")
+        return
+
+    await ci_manager.enqueue(
+        installation_id=em.installation.id,
+        repo_name=em.repository.name,
+        repo_owner=em.repository.owner.login,
+        ref=em.check_run.head_sha,
+    )
+
+    context.logger.info("github ci webhook handled")

src/mobu/main.py

@@ -23,11 +23,13 @@
 
 from .asyncio import schedule_periodic
 from .config import config
-from .dependencies.context import context_dependency
+from .dependencies.context import ContextDependency, context_dependency
 from .dependencies.github import (
+    ci_manager_dependency,
     github_config_dependency,
 )
 from .handlers.external import external_router
+from .handlers.github_ci_app import api_router as github_ci_app_router
 from .handlers.github_refresh_app import (
     api_router as github_refresh_app_router,
 )
@@ -44,6 +46,7 @@ async def base_lifespan(app: FastAPI) -> AsyncIterator[ContextDependency]:
         raise RuntimeError("MOBU_ENVIRONMENT_URL was not set")
     if not config.gafaelfawr_token:
         raise RuntimeError("MOBU_GAFAELFAWR_TOKEN was not set")
+
     await context_dependency.initialize()
     await context_dependency.process_context.manager.autostart()
 
@@ -56,6 +59,32 @@ async def base_lifespan(app: FastAPI) -> AsyncIterator[ContextDependency]:
     app.state.periodic_status.cancel()
 
 
+@asynccontextmanager
+async def github_ci_app_lifespan(
+    base_context: ContextDependency,
+) -> AsyncIterator[None]:
+    """Set up and tear down the GitHub CI app functionality."""
+    if not config.github_config_path:
+        raise RuntimeError("MOBU_GITHUB_CONFIG_PATH was not set")
+    if not config.github_ci_app.webhook_secret:
+        raise RuntimeError("MOBU_GITHUB_CI_APP_WEBHOOK_SECRET was not set")
+    if not config.github_ci_app.private_key:
+        raise RuntimeError("MOBU_GITHUB_CI_APP_PRIVATE_KEY was not set")
+    if not config.github_ci_app.id:
+        raise RuntimeError("MOBU_GITHUB_CI_APP_ID was not set")
+
+    github_config_dependency.initialize(config.github_config_path)
+    await ci_manager_dependency.initialize(
+        base_context=base_context,
+        users=github_config_dependency.config.users,
+    )
+    await ci_manager_dependency.ci_manager.start()
+
+    yield
+
+    await ci_manager_dependency.aclose()
+
+
 @asynccontextmanager
 async def github_refresh_app_lifespan() -> AsyncIterator[None]:
     """Set up and tear down the GitHub refresh app functionality."""
@@ -79,6 +108,10 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     """
     async with AsyncExitStack() as stack:
         base_context = await stack.enter_async_context(base_lifespan(app))
+        if config.github_ci_app.enabled:
+            await stack.enter_async_context(
+                github_ci_app_lifespan(base_context)
+            )
         if config.github_refresh_app.enabled:
             await stack.enter_async_context(github_refresh_app_lifespan())
 
@@ -106,6 +139,10 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
 app.include_router(internal_router)
 app.include_router(external_router, prefix=config.path_prefix)
 
+if config.github_ci_app.enabled:
+    app.include_router(
+        github_ci_app_router, prefix=f"{config.path_prefix}/github/ci"
+    )
 if config.github_refresh_app.enabled:
     app.include_router(
         github_refresh_app_router,

src/mobu/models/github.py

@@ -1,15 +1,37 @@
 """GitHub app integration models.
+
+Some of these could probably make their way back into Safir.
 """
+
+import safir.github.models
+import safir.github.webhooks
 from pydantic import BaseModel, Field, field_validator
 
+from .user import User
+
 __all__ = [
+    "GitHubCheckSuiteEventModel",
+    "GitHubCheckSuiteModel",
     "GitHubConfig",
 ]
 
 
 class GitHubConfig(BaseModel):
     """Config for the GitHub CI app funcionality."""
 
+    users: list[User] = Field(
+        [],
+        title="Environment users for CI jobs to run as.",
+        description=(
+            "Must be prefixed with 'bot-', like all mobu users. In "
+            " environments without Firestore, users have to be provisioned"
+            " by environment admins, and their usernames, uids, and guids must"
+            " be specified here. In environments with firestore, only "
+            " usernames need to be specified, but you still need to explicitly"
+            " specify as many users as needed to get the amount of concurrency"
+            " that you want."
+        ),
+    )
     accepted_github_orgs: list[str] = Field(
         [],
         title="GitHub organizations to accept webhook requests from.",
@@ -18,3 +40,31 @@ class GitHubConfig(BaseModel):
             " this list will get a 403 response."
         ),
     )
+
+    @field_validator("users")
+    @classmethod
+    def check_bot_user(cls, v: list[User]) -> list[User]:
+        bad = [u.username for u in v if not u.username.startswith("bot-")]
+        if any(bad):
+            raise ValueError(
+                f"All usernames must start with 'bot-'. These don't: {bad}"
+            )
+        return v
+
+
+class GitHubCheckSuiteModel(
+    safir.github.models.GitHubCheckSuiteModel,
+):
+    """Adding ``pull_requests`` field to the existing check suite model."""
+
+    pull_requests: list[safir.github.models.GitHubCheckRunPrInfoModel] = (
+        Field()
+    )
+
+
+class GitHubCheckSuiteEventModel(
+    safir.github.webhooks.GitHubCheckSuiteEventModel,
+):
+    """Overriding ``check_suite`` to add ``pull_requests``."""
+
+    check_suite: GitHubCheckSuiteModel = Field()