feat: Add support for plugins

The System engineering toolchain team for Digitale Schiene is offering
several model modifiers and model derivative jobs.

Example for model modifiers are:
- ID injection bot (inject IDs into model elements)

Example for model derivates:
- Model complexity badge generation
- Diagram cache generation
- Automatic documentation generation

Synchronisation between TeamForCapella and Git is also considered as job,
somewhere in between.

Currently, all "jobs" are run in the Gitlab CI. There is no UI for the setup of these jobs.
Each job repository provides a Gitlab CI templates, which can be easily integrated in project repositories.
However, this approach is not intuitive for external people, leading to operation support effort for operation teams, which have to take care of the creation and updates of jobs.

Another disadavantage of the current approach is the Gitlab dependency.
Not all environments have a Gitlab instance available. Getting it running on other platforms is even more effort.

This commit adds a plugin/job store to the Capella Colloration Manager.
A plugin/job is still developed in a Git repository. Plugins have to define a
`mbse-works-plugin.yml`, providing metadata, input, output, trigger and job information.

With this information, registered plugins can be easily integrated into projects.
The integration should be intuitive for project leads.

The steps are:
- Create a new pipeline
- Select a supported plugin from the store
- Configure the plugin: Supported types for the beginning are:
    - git (Select a linked git model from the project, so that the job can access the Git repository information)
    - t4c (Select a linked T4C repository/project from the project). The job gets the connection information + a session token.
    - yml (Most flexible option, the yml configuration file is mounted into the job container, validation via yml schema)
    - environment (Key/value pairs, which are used as environment variable for the job)
- Confirm the creation

As part of this commit, there are only two options:
- Run the pipeline manually
- Run it as 3am during the night.

Co-authored-by: ewuerger <[email protected]>
Co-authored-by: Paula-Kli <[email protected]>

backend/capellacollab/alembic/versions/e3f1006f6b49_add_plugins_table.py

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Add plugins table
+
+Revision ID: e3f1006f6b49
+Revises: d0cbf2813066
+Create Date: 2023-05-26 11:56:13.928534
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "e3f1006f6b49"
+down_revision = "ac0e6e0f77ee"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "plugins",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("username", sa.String(), nullable=True),
+        sa.Column("password", sa.String(), nullable=True),
+        sa.Column("remote", sa.String(), nullable=False),
+        sa.Column(
+            "content", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_plugins_id"), "plugins", ["id"], unique=True)

backend/capellacollab/core/database/__init__.py

@@ -18,7 +18,10 @@
 
 
 class Base(orm.DeclarativeBase):
-    type_annotation_map = {dict[str, str]: postgresql.JSONB}
+    type_annotation_map = {
+        dict[str, str]: postgresql.JSONB,
+        dict[str, t.Any]: postgresql.JSONB,
+    }
 
 
 ### SQL MODELS ARE IMPORTED HERE ###

backend/capellacollab/core/database/models.py

@@ -5,6 +5,7 @@
 # These import statements of the models are required and should not be removed! (SQLAlchemy will not load the models otherwise)
 
 import capellacollab.notices.models
+import capellacollab.plugins.models
 import capellacollab.projects.models
 import capellacollab.projects.toolmodels.backups.models
 import capellacollab.projects.toolmodels.backups.runs.models

backend/capellacollab/plugins/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors
+# SPDX-License-Identifier: Apache-2.0

backend/capellacollab/plugins/crud.py

@@ -0,0 +1,61 @@
+# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors
+# SPDX-License-Identifier: Apache-2.0
+
+
+from collections import abc
+
+import sqlalchemy as sa
+from sqlalchemy import orm
+
+from . import models
+
+
+def get_plugins(db: orm.Session) -> abc.Sequence[models.DatabasePlugin]:
+    return db.execute(sa.select(models.DatabasePlugin)).scalars().all()
+
+
+def get_plugin_by_id(
+    db: orm.Session, plugin_id: int
+) -> models.DatabasePlugin | None:
+    return db.execute(
+        sa.select(models.DatabasePlugin).where(
+            models.DatabasePlugin.id == plugin_id
+        )
+    ).scalar_one_or_none()
+
+
+def create_plugin(
+    db: orm.Session,
+    remote: str,
+    username: str | None,
+    password: str | None,
+    content: dict | None,
+) -> models.DatabasePlugin:
+    plugin = models.DatabasePlugin(
+        remote=remote, username=username, password=password, content=content
+    )
+    db.add(plugin)
+    db.commit()
+    return plugin
+
+
+def update_plugin(
+    db: orm.Session,
+    plugin: models.DatabasePlugin,
+    patch_plugin: models.PatchPlugin,
+) -> models.DatabasePlugin:
+    if patch_plugin.username:
+        plugin.username = patch_plugin.username  # type: ignore[assignment]
+    if patch_plugin.password:
+        plugin.password = patch_plugin.password  # type: ignore[assignment]
+    if patch_plugin.remote:
+        plugin.remote = patch_plugin.remote  # type: ignore[assignment]
+    if patch_plugin.content:
+        plugin.content = patch_plugin.content  # type: ignore[assignment]
+    db.commit()
+    return plugin
+
+
+def delete_plugin(db: orm.Session, plugin: models.DatabasePlugin) -> None:
+    db.delete(plugin)
+    db.commit()

backend/capellacollab/plugins/injectables.py

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors
+# SPDX-License-Identifier: Apache-2.0
+
+from fastapi import Depends, HTTPException
+from sqlalchemy.orm import Session
+
+from capellacollab.core import database
+
+from . import crud, models
+
+
+def get_existing_plugin(
+    plugin_id: int, db: Session = Depends(database.get_db)
+) -> models.DatabasePlugin:
+    if plugin := crud.get_plugin_by_id(db, plugin_id):
+        return plugin
+
+    raise HTTPException(
+        404,
+        {
+            "reason": f"The plugin with the id {plugin_id} was not found.",
+        },
+    )

backend/capellacollab/plugins/models.py

@@ -0,0 +1,40 @@
+# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import typing as t
+
+import pydantic
+from sqlalchemy import orm
+
+from capellacollab.core import database
+
+
+class CreatePlugin(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(from_attributes=True)
+
+    username: str | None
+    password: str | None
+    remote: str
+
+
+class PatchPlugin(CreatePlugin):
+    content: dict | None
+
+
+class Plugin(CreatePlugin):
+    id: int
+    content: dict | None
+
+
+class DatabasePlugin(database.Base):
+    __tablename__ = "plugins"
+
+    id: orm.Mapped[int] = orm.mapped_column(
+        unique=True, primary_key=True, index=True
+    )
+    username: orm.Mapped[str | None]
+    password: orm.Mapped[str | None]
+    remote: orm.Mapped[str]
+    content: orm.Mapped[dict[str, t.Any] | None]

backend/capellacollab/plugins/routes.py

@@ -0,0 +1,154 @@
+# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors
+# SPDX-License-Identifier: Apache-2.0
+
+
+import logging
+import typing as t
+
+import fastapi
+import requests
+import yaml
+from requests import auth
+from sqlalchemy.orm import Session
+
+from capellacollab.core import database
+from capellacollab.core.authentication import injectables as auth_injectables
+from capellacollab.users import models as user_models
+
+logger = logging.getLogger(__name__)
+router = fastapi.APIRouter(
+    dependencies=[
+        fastapi.Depends(
+            auth_injectables.RoleVerification(
+                required_role=user_models.Role.USER
+            )
+        )
+    ]
+)
+
+from . import crud, injectables, models
+
+
+@router.get(
+    "",
+    response_model=list[models.Plugin],
+)
+def get_plugins(
+    db: Session = fastapi.Depends(database.get_db),
+) -> list[models.Plugin]:
+    return [
+        models.Plugin.model_validate(plugin) for plugin in crud.get_plugins(db)
+    ]
+
+
+@router.get(
+    "/{plugin_id}",
+    response_model=models.Plugin,
+    tags=["Plugins"],
+)
+def get_plugin_by_id(
+    plugin: models.DatabasePlugin = fastapi.Depends(
+        injectables.get_existing_plugin
+    ),
+) -> models.DatabasePlugin:
+    return plugin
+
+
+@router.get(
+    "/{plugin_id}/refresh",
+    response_model=list[models.Plugin],
+)
+def refresh_plugins(
+    db: Session = fastapi.Depends(database.get_db),
+    plugin: models.DatabasePlugin = fastapi.Depends(
+        injectables.get_existing_plugin
+    ),
+):
+    plugin.content = fetch_content(
+        plugin.remote, plugin.username, plugin.password
+    )
+    db.commit()
+
+
+@router.post("/peek-plugin-content", response_model=models.Plugin)
+def fetch_plugin_content(plugin: models.CreatePlugin) -> models.Plugin:
+    content = fetch_content(plugin.remote, plugin.username, plugin.password)
+    return models.Plugin(
+        id=0,
+        remote=plugin.remote,
+        username=plugin.username,
+        password=plugin.password,
+        content=content,
+    )
+
+
+def fetch_content(
+    url: str, username: str | None, password: str | None
+) -> dict[str, t.Any]:
+    basic_auth = None
+    if username and password:
+        basic_auth = auth.HTTPBasicAuth(username=username, password=password)
+
+    response = requests.get(url, auth=basic_auth, timeout=2)
+    response.raise_for_status()
+    return yaml.safe_load(response.content.decode())
+
+
+@router.patch(
+    "/{plugin_id}",
+    response_model=models.Plugin,
+    tags=["Plugins"],
+)
+def update_plugin(
+    patch_plugin: models.PatchPlugin,
+    plugin: models.DatabasePlugin = fastapi.Depends(
+        injectables.get_existing_plugin
+    ),
+    db: Session = fastapi.Depends(database.get_db),
+) -> models.DatabasePlugin:
+    patch_plugin.content = fetch_content(
+        patch_plugin.remote, patch_plugin.username, patch_plugin.password
+    )
+    return crud.update_plugin(db, plugin, patch_plugin)
+
+
+@router.post(
+    "",
+    response_model=models.Plugin,
+    dependencies=[
+        fastapi.Depends(
+            auth_injectables.RoleVerification(
+                required_role=user_models.Role.ADMIN
+            )
+        )
+    ],
+    tags=["Plugins"],
+)
+def create_plugin(
+    body: models.CreatePlugin,
+    db: Session = fastapi.Depends(database.get_db),
+) -> models.Plugin:
+    content = fetch_content(body.remote, body.username, body.password)
+    return models.Plugin.model_validate(
+        crud.create_plugin(
+            db,
+            remote=body.remote,
+            username=body.username,
+            password=body.password,
+            content=content,
+        )
+    )
+
+
+@router.delete(
+    "/{plugin_id}",
+    status_code=204,
+    tags=["Plugins"],
+)
+def delete_plugin(
+    plugin: models.DatabasePlugin = fastapi.Depends(
+        injectables.get_existing_plugin
+    ),
+    db: Session = fastapi.Depends(database.get_db),
+) -> None:
+    crud.delete_plugin(db, plugin)

backend/capellacollab/routes.py

@@ -12,6 +12,7 @@
 from capellacollab.core.authentication import responses as auth_responses
 from capellacollab.health import routes as health_routes
 from capellacollab.notices import routes as notices_routes
+from capellacollab.plugins import routes as plugins_routes
 from capellacollab.projects import routes as projects_routes
 from capellacollab.sessions import routes as sessions_routes
 from capellacollab.settings import routes as settings_routes
@@ -40,6 +41,11 @@
     prefix="/projects",
     responses=auth_responses.AUTHENTICATION_RESPONSES,
 )
+router.include_router(
+    plugins_routes.router,
+    prefix="/plugins",
+    responses=auth_responses.AUTHENTICATION_RESPONSES,
+)
 router.include_router(
     tools_routes.router,
     prefix="/tools",