feat: Add support for plugins

The System engineering toolchain team for Digitale Schiene is offering
several model modifiers and model derivative jobs.

Example for model modifiers are:
- ID injection bot (inject IDs into model elements)

Example for model derivates:
- Model complexity badge generation
- Diagram cache generation
- Automatic documentation generation

Synchronisation between TeamForCapella and Git is also considered as job,
somewhere in between.

Currently, all "jobs" are run in the Gitlab CI. There is no UI for the setup of these jobs.
Each job repository provides a Gitlab CI templates, which can be easily integrated in project repositories.
However, this approach is not intuitive for external people, leading to operation support effort for operation teams, which have to take care of the creation and updates of jobs.

Another disadavantage of the current approach is the Gitlab dependency.
Not all environments have a Gitlab instance available. Getting it running on other platforms is even more effort.

This commit adds a plugin/job store to the Capella Colloration Manager.
A plugin/job is still developed in a Git repository. Plugins have to define a
`mbse-works-plugin.yml`, providing metadata, input, output, trigger and job information.

The JSON schema can be fetched via `GET /api/v1.0/plugins-schema` (backend).
A human-readable documentation is available via `/docs/plugin-schema` (backend).

With this information, registered plugins can be easily integrated into projects.
The integration should be intuitive for project leads.

The steps are:
- Create a new pipeline
- Select a supported plugin from the store
- Configure the plugin: Supported types for the beginning are:
    - git (Select a linked git model from the project, so that the job can access the Git repository information)
    - t4c (Select a linked T4C repository/project from the project). The job gets the connection information + a session token.
    - yml (Most flexible option, the yml configuration file is mounted into the job container, validation via yml schema)
    - environment (Key/value pairs, which are used as environment variable for the job)
- Confirm the creation

As part of this commit, there are only two options:
- Run the pipeline manually
- Run it as 3am during the night.

Co-authored-by: ewuerger <[email protected]>
Co-authored-by: Paula-Kli <[email protected]>

backend/Makefile

@@ -68,3 +68,9 @@ load: clear
 
 save:
 	docker run -i -e PGPASSWORD=$(DB_PASSWORD) --network host --entrypoint="pg_dump" postgres:latest -h 'localhost' -p $(DB_PORT) -U '$(DB_USER)' $(DB_NAME) > $(DATABASE_SAVE_DIR)/$(shell date +"%FT%T").sql
+
+plugin-schema:
+	$(VENV)/bin/pip show json-schema-for-humans > /dev/null || $(VENV)/bin/pip install json-schema-for-humans
+	$(VENV)/bin/python -c 'import pathlib; import json; from capellacollab.plugins import models; pathlib.Path("plugin_schema.yml").write_text(json.dumps(models.PluginContent.model_json_schema()))'
+	mkdir -p plugin_schema
+	$(VENV)/bin/generate-schema-doc plugin_schema.yml plugin_schema/plugin_schema.html

backend/capellacollab/__main__.py

@@ -9,10 +9,10 @@
 import fastapi_pagination
 import starlette_prometheus
 import uvicorn
-from fastapi import middleware, responses
+from fastapi import middleware, responses, staticfiles
 from fastapi.middleware import cors
 
-import capellacollab.projects.toolmodels.backups.runs.interface as pipeline_runs_interface
+import capellacollab.projects.toolmodels.pipelines.runs.interface as pipeline_runs_interface
 import capellacollab.sessions.metrics
 
 # This import statement is required and should not be removed! (Alembic will not work otherwise)
@@ -21,12 +21,10 @@
 from capellacollab.core import logging as core_logging
 from capellacollab.core.database import engine, migration
 from capellacollab.core.logging import exceptions as logging_exceptions
+from capellacollab.plugins import schema as pipeline_schema
 from capellacollab.projects.toolmodels import (
     exceptions as toolmodels_exceptions,
 )
-from capellacollab.projects.toolmodels.backups import (
-    exceptions as backups_exceptions,
-)
 from capellacollab.projects.toolmodels.modelsources.git import (
     exceptions as git_exceptions,
 )
@@ -36,6 +34,9 @@
 from capellacollab.projects.toolmodels.modelsources.git.handler import (
     exceptions as git_handler_exceptions,
 )
+from capellacollab.projects.toolmodels.pipelines import (
+    exceptions as backups_exceptions,
+)
 from capellacollab.routes import router
 from capellacollab.sessions import exceptions as sessions_exceptions
 from capellacollab.sessions import idletimeout, operators
@@ -66,7 +67,7 @@ async def startup():
     operators.get_operator()
 
     logging.getLogger("uvicorn.access").disabled = True
-    logging.getLogger("uvicorn.error").disabled = True
+    logging.getLogger("uvicorn.error").disabled = False
     logging.getLogger("requests_oauthlib.oauth2_session").setLevel("INFO")
     logging.getLogger("kubernetes.client.rest").setLevel("INFO")
 
@@ -83,6 +84,7 @@ async def shutdown():
         idletimeout.terminate_idle_sessions_in_background,
         capellacollab.sessions.metrics.register,
         pipeline_runs_interface.schedule_refresh_and_trigger_pipeline_jobs,
+        pipeline_schema.generate_schema_documentation,
     ],
     middleware=[
         middleware.Middleware(
@@ -98,7 +100,7 @@ async def shutdown():
         middleware.Middleware(core_logging.LogRequestsMiddleware),
         middleware.Middleware(starlette_prometheus.PrometheusMiddleware),
     ],
-    on_shutdown=[shutdown],
+    on_shutdown=[shutdown, pipeline_schema.cleanup_documentation_directory],
 )
 
 fastapi_pagination.add_pagination(app)
@@ -135,6 +137,7 @@ async def healthcheck():
 
 app.add_route("/metrics", starlette_prometheus.metrics)
 app.include_router(router, prefix="/api/v1")
+pipeline_schema.mount_schema_documentation(app)
 
 
 def register_exceptions():

backend/capellacollab/alembic/versions/e3f1006f6b49_add_plugins_table.py

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Add plugins table
+
+Revision ID: e3f1006f6b49
+Revises: d0cbf2813066
+Create Date: 2023-05-26 11:56:13.928534
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "e3f1006f6b49"
+down_revision = "ac0e6e0f77ee"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "plugins",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("username", sa.String(), nullable=True),
+        sa.Column("password", sa.String(), nullable=True),
+        sa.Column("remote", sa.String(), nullable=False),
+        sa.Column(
+            "content", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_plugins_id"), "plugins", ["id"], unique=True)

backend/capellacollab/core/database/__init__.py

@@ -18,7 +18,10 @@
 
 
 class Base(orm.DeclarativeBase):
-    type_annotation_map = {dict[str, str]: postgresql.JSONB}
+    type_annotation_map = {
+        dict[str, str]: postgresql.JSONB,
+        dict[str, t.Any]: postgresql.JSONB,
+    }
 
 
 ### SQL MODELS ARE IMPORTED HERE ###

backend/capellacollab/core/database/models.py

@@ -5,12 +5,13 @@
 # These import statements of the models are required and should not be removed! (SQLAlchemy will not load the models otherwise)
 
 import capellacollab.notices.models
+import capellacollab.plugins.models
 import capellacollab.projects.models
-import capellacollab.projects.toolmodels.backups.models
-import capellacollab.projects.toolmodels.backups.runs.models
 import capellacollab.projects.toolmodels.models
 import capellacollab.projects.toolmodels.modelsources.git.models
 import capellacollab.projects.toolmodels.modelsources.t4c.models
+import capellacollab.projects.toolmodels.pipelines.models
+import capellacollab.projects.toolmodels.pipelines.runs.models
 import capellacollab.projects.toolmodels.restrictions.models
 import capellacollab.projects.users.models
 import capellacollab.sessions.models

backend/capellacollab/health/models.py

@@ -6,12 +6,12 @@
 
 import pydantic
 
-from capellacollab.projects.toolmodels.backups.runs import (
-    models as pipeline_run_models,
-)
 from capellacollab.projects.toolmodels.modelsources.git import (
     models as git_models,
 )
+from capellacollab.projects.toolmodels.pipelines.runs import (
+    models as pipeline_run_models,
+)
 
 
 class StatusResponse(pydantic.BaseModel):

backend/capellacollab/health/routes.py

@@ -11,11 +11,11 @@
 import capellacollab.core.authentication.injectables as auth_injectables
 import capellacollab.core.logging as core_logging
 import capellacollab.projects.crud as projects_crud
-import capellacollab.projects.toolmodels.backups.validation as pipelines_validation
 import capellacollab.projects.toolmodels.crud as toolmodels_crud
 import capellacollab.projects.toolmodels.diagrams.validation as diagrams_validation
 import capellacollab.projects.toolmodels.modelbadge.validation as modelbadge_validation
 import capellacollab.projects.toolmodels.modelsources.git.validation as git_validation
+import capellacollab.projects.toolmodels.pipelines.validation as pipelines_validation
 import capellacollab.projects.toolmodels.validation as toolmodels_validation
 import capellacollab.projects.validation as projects_validation
 import capellacollab.users.models as users_models

backend/capellacollab/plugins/crud.py

@@ -0,0 +1,62 @@
+# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors
+# SPDX-License-Identifier: Apache-2.0
+
+
+from collections import abc
+
+import sqlalchemy as sa
+from sqlalchemy import orm
+
+from . import models
+
+
+def get_plugins(db: orm.Session) -> abc.Sequence[models.DatabasePlugin]:
+    return db.execute(sa.select(models.DatabasePlugin)).scalars().all()
+
+
+def get_plugin_by_id(
+    db: orm.Session, plugin_id: int
+) -> models.DatabasePlugin | None:
+    return db.execute(
+        sa.select(models.DatabasePlugin).where(
+            models.DatabasePlugin.id == plugin_id
+        )
+    ).scalar_one_or_none()
+
+
+def create_plugin(
+    db: orm.Session,
+    remote: str,
+    username: str | None,
+    password: str | None,
+    content: dict | None,
+) -> models.DatabasePlugin:
+    plugin = models.DatabasePlugin(
+        remote=remote, username=username, password=password, content=content
+    )
+    db.add(plugin)
+    db.commit()
+    return plugin
+
+
+def update_plugin(
+    db: orm.Session,
+    plugin: models.DatabasePlugin,
+    patch_plugin: models.PatchPlugin,
+    content: str,
+) -> models.DatabasePlugin:
+    if patch_plugin.username:
+        plugin.username = patch_plugin.username
+    if patch_plugin.password:
+        plugin.password = patch_plugin.password
+    if patch_plugin.remote:
+        plugin.remote = patch_plugin.remote
+    if content:
+        plugin.content = content
+    db.commit()
+    return plugin
+
+
+def delete_plugin(db: orm.Session, plugin: models.DatabasePlugin) -> None:
+    db.delete(plugin)
+    db.commit()

backend/capellacollab/plugins/injectables.py

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors
+# SPDX-License-Identifier: Apache-2.0
+
+from fastapi import Depends, HTTPException
+from sqlalchemy.orm import Session
+
+from capellacollab.core import database
+
+from . import crud, models
+
+
+def get_existing_plugin(
+    plugin_id: int, db: Session = Depends(database.get_db)
+) -> models.DatabasePlugin:
+    if plugin := crud.get_plugin_by_id(db, plugin_id):
+        return plugin
+
+    raise HTTPException(
+        404,
+        {
+            "reason": f"The plugin with the id {plugin_id} was not found.",
+        },
+    )

backend/capellacollab/plugins/models.py

@@ -0,0 +1,97 @@
+# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import typing as t
+
+import croniter
+import pydantic
+from sqlalchemy import orm
+
+from capellacollab.core import database
+
+
+class CreatePlugin(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(from_attributes=True)
+
+    username: str | None
+    password: str | None
+    remote: str
+
+
+class PatchPlugin(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(from_attributes=True)
+
+    username: str | None = None
+    password: str | None = None
+    remote: str | None = None
+
+
+class PluginMetadata(pydantic.BaseModel):
+    id: str = pydantic.Field(
+        title="Plugin identifier",
+        description="Unique identifier of the plugin.",
+        examples=["hello-world", "test-plugin"],
+        min_length=1,
+        max_length=50,
+    )
+    displayName: str | None = pydantic.Field(
+        title="Display name of the plugin for the frontend",
+        description="Display name for the plugin. The name is used in the frontend to display the plugin. If ommitted, the id is used instead.",
+        examples=["Hello world", "A to B synchronization"],
+        min_length=0,
+        max_length=200,
+    )
+    description: str | None = pydantic.Field(
+        title="Description of the plugin",
+        description=" ".join(
+            [
+                "Description of the plugin.",
+                "The description should explain the purpose of the plugin.",
+                "The user should be able to understand what the plugin does by reading the description.",
+            ],
+        ),
+        examples=[
+            "This plugin runs the hello-world Docker container.",
+            "Synchronize the content from A to B.",
+        ],
+        min_length=0,
+        max_length=500,
+    )
+
+
+class PluginContent(pydantic.BaseModel):
+    metadata: PluginMetadata
+
+
+class Plugin(CreatePlugin):
+    id: int
+    content: dict | None
+
+
+class PluginTrigger(pydantic.BaseModel):
+    cron: str
+    manual: bool
+
+    @pydantic.field_validator("cron")
+    @classmethod
+    def validate_cron_expression(cls, value):
+        try:
+            # Attempt to create a croniter object with the expression
+            croniter.croniter(value)
+        except (ValueError, croniter.CroniterBadCronError):
+            raise ValueError("Invalid cron expression")
+        return value
+
+
+class DatabasePlugin(database.Base):
+    __tablename__ = "plugins"
+
+    id: orm.Mapped[int] = orm.mapped_column(
+        unique=True, primary_key=True, index=True
+    )
+    username: orm.Mapped[str | None]
+    password: orm.Mapped[str | None]
+    remote: orm.Mapped[str]
+    content: orm.Mapped[dict[str, t.Any] | None]