feat: Add support for pipeline plugins

backend/Makefile

@@ -68,3 +68,9 @@ load: clear
 
 save:
 	docker run -i -e PGPASSWORD=$(DB_PASSWORD) --network host --entrypoint="pg_dump" postgres:latest -h 'localhost' -p $(DB_PORT) -U '$(DB_USER)' $(DB_NAME) > $(DATABASE_SAVE_DIR)/$(shell date +"%FT%T").sql
+
+plugin-schema:
+	$(VENV)/bin/pip show json-schema-for-humans > /dev/null || $(VENV)/bin/pip install json-schema-for-humans
+	$(VENV)/bin/python -c 'import pathlib; import json; from capellacollab.plugins import models; pathlib.Path("plugin_schema.yml").write_text(json.dumps(models.PluginContent.model_json_schema()))'
+	mkdir -p plugin_schema
+	$(VENV)/bin/generate-schema-doc plugin_schema.yml plugin_schema/plugin_schema.html

backend/capellacollab/__main__.py

@@ -9,10 +9,10 @@
 import fastapi_pagination
 import starlette_prometheus
 import uvicorn
-from fastapi import middleware, responses
+from fastapi import middleware, responses, staticfiles
 from fastapi.middleware import cors
 
-import capellacollab.projects.toolmodels.backups.runs.interface as pipeline_runs_interface
+import capellacollab.projects.toolmodels.pipelines.runs.interface as pipeline_runs_interface
 import capellacollab.sessions.metrics
 
 # This import statement is required and should not be removed! (Alembic will not work otherwise)
@@ -21,12 +21,10 @@
 from capellacollab.core import logging as core_logging
 from capellacollab.core.database import engine, migration
 from capellacollab.core.logging import exceptions as logging_exceptions
+from capellacollab.plugins import schema as pipeline_schema
 from capellacollab.projects.toolmodels import (
     exceptions as toolmodels_exceptions,
 )
-from capellacollab.projects.toolmodels.backups import (
-    exceptions as backups_exceptions,
-)
 from capellacollab.projects.toolmodels.modelsources.git import (
     exceptions as git_exceptions,
 )
@@ -36,6 +34,9 @@
 from capellacollab.projects.toolmodels.modelsources.git.handler import (
     exceptions as git_handler_exceptions,
 )
+from capellacollab.projects.toolmodels.pipelines import (
+    exceptions as backups_exceptions,
+)
 from capellacollab.routes import router
 from capellacollab.sessions import exceptions as sessions_exceptions
 from capellacollab.sessions import idletimeout, operators
@@ -66,7 +67,7 @@ async def startup():
     operators.get_operator()
 
     logging.getLogger("uvicorn.access").disabled = True
-    logging.getLogger("uvicorn.error").disabled = True
+    logging.getLogger("uvicorn.error").disabled = False
     logging.getLogger("requests_oauthlib.oauth2_session").setLevel("INFO")
     logging.getLogger("kubernetes.client.rest").setLevel("INFO")
 
@@ -83,6 +84,7 @@ async def shutdown():
         idletimeout.terminate_idle_sessions_in_background,
         capellacollab.sessions.metrics.register,
         pipeline_runs_interface.schedule_refresh_and_trigger_pipeline_jobs,
+        pipeline_schema.generate_schema_documentation,
     ],
     middleware=[
         middleware.Middleware(
@@ -98,7 +100,7 @@ async def shutdown():
         middleware.Middleware(core_logging.LogRequestsMiddleware),
         middleware.Middleware(starlette_prometheus.PrometheusMiddleware),
     ],
-    on_shutdown=[shutdown],
+    on_shutdown=[shutdown, pipeline_schema.cleanup_documentation_directory],
 )
 
 fastapi_pagination.add_pagination(app)
@@ -135,6 +137,7 @@ async def healthcheck():
 
 app.add_route("/metrics", starlette_prometheus.metrics)
 app.include_router(router, prefix="/api/v1")
+pipeline_schema.mount_schema_documentation(app)
 
 
 def register_exceptions():

backend/capellacollab/alembic/versions/e3f1006f6b49_add_plugins_table.py

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Add plugins table
+
+Revision ID: e3f1006f6b49
+Revises: d0cbf2813066
+Create Date: 2023-05-26 11:56:13.928534
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "e3f1006f6b49"
+down_revision = "ac0e6e0f77ee"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "plugins",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("username", sa.String(), nullable=True),
+        sa.Column("password", sa.String(), nullable=True),
+        sa.Column("remote", sa.String(), nullable=False),
+        sa.Column(
+            "content", postgresql.JSONB(astext_type=sa.Text()), nullable=True
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_plugins_id"), "plugins", ["id"], unique=True)

backend/capellacollab/core/database/__init__.py

@@ -18,7 +18,10 @@
 
 
 class Base(orm.DeclarativeBase):
-    type_annotation_map = {dict[str, str]: postgresql.JSONB}
+    type_annotation_map = {
+        dict[str, str]: postgresql.JSONB,
+        dict[str, t.Any]: postgresql.JSONB,
+    }
 
 
 ### SQL MODELS ARE IMPORTED HERE ###

backend/capellacollab/core/database/models.py

@@ -5,12 +5,13 @@
 # These import statements of the models are required and should not be removed! (SQLAlchemy will not load the models otherwise)
 
 import capellacollab.notices.models
+import capellacollab.plugins.models
 import capellacollab.projects.models
-import capellacollab.projects.toolmodels.backups.models
-import capellacollab.projects.toolmodels.backups.runs.models
 import capellacollab.projects.toolmodels.models
 import capellacollab.projects.toolmodels.modelsources.git.models
 import capellacollab.projects.toolmodels.modelsources.t4c.models
+import capellacollab.projects.toolmodels.pipelines.models
+import capellacollab.projects.toolmodels.pipelines.runs.models
 import capellacollab.projects.toolmodels.restrictions.models
 import capellacollab.projects.users.models
 import capellacollab.sessions.models

backend/capellacollab/health/models.py

@@ -6,12 +6,12 @@
 
 import pydantic
 
-from capellacollab.projects.toolmodels.backups.runs import (
-    models as pipeline_run_models,
-)
 from capellacollab.projects.toolmodels.modelsources.git import (
     models as git_models,
 )
+from capellacollab.projects.toolmodels.pipelines.runs import (
+    models as pipeline_run_models,
+)
 
 
 class StatusResponse(pydantic.BaseModel):

backend/capellacollab/health/routes.py

@@ -11,11 +11,11 @@
 import capellacollab.core.authentication.injectables as auth_injectables
 import capellacollab.core.logging as core_logging
 import capellacollab.projects.crud as projects_crud
-import capellacollab.projects.toolmodels.backups.validation as pipelines_validation
 import capellacollab.projects.toolmodels.crud as toolmodels_crud
 import capellacollab.projects.toolmodels.diagrams.validation as diagrams_validation
 import capellacollab.projects.toolmodels.modelbadge.validation as modelbadge_validation
 import capellacollab.projects.toolmodels.modelsources.git.validation as git_validation
+import capellacollab.projects.toolmodels.pipelines.validation as pipelines_validation
 import capellacollab.projects.toolmodels.validation as toolmodels_validation
 import capellacollab.projects.validation as projects_validation
 import capellacollab.users.models as users_models

backend/capellacollab/plugins/crud.py

@@ -0,0 +1,62 @@
+# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors
+# SPDX-License-Identifier: Apache-2.0
+
+
+from collections import abc
+
+import sqlalchemy as sa
+from sqlalchemy import orm
+
+from . import models
+
+
+def get_plugins(db: orm.Session) -> abc.Sequence[models.DatabasePlugin]:
+    return db.execute(sa.select(models.DatabasePlugin)).scalars().all()
+
+
+def get_plugin_by_id(
+    db: orm.Session, plugin_id: int
+) -> models.DatabasePlugin | None:
+    return db.execute(
+        sa.select(models.DatabasePlugin).where(
+            models.DatabasePlugin.id == plugin_id
+        )
+    ).scalar_one_or_none()
+
+
+def create_plugin(
+    db: orm.Session,
+    remote: str,
+    username: str | None,
+    password: str | None,
+    content: dict | None,
+) -> models.DatabasePlugin:
+    plugin = models.DatabasePlugin(
+        remote=remote, username=username, password=password, content=content
+    )
+    db.add(plugin)
+    db.commit()
+    return plugin
+
+
+def update_plugin(
+    db: orm.Session,
+    plugin: models.DatabasePlugin,
+    patch_plugin: models.PatchPlugin,
+    content: str,
+) -> models.DatabasePlugin:
+    if patch_plugin.username:
+        plugin.username = patch_plugin.username
+    if patch_plugin.password:
+        plugin.password = patch_plugin.password
+    if patch_plugin.remote:
+        plugin.remote = patch_plugin.remote
+    if content:
+        plugin.content = content
+    db.commit()
+    return plugin
+
+
+def delete_plugin(db: orm.Session, plugin: models.DatabasePlugin) -> None:
+    db.delete(plugin)
+    db.commit()

backend/capellacollab/plugins/injectables.py

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors
+# SPDX-License-Identifier: Apache-2.0
+
+from fastapi import Depends, HTTPException
+from sqlalchemy.orm import Session
+
+from capellacollab.core import database
+
+from . import crud, models
+
+
+def get_existing_plugin(
+    plugin_id: int, db: Session = Depends(database.get_db)
+) -> models.DatabasePlugin:
+    if plugin := crud.get_plugin_by_id(db, plugin_id):
+        return plugin
+
+    raise HTTPException(
+        404,
+        {
+            "reason": f"The plugin with the id {plugin_id} was not found.",
+        },
+    )