feat: Improve git handler and introduce caching

.pre-commit-config.yaml

@@ -65,6 +65,7 @@ repos:
           - capellambse
           - typer
           - types-lxml
+          - types-redis
   - repo: local
     hooks:
       - id: pylint

backend/Makefile

@@ -7,6 +7,9 @@ DB_USER = dev
 DB_NAME = dev
 VENV = .venv
 
+REDIS_PORT = 6379
+REDIS_INSIGHT_PORT = 8001
+
 DATABASE_LOAD_FILE ?= ../local/load.sql
 DATABASE_SAVE_DIR ?= ../local
 
@@ -29,6 +32,14 @@ database:
 		-e POSTGRES_DB=$(DB_NAME) \
 		postgres
 
+redis:
+	docker start redis || \
+	docker run -d \
+		--name redis \
+		-p $(REDIS_PORT):6379 \
+		-p $(REDIS_INSIGHT_PORT):8001 \
+		redis/redis-stack:latest
+
 app:
 	if [ -d "$(VENV)/bin" ]; then
 		source $(VENV)/bin/activate;

backend/capellacollab/alembic/versions/abddaf015966_add_repository_id_to_git_model_and_remove_unused_name.py

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Add repository id to git model and remove unused git model name
+
+Revision ID: abddaf015966
+Revises: 028c72ddfd20
+Create Date: 2024-08-12 11:43:34.158404
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "abddaf015966"
+down_revision = "028c72ddfd20"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "git_models", sa.Column("repository_id", sa.String(), nullable=True)
+    )
+    op.drop_column("git_models", "name")

backend/capellacollab/core/database/__init__.py

@@ -1,9 +1,11 @@
 # SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
 # SPDX-License-Identifier: Apache-2.0
 
+import functools
 import typing as t
 
 import pydantic
+import redis
 import sqlalchemy as sa
 from sqlalchemy import orm
 from sqlalchemy.dialects import postgresql
@@ -35,6 +37,11 @@ def get_db() -> t.Iterator[orm.Session]:
         yield session
 
 
+@functools.lru_cache
+def get_redis() -> redis.Redis:
+    return redis.Redis(host="localhost", port=6379, decode_responses=True)
+
+
 def patch_database_with_pydantic_object(
     database_object: Base, pydantic_object: pydantic.BaseModel
 ):

backend/capellacollab/projects/toolmodels/diagrams/models.py

@@ -18,6 +18,7 @@ class DiagramMetadata(core_pydantic.BaseModel):
 class DiagramCacheMetadata(core_pydantic.BaseModel):
     diagrams: list[DiagramMetadata]
     last_updated: datetime.datetime
+    job_id: str | None = None
 
     _validate_last_updated = pydantic.field_serializer("last_updated")(
         core_pydantic.datetime_serializer

backend/capellacollab/projects/toolmodels/diagrams/routes.py

@@ -3,12 +3,14 @@
 
 from __future__ import annotations
 
+import json
 import logging
 import pathlib
 from urllib import parse
 
 import fastapi
 import requests
+from aiohttp import web
 
 import capellacollab.projects.toolmodels.modelsources.git.injectables as git_injectables
 from capellacollab.core import logging as log
@@ -40,25 +42,43 @@ async def get_diagram_metadata(
     ),
     logger: logging.LoggerAdapter = fastapi.Depends(log.get_request_logger),
 ):
+    job_id = None
     try:
-        (
-            last_updated,
-            diagram_metadata_entries,
-        ) = await handler.get_file_from_repository_or_artifacts_as_json(
-            "diagram_cache/index.json",
-            "update_capella_diagram_cache",
-            "diagram-cache/" + handler.git_model.revision,
+        last_updated, diagram_metadata_entries = await handler.get_file(
+            trusted_file_path="diagram_cache/index.json",
+            revision=f"diagram-cache/{handler.revision}",
         )
-    except requests.exceptions.HTTPError:
-        logger.info("Failed fetching diagram metadata", exc_info=True)
-        raise exceptions.DiagramCacheNotConfiguredProperlyError()
+    except Exception:
+        logger.info(
+            "Failed fetching diagram metadata file for %s on revision %s.",
+            handler.path,
+            f"diagram-cache/{handler.revision}",
+            exc_info=True,
+        )
+        try:
+            job_id, last_updated, diagram_metadata_entries = (
+                await handler.get_artifact(
+                    trusted_file_path="diagram_cache/index.json",
+                    job_name="update_capella_diagram_cache",
+                )
+            )
+        except (web.HTTPError, requests.HTTPError):
+            logger.info(
+                "Failed fetching diagram metadata artifact for %s on revision %s",
+                handler.path,
+                handler.revision,
+                exc_info=True,
+            )
+            raise exceptions.DiagramCacheNotConfiguredProperlyError()
 
+    diagram_metadata_entries = json.loads(diagram_metadata_entries.decode())
     return models.DiagramCacheMetadata(
         diagrams=[
             models.DiagramMetadata.model_validate(diagram_metadata)
             for diagram_metadata in diagram_metadata_entries
         ],
         last_updated=last_updated,
+        job_id=job_id,
     )
 
 
@@ -69,6 +89,7 @@ async def get_diagram_metadata(
 )
 async def get_diagram(
     diagram_uuid_or_filename: str,
+    job_id: str | None = None,
     handler: git_handler.GitHandler = fastapi.Depends(
         git_injectables.get_git_handler
     ),
@@ -79,16 +100,37 @@ async def get_diagram(
         raise exceptions.FileExtensionNotSupportedError(fileextension)
 
     diagram_uuid = pathlib.PurePosixPath(diagram_uuid_or_filename).stem
+    file_path = f"diagram_cache/{parse.quote(diagram_uuid, safe='')}.svg"
+
+    if not job_id:
+        try:
+            file = await handler.get_file(
+                trusted_file_path=file_path,
+                revision=f"diagram-cache/{handler.revision}",
+            )
+            return responses.SVGResponse(content=file[1])
+        except Exception:
+            logger.info(
+                "Failed fetching diagram file %s for %s on revision %s.",
+                diagram_uuid,
+                handler.path,
+                f"diagram-cache/{handler.revision}",
+                exc_info=True,
+            )
+
     try:
-        _, diagram = await handler.get_file_from_repository_or_artifacts(
-            f"diagram_cache/{parse.quote(diagram_uuid, safe='')}.svg",
-            "update_capella_diagram_cache",
-            "diagram-cache/" + handler.git_model.revision,
+        artifact = await handler.get_artifact(
+            trusted_file_path=file_path,
+            job_name="update_capella_diagram_cache",
+            job_id=job_id,
+        )
+        return responses.SVGResponse(content=artifact[2])
+    except (web.HTTPError, requests.HTTPError):
+        logger.info(
+            "Failed fetching diagram artifact %s for %s on revision %s.",
+            diagram_uuid,
+            handler.path,
+            f"diagram-cache/{handler.revision}",
+            exc_info=True,
         )
-    except requests.exceptions.HTTPError:
-        logger.info("Failed fetching diagram", exc_info=True)
         raise exceptions.DiagramCacheNotConfiguredProperlyError()
-
-    return responses.SVGResponse(
-        content=diagram,
-    )

backend/capellacollab/projects/toolmodels/modelbadge/routes.py

@@ -5,9 +5,9 @@
 
 import logging
 
-import aiohttp.web
 import fastapi
 import requests
+from aiohttp import web
 
 import capellacollab.projects.toolmodels.modelsources.git.injectables as git_injectables
 from capellacollab.core import logging as log
@@ -41,13 +41,26 @@ async def get_model_complexity_badge(
     logger: logging.LoggerAdapter = fastapi.Depends(log.get_request_logger),
 ):
     try:
-        return responses.SVGResponse(
-            content=(
-                await git_handler.get_file_from_repository_or_artifacts(
-                    "model-complexity-badge.svg", "generate-model-badge"
-                )
-            )[1],
+        file = await git_handler.get_file("model-complexity-badge.svg")
+        return responses.SVGResponse(content=file[1])
+    except Exception:
+        logger.debug(
+            "Failed fetching model badge file for %s on revision %s.",
+            git_handler.path,
+            git_handler.revision,
+            exc_info=True,
+        )
+
+    try:
+        artifact = await git_handler.get_artifact(
+            "model-complexity-badge.svg", "generate-model-badge"
+        )
+        return responses.SVGResponse(content=artifact[2])
+    except (web.HTTPError, requests.HTTPError):
+        logger.debug(
+            "Failed fetching model badge artifact for %s on revision %s.",
+            git_handler.path,
+            git_handler.revision,
+            exc_info=True,
         )
-    except (aiohttp.web.HTTPException, requests.exceptions.HTTPError):
-        logger.info("Failed fetching model complexity badge", exc_info=True)
         raise exceptions.ModelBadgeNotConfiguredProperlyError()

backend/capellacollab/projects/toolmodels/modelsources/git/crud.py

@@ -61,26 +61,38 @@ def make_git_model_primary(
 def update_git_model(
     db: orm.Session,
     git_model: models.DatabaseGitModel,
-    patch_model: models.PatchGitModel,
+    put_model: models.PutGitModel,
 ) -> models.DatabaseGitModel:
-    git_model.path = patch_model.path
-    git_model.entrypoint = patch_model.entrypoint
-    git_model.revision = patch_model.revision
-
-    if patch_model.password:
-        git_model.username = patch_model.username
-        git_model.password = patch_model.password
-    elif not patch_model.username:
+    git_model.entrypoint = put_model.entrypoint
+    git_model.revision = put_model.revision
+
+    if put_model.path != git_model.path:
+        git_model.path = put_model.path
+        git_model.repository_id = None
+
+    if put_model.password:
+        git_model.username = put_model.username
+        git_model.password = put_model.password
+    elif not put_model.username:
         git_model.username = ""
         git_model.password = ""
 
-    if patch_model.primary and not git_model.primary:
+    if put_model.primary and not git_model.primary:
         git_model = make_git_model_primary(db, git_model)
 
     db.commit()
     return git_model
 
 
+def update_git_model_repository_id(
+    db: orm.Session, git_model: models.DatabaseGitModel, repository_id: str
+) -> models.DatabaseGitModel:
+    git_model.repository_id = repository_id
+
+    db.commit()
+    return git_model
+
+
 def delete_git_model(db: orm.Session, git_model: models.DatabaseGitModel):
     db.delete(git_model)
     db.commit()

backend/capellacollab/projects/toolmodels/modelsources/git/exceptions.py

@@ -71,19 +71,6 @@ def __init__(self, filename: str):
         )
 
 
-class GitInstanceAPIEndpointNotFoundError(core_exceptions.BaseError):
-    def __init__(self):
-        super().__init__(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            title="Git instance API endpoint not found",
-            reason=(
-                "The used Git instance has no API endpoint defined. "
-                "Please contact your administrator."
-            ),
-            err_code="GIT_INSTANCE_NO_API_ENDPOINT_DEFINED",
-        )
-
-
 class GitPipelineJobNotFoundError(core_exceptions.BaseError):
     def __init__(self, job_name: str, revision: str):
         super().__init__(
@@ -97,31 +84,18 @@ def __init__(self, job_name: str, revision: str):
         )
 
 
-class GitPipelineJobFailedError(core_exceptions.BaseError):
-    def __init__(self, job_name: str):
-        super().__init__(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            title="Failed job found",
-            reason=f"The last job with the name '{job_name}' has failed.",
-            err_code="FAILED_JOB_FOUND",
-        )
-
-
-class GitPipelineJobUnknownStateError(core_exceptions.BaseError):
-    job_name: str
-    state: str
-
+class GitPipelineJobUnsuccessfulError(core_exceptions.BaseError):
     def __init__(self, job_name: str, state: str):
         self.job_name = job_name
         self.state = state
         super().__init__(
             status_code=status.HTTP_400_BAD_REQUEST,
-            title="Unknown job state",
+            title="Unsuccessful job",
             reason=(
-                f"Job '{job_name}' has an unhandled or unknown state: '{state}'. "
+                f"Job '{job_name}' has an unsuccessful state: {self.state}."
                 "Please contact your administrator."
             ),
-            err_code="UNKNOWN_STATE_ERROR",
+            err_code="UNSUCCESSFUL_JOB_STATE_ERROR",
         )