Merge pull request #1689 from DSD-DBS/refactor-git-handler

feat: Improve Git handler and introduce caching

.gitignore

@@ -17,3 +17,4 @@ certs/*
 /helm/charts/*
 /logs/*
 .env
+autoscaler

Makefile

@@ -152,6 +152,12 @@ create-cluster: registry
 	kubectl cluster-info
 	kubectl config set-context --current --namespace=$(NAMESPACE)
 
+install-vpa:
+	git clone https://github.com/kubernetes/autoscaler.git
+	cd autoscaler/vertical-pod-autoscaler
+	./hack/vpa-up.sh
+	kubectl --namespace=kube-system get pods | grep vpa
+
 delete-cluster:
 	k3d cluster list $(CLUSTER_NAME) 2>&- && k3d cluster delete $(CLUSTER_NAME)
 

backend/Makefile

@@ -5,10 +5,15 @@ DB_PORT = 5432
 DB_PASSWORD = dev
 DB_USER = dev
 DB_NAME = dev
+
+VALKEY_PASSWORD ?= password
+
 VENV = .venv
 
 HOST ?= 127.0.0.1
 
+VALKEY_PORT = 6379
+
 DATABASE_LOAD_FILE ?= ../local/load.sql
 DATABASE_SAVE_DIR ?= ../local
 
@@ -31,6 +36,18 @@ database:
 		-e POSTGRES_DB=$(DB_NAME) \
 		postgres
 
+valkey:
+	TMP_FILE=$$(mktemp)
+	echo "requirepass $(VALKEY_PASSWORD)" > $$TMP_FILE
+	docker start capella-collab-valkey || \
+	docker run -d \
+		--name capella-collab-valkey \
+		-p $(VALKEY_PORT):6379 \
+		-v $$TMP_FILE:/usr/local/etc/valkey/valkey.conf \
+		valkey/valkey:latest \
+		valkey-server \
+		/usr/local/etc/valkey/valkey.conf
+
 app:
 	if [ -d "$(VENV)/bin" ]; then
 		source $(VENV)/bin/activate;
@@ -57,7 +74,7 @@ install:
 openapi:
 	CAPELLACOLLAB_SKIP_OPENAPI_ERROR_RESPONSES=1 $(VENV)/bin/python -m capellacollab.cli openapi generate /tmp/openapi.json
 
-dev: database app
+dev: database valkey app
 
 cleanup:
 	docker stop capella-collab-postgres

backend/capellacollab/alembic/versions/abddaf015966_add_repository_id_to_git_model_and_remove_unused_name.py

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
+# SPDX-License-Identifier: Apache-2.0
+
+"""Add repository id to git model and remove unused git model name
+
+Revision ID: abddaf015966
+Revises: 028c72ddfd20
+Create Date: 2024-08-12 11:43:34.158404
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "abddaf015966"
+down_revision = "028c72ddfd20"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "git_models", sa.Column("repository_id", sa.String(), nullable=True)
+    )
+    op.drop_column("git_models", "name")

backend/capellacollab/config/models.py

@@ -288,6 +288,12 @@ class DatabaseConfig(BaseConfig):
     )
 
 
+class ValkeyConfig(BaseConfig):
+    url: str = pydantic.Field(
+        default="valkey://default:password@localhost:6379/0"
+    )
+
+
 class InitialConfig(BaseConfig):
     admin: str = pydantic.Field(
         default="admin",
@@ -367,6 +373,7 @@ class AppConfig(BaseConfig):
     authentication: AuthenticationConfig = AuthenticationConfig()
     prometheus: PrometheusConfig = PrometheusConfig()
     database: DatabaseConfig = DatabaseConfig()
+    valkey: ValkeyConfig = ValkeyConfig()
     initial: InitialConfig = InitialConfig()
     logging: LoggingConfig = LoggingConfig()
     requests: RequestsConfig = RequestsConfig()

backend/capellacollab/core/database/__init__.py

@@ -1,10 +1,12 @@
 # SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors
 # SPDX-License-Identifier: Apache-2.0
 
+import functools
 import typing as t
 
 import pydantic
 import sqlalchemy as sa
+import valkey
 from sqlalchemy import orm
 from sqlalchemy.dialects import postgresql
 
@@ -35,6 +37,11 @@ def get_db() -> t.Iterator[orm.Session]:
         yield session
 
 
+@functools.lru_cache
+def get_valkey() -> valkey.Valkey:
+    return valkey.Valkey.from_url(config.valkey.url, decode_responses=True)
+
+
 def patch_database_with_pydantic_object(
     database_object: Base, pydantic_object: pydantic.BaseModel
 ):

backend/capellacollab/projects/toolmodels/diagrams/models.py

@@ -18,6 +18,7 @@ class DiagramMetadata(core_pydantic.BaseModel):
 class DiagramCacheMetadata(core_pydantic.BaseModel):
     diagrams: list[DiagramMetadata]
     last_updated: datetime.datetime
+    job_id: str | None = None
 
     _validate_last_updated = pydantic.field_serializer("last_updated")(
         core_pydantic.datetime_serializer

backend/capellacollab/projects/toolmodels/diagrams/routes.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import json
 import logging
 import pathlib
 from urllib import parse
@@ -41,24 +42,30 @@ async def get_diagram_metadata(
     logger: logging.LoggerAdapter = fastapi.Depends(log.get_request_logger),
 ):
     try:
-        (
-            last_updated,
-            diagram_metadata_entries,
-        ) = await handler.get_file_from_repository_or_artifacts_as_json(
-            "diagram_cache/index.json",
-            "update_capella_diagram_cache",
-            "diagram-cache/" + handler.git_model.revision,
+        job_id, last_updated, diagram_metadata_entries = (
+            await handler.get_file_or_artifact(
+                trusted_file_path="diagram_cache/index.json",
+                logger=logger,
+                job_name="update_capella_diagram_cache",
+                file_revision=f"diagram-cache/{handler.revision}",
+            )
+        )
+    except requests.HTTPError:
+        logger.info(
+            "Failed fetching diagram metadata file or artifact for %s",
+            handler.path,
+            exc_info=True,
         )
-    except requests.exceptions.HTTPError:
-        logger.info("Failed fetching diagram metadata", exc_info=True)
         raise exceptions.DiagramCacheNotConfiguredProperlyError()
 
+    diagram_metadata_entries = json.loads(diagram_metadata_entries)
     return models.DiagramCacheMetadata(
         diagrams=[
             models.DiagramMetadata.model_validate(diagram_metadata)
             for diagram_metadata in diagram_metadata_entries
         ],
         last_updated=last_updated,
+        job_id=job_id,
     )
 
 
@@ -69,6 +76,7 @@ async def get_diagram_metadata(
 )
 async def get_diagram(
     diagram_uuid_or_filename: str,
+    job_id: str | None = None,
     handler: git_handler.GitHandler = fastapi.Depends(
         git_injectables.get_git_handler
     ),
@@ -79,16 +87,23 @@ async def get_diagram(
         raise exceptions.FileExtensionNotSupportedError(fileextension)
 
     diagram_uuid = pathlib.PurePosixPath(diagram_uuid_or_filename).stem
+    file_path = f"diagram_cache/{parse.quote(diagram_uuid, safe='')}.svg"
+
     try:
-        _, diagram = await handler.get_file_from_repository_or_artifacts(
-            f"diagram_cache/{parse.quote(diagram_uuid, safe='')}.svg",
-            "update_capella_diagram_cache",
-            "diagram-cache/" + handler.git_model.revision,
+        file_or_artifact = await handler.get_file_or_artifact(
+            trusted_file_path=file_path,
+            logger=logger,
+            job_name="update_capella_diagram_cache",
+            job_id=job_id,
+            file_revision=f"diagram-cache/{handler.revision}",
+        )
+        return responses.SVGResponse(content=file_or_artifact[2])
+    except requests.HTTPError:
+        logger.info(
+            "Failed fetching diagram file or artifact %s for %s.",
+            diagram_uuid,
+            handler.path,
+            f"diagram-cache/{handler.revision}",
+            exc_info=True,
         )
-    except requests.exceptions.HTTPError:
-        logger.info("Failed fetching diagram", exc_info=True)
         raise exceptions.DiagramCacheNotConfiguredProperlyError()
-
-    return responses.SVGResponse(
-        content=diagram,
-    )

backend/capellacollab/projects/toolmodels/modelbadge/routes.py

@@ -5,9 +5,7 @@
 
 import logging
 
-import aiohttp.web
 import fastapi
-import requests
 
 import capellacollab.projects.toolmodels.modelsources.git.injectables as git_injectables
 from capellacollab.core import logging as log
@@ -41,13 +39,17 @@ async def get_model_complexity_badge(
     logger: logging.LoggerAdapter = fastapi.Depends(log.get_request_logger),
 ):
     try:
-        return responses.SVGResponse(
-            content=(
-                await git_handler.get_file_from_repository_or_artifacts(
-                    "model-complexity-badge.svg", "generate-model-badge"
-                )
-            )[1],
+        file_or_artifact = await git_handler.get_file_or_artifact(
+            trusted_file_path="model-complexity-badge.svg",
+            job_name="generate-model-badge",
+            logger=logger,
+        )
+        return responses.SVGResponse(content=file_or_artifact[2])
+    except Exception:
+        logger.debug(
+            "Failed fetching model badge file or artifact for %s on revision %s.",
+            git_handler.path,
+            git_handler.revision,
+            exc_info=True,
         )
-    except (aiohttp.web.HTTPException, requests.exceptions.HTTPError):
-        logger.info("Failed fetching model complexity badge", exc_info=True)
         raise exceptions.ModelBadgeNotConfiguredProperlyError()

backend/capellacollab/projects/toolmodels/modelsources/git/crud.py

@@ -61,26 +61,36 @@ def make_git_model_primary(
 def update_git_model(
     db: orm.Session,
     git_model: models.DatabaseGitModel,
-    patch_model: models.PatchGitModel,
+    put_model: models.PutGitModel,
 ) -> models.DatabaseGitModel:
-    git_model.path = patch_model.path
-    git_model.entrypoint = patch_model.entrypoint
-    git_model.revision = patch_model.revision
-
-    if patch_model.password:
-        git_model.username = patch_model.username
-        git_model.password = patch_model.password
-    elif not patch_model.username:
+    git_model.path = put_model.path
+    git_model.entrypoint = put_model.entrypoint
+    git_model.revision = put_model.revision
+    git_model.repository_id = None
+
+    if put_model.password:
+        git_model.username = put_model.username
+        git_model.password = put_model.password
+    elif not put_model.username:
         git_model.username = ""
         git_model.password = ""
 
-    if patch_model.primary and not git_model.primary:
+    if put_model.primary and not git_model.primary:
         git_model = make_git_model_primary(db, git_model)
 
     db.commit()
     return git_model
 
 
+def update_git_model_repository_id(
+    db: orm.Session, git_model: models.DatabaseGitModel, repository_id: str
+) -> models.DatabaseGitModel:
+    git_model.repository_id = repository_id
+
+    db.commit()
+    return git_model
+
+
 def delete_git_model(db: orm.Session, git_model: models.DatabaseGitModel):
     db.delete(git_model)
     db.commit()

backend/capellacollab/projects/toolmodels/modelsources/git/exceptions.py

@@ -71,19 +71,6 @@ def __init__(self, filename: str):
         )
 
 
-class GitInstanceAPIEndpointNotFoundError(core_exceptions.BaseError):
-    def __init__(self):
-        super().__init__(
-            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            title="Git instance API endpoint not found",
-            reason=(
-                "The used Git instance has no API endpoint defined. "
-                "Please contact your administrator."
-            ),
-            err_code="GIT_INSTANCE_NO_API_ENDPOINT_DEFINED",
-        )
-
-
 class GitPipelineJobNotFoundError(core_exceptions.BaseError):
     def __init__(self, job_name: str, revision: str):
         super().__init__(
@@ -97,31 +84,18 @@ def __init__(self, job_name: str, revision: str):
         )
 
 
-class GitPipelineJobFailedError(core_exceptions.BaseError):
-    def __init__(self, job_name: str):
-        super().__init__(
-            status_code=status.HTTP_400_BAD_REQUEST,
-            title="Failed job found",
-            reason=f"The last job with the name '{job_name}' has failed.",
-            err_code="FAILED_JOB_FOUND",
-        )
-
-
-class GitPipelineJobUnknownStateError(core_exceptions.BaseError):
-    job_name: str
-    state: str
-
+class GitPipelineJobUnsuccessfulError(core_exceptions.BaseError):
     def __init__(self, job_name: str, state: str):
         self.job_name = job_name
         self.state = state
         super().__init__(
             status_code=status.HTTP_400_BAD_REQUEST,
-            title="Unknown job state",
+            title="Unsuccessful job",
             reason=(
-                f"Job '{job_name}' has an unhandled or unknown state: '{state}'. "
+                f"Job '{job_name}' has an unsuccessful state: {self.state}."
                 "Please contact your administrator."
             ),
-            err_code="UNKNOWN_STATE_ERROR",
+            err_code="UNSUCCESSFUL_JOB_STATE_ERROR",
         )