From 0a610360a34fe7a51d7255fed4c9565d286f05d8 Mon Sep 17 00:00:00 2001 From: dominik003 Date: Mon, 5 Aug 2024 17:35:08 +0200 Subject: [PATCH] feat: Improve git handler and introduce caching --- backend/capellacollab/core/cache.py | 40 ++++++ .../projects/toolmodels/diagrams/routes.py | 4 +- .../toolmodels/modelsources/git/exceptions.py | 13 -- .../modelsources/git/github/handler.py | 96 +++++-------- .../modelsources/git/gitlab/handler.py | 100 +++++--------- .../modelsources/git/handler/exceptions.py | 13 ++ .../modelsources/git/handler/factory.py | 95 +++++++++++-- .../modelsources/git/handler/handler.py | 127 +++++++++++------- .../modelsources/git/injectables.py | 4 +- .../toolmodels/modelsources/git/validation.py | 4 +- backend/tests/projects/toolmodels/conftest.py | 2 +- 11 files changed, 280 insertions(+), 218 deletions(-) create mode 100644 backend/capellacollab/core/cache.py diff --git a/backend/capellacollab/core/cache.py b/backend/capellacollab/core/cache.py new file mode 100644 index 0000000000..0822f51190 --- /dev/null +++ b/backend/capellacollab/core/cache.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + + +import abc + + +class Cache(abc.ABC): + @abc.abstractmethod + def get(self, key: str) -> bytes | None: + pass + + @abc.abstractmethod + def set(self, key: str, value: bytes) -> None: + pass + + @abc.abstractmethod + def delete(self, key: str) -> None: + pass + + @abc.abstractmethod + def clear(self) -> None: + pass + + +class InMemoryCache(Cache): + def __init__(self) -> None: + self.cache: dict[str, bytes] = {} + + def get(self, key: str) -> bytes | None: + return self.cache.get(key, None) + + def set(self, key: str, value: bytes) -> None: + self.cache[key] = value + + def delete(self, key: str) -> None: + self.cache.pop(key) + + def clear(self) -> None: + self.cache.clear() diff --git a/backend/capellacollab/projects/toolmodels/diagrams/routes.py b/backend/capellacollab/projects/toolmodels/diagrams/routes.py index 46beaf9a16..b24a5519ce 100644 --- a/backend/capellacollab/projects/toolmodels/diagrams/routes.py +++ b/backend/capellacollab/projects/toolmodels/diagrams/routes.py @@ -47,7 +47,7 @@ async def get_diagram_metadata( ) = await handler.get_file_from_repository_or_artifacts_as_json( "diagram_cache/index.json", "update_capella_diagram_cache", - "diagram-cache/" + handler.git_model.revision, + "diagram-cache/" + handler.revision, ) except requests.exceptions.HTTPError: logger.info("Failed fetching diagram metadata", exc_info=True) @@ -83,7 +83,7 @@ async def get_diagram( _, diagram = await handler.get_file_from_repository_or_artifacts( f"diagram_cache/{parse.quote(diagram_uuid, safe='')}.svg", "update_capella_diagram_cache", - "diagram-cache/" + handler.git_model.revision, + "diagram-cache/" + handler.revision, ) except requests.exceptions.HTTPError: logger.info("Failed fetching diagram", exc_info=True) diff --git a/backend/capellacollab/projects/toolmodels/modelsources/git/exceptions.py b/backend/capellacollab/projects/toolmodels/modelsources/git/exceptions.py index 8969cfd6fb..2a7ed6d94c 100644 --- a/backend/capellacollab/projects/toolmodels/modelsources/git/exceptions.py +++ b/backend/capellacollab/projects/toolmodels/modelsources/git/exceptions.py @@ -71,19 +71,6 @@ def __init__(self, filename: str): ) -class GitInstanceAPIEndpointNotFoundError(core_exceptions.BaseError): - def __init__(self): - super().__init__( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - title="Git instance API endpoint not found", - reason=( - "The used Git instance has no API endpoint defined. " - "Please contact your administrator." - ), - err_code="GIT_INSTANCE_NO_API_ENDPOINT_DEFINED", - ) - - class GitPipelineJobNotFoundError(core_exceptions.BaseError): def __init__(self, job_name: str, revision: str): super().__init__( diff --git a/backend/capellacollab/projects/toolmodels/modelsources/git/github/handler.py b/backend/capellacollab/projects/toolmodels/modelsources/git/github/handler.py index a02e4af67a..a3cd0e2573 100644 --- a/backend/capellacollab/projects/toolmodels/modelsources/git/github/handler.py +++ b/backend/capellacollab/projects/toolmodels/modelsources/git/github/handler.py @@ -4,7 +4,6 @@ import base64 import datetime import io -import json import typing as t import zipfile from urllib import parse @@ -18,63 +17,37 @@ class GithubHandler(handler.GitHandler): - async def get_project_id_by_git_url(self) -> str: + @classmethod + async def get_project_id_by_git_url(cls, path: str, *_) -> str: # Project ID has the format '{owner}/{repo_name}' - return parse.urlparse(self.git_model.path).path[1:] - - async def get_last_job_run_id_for_git_model( - self, job_name: str, project_id: str | None = None - ) -> tuple[str, str]: - if not project_id: - project_id = await self.get_project_id_by_git_url() - jobs = self.get_last_pipeline_runs(project_id) + return parse.urlparse(path).path[1:] + + async def get_last_job_run_id(self, job_name: str) -> tuple[str, str]: + jobs = self.get_last_pipeline_runs() latest_job = self.__get_latest_successful_job(jobs, job_name) return (latest_job["id"], latest_job["created_at"]) - def get_artifact_from_job_as_json( - self, - project_id: str, - job_id: str, - trusted_path_to_artifact: str, - ) -> dict: - return json.loads( - self.get_artifact_from_job( - project_id, - job_id, - trusted_path_to_artifact, - ) - ) - def get_artifact_from_job_as_content( - self, - project_id: str, - job_id: str, - trusted_path_to_artifact: str, + self, job_id: str, trusted_path_to_artifact: str ) -> bytes: return self.get_artifact_from_job( - project_id, - job_id, - trusted_path_to_artifact, + job_id, trusted_path_to_artifact ).encode() def __get_file_from_repository( self, - project_id: str, trusted_file_path: str, revision: str, headers: dict[str, str] | None = None, ) -> requests.Response: return requests.get( - f"{self.git_instance.api_url}/repos/{project_id}/contents/{parse.quote(trusted_file_path)}?ref={parse.quote(revision, safe='')}", + f"{self.api_url}/repos/{self.project_id}/contents/{parse.quote(trusted_file_path)}?ref={parse.quote(revision, safe='')}", timeout=config.requests.timeout, headers=headers, ) - async def get_file_from_repository( - self, - project_id: str, - trusted_file_path: str, - revision: str | None = None, + def get_file_from_repository( + self, trusted_file_path: str, revision: str | None = None ) -> bytes: """ If a repository is public but the permissions are not set correctly, you might be able to download the file without authentication @@ -83,15 +56,14 @@ async def get_file_from_repository( For that purpose first we try to reach it without authentication and only if that fails try to get the file authenticated. """ response = self.__get_file_from_repository( - project_id, trusted_file_path, revision or self.git_model.revision + trusted_file_path, revision or self.revision ) - if not response.ok and self.git_model.password: + if not response.ok and self.password: response = self.__get_file_from_repository( - project_id, trusted_file_path, - revision=revision or self.git_model.revision, - headers=self.__get_headers(self.git_model.password), + revision=revision or self.revision, + headers=self.__get_headers(self.password), ) if response.status_code == 404: @@ -102,15 +74,12 @@ async def get_file_from_repository( return base64.b64decode(response.json()["content"]) - def get_last_pipeline_runs( - self, - project_id: str, - ) -> t.Any: + def get_last_pipeline_runs(self) -> t.Any: headers = None - if self.git_model.password: - headers = self.__get_headers(self.git_model.password) + if self.password: + headers = self.__get_headers(self.password) response = requests.get( - f"{self.git_instance.api_url}/repos/{project_id}/actions/runs?branch={parse.quote(self.git_model.revision, safe='')}&per_page=20", + f"{self.api_url}/repos/{self.project_id}/actions/runs?branch={parse.quote(self.revision, safe='')}&per_page=20", headers=headers, timeout=config.requests.timeout, ) @@ -118,16 +87,13 @@ def get_last_pipeline_runs( return response.json()["workflow_runs"] def get_artifact_from_job( - self, - project_id: str, - job_id: str, - trusted_path_to_artifact: str, + self, job_id: str, trusted_path_to_artifact: str ) -> str: - artifact = self.__get_latest_artifact_metadata(project_id, job_id) + artifact = self.__get_latest_artifact_metadata(job_id) artifact_id = artifact["id"] artifact_response = requests.get( - f"{self.git_instance.api_url}/repos/{project_id}/actions/artifacts/{artifact_id}/zip", - headers=self.__get_headers(self.git_model.password), + f"{self.api_url}/repos/{self.project_id}/actions/artifacts/{artifact_id}/zip", + headers=self.__get_headers(self.password), timeout=config.requests.timeout, ) artifact_response.raise_for_status() @@ -137,14 +103,12 @@ def get_artifact_from_job( ) def get_last_updated_for_file_path( - self, project_id: str, file_path: str, revision: str | None + self, file_path: str, revision: str | None ) -> datetime.datetime | None: response = requests.get( - f"{self.git_instance.api_url}/repos/{project_id}/commits?path={file_path}&sha={revision or self.git_model.revision}", + f"{self.api_url}/repos/{self.project_id}/commits?path={file_path}&sha={revision or self.revision}", headers=( - self.__get_headers(self.git_model.password) - if self.git_model.password - else None + self.__get_headers(self.password) if self.password else None ), timeout=config.requests.timeout, ) @@ -169,7 +133,7 @@ def __get_latest_successful_job(self, jobs: list, job_name: str) -> dict: matched_jobs = [job for job in jobs if job["name"] == job_name] if not matched_jobs: raise git_exceptions.GitPipelineJobNotFoundError( - job_name=job_name, revision=self.git_model.revision + job_name=job_name, revision=self.revision ) matched_jobs.sort(key=lambda job: job["created_at"], reverse=True) if matched_jobs[0]["conclusion"] == "success": @@ -184,10 +148,10 @@ def __get_latest_successful_job(self, jobs: list, job_name: str) -> dict: job_name, matched_jobs[0]["conclusion"] ) - def __get_latest_artifact_metadata(self, project_id: str, job_id: str): + def __get_latest_artifact_metadata(self, job_id: str): response = requests.get( - f"{self.git_instance.api_url}/repos/{project_id}/actions/runs/{job_id}/artifacts", - headers=self.__get_headers(self.git_model.password), + f"{self.api_url}/repos/{self.project_id}/actions/runs/{job_id}/artifacts", + headers=self.__get_headers(self.password), timeout=config.requests.timeout, ) response.raise_for_status() diff --git a/backend/capellacollab/projects/toolmodels/modelsources/git/gitlab/handler.py b/backend/capellacollab/projects/toolmodels/modelsources/git/gitlab/handler.py index ae3cfd2308..70221eafab 100644 --- a/backend/capellacollab/projects/toolmodels/modelsources/git/gitlab/handler.py +++ b/backend/capellacollab/projects/toolmodels/modelsources/git/gitlab/handler.py @@ -16,17 +16,18 @@ class GitlabHandler(handler.GitHandler): - async def get_project_id_by_git_url(self) -> str: + @classmethod + async def get_project_id_by_git_url( + cls, path: str, password: str, api_url: str + ) -> str: project_name_encoded = parse.quote( - parse.urlparse(self.git_model.path) - .path.lstrip("/") - .removesuffix(".git"), + parse.urlparse(path).path.lstrip("/").removesuffix(".git"), safe="", ) async with aiohttp.ClientSession() as session: async with session.get( - f"{self.git_instance.api_url}/projects/{project_name_encoded}", - headers={"PRIVATE-TOKEN": self.git_model.password}, + f"{api_url}/projects/{project_name_encoded}", + headers={"PRIVATE-TOKEN": password}, timeout=config.requests.timeout, ) as response: if response.status == 403: @@ -39,29 +40,23 @@ async def get_project_id_by_git_url(self) -> str: response.raise_for_status() return (await response.json())["id"] - async def get_last_job_run_id_for_git_model( - self, job_name: str, project_id: str | None = None - ) -> tuple[str, str]: - if not project_id: - project_id = await self.get_project_id_by_git_url() - for pipeline_id in await self.__get_last_pipeline_run_ids(project_id): + async def get_last_job_run_id(self, job_name: str) -> tuple[str, str]: + for pipeline_id in await self.__get_last_pipeline_run_ids(): if job := await self.__get_job_id_for_job_name( - project_id, - pipeline_id, - job_name, + pipeline_id, job_name ): return job raise git_exceptions.GitPipelineJobNotFoundError( - job_name=job_name, revision=self.git_model.revision + job_name=job_name, revision=self.revision ) def get_last_updated_for_file_path( - self, project_id: str, file_path: str, revision: str | None + self, file_path: str, revision: str | None ) -> datetime.datetime | None: response = requests.get( - f"{self.git_instance.api_url}/projects/{project_id}/repository/commits?ref_name={revision or self.git_model.revision}&path={file_path}", - headers={"PRIVATE-TOKEN": self.git_model.password}, + f"{self.api_url}/projects/{self.project_id}/repository/commits?ref_name={revision or self.revision}&path={file_path}", + headers={"PRIVATE-TOKEN": self.password}, timeout=config.requests.timeout, ) response.raise_for_status() @@ -71,14 +66,11 @@ def get_last_updated_for_file_path( ) return response.json()[0]["authored_date"] - async def __get_last_pipeline_run_ids( - self, - project_id: str, - ) -> list[str]: + async def __get_last_pipeline_run_ids(self) -> list[str]: async with aiohttp.ClientSession() as session: async with session.get( - f"{self.git_instance.api_url}/projects/{project_id}/pipelines?ref={parse.quote(self.git_model.revision, safe='')}&per_page=20", - headers={"PRIVATE-TOKEN": self.git_model.password}, + f"{self.api_url}/projects/{self.project_id}/pipelines?ref={parse.quote(self.revision, safe='')}&per_page=20", + headers={"PRIVATE-TOKEN": self.password}, timeout=config.requests.timeout, ) as response: response.raise_for_status() @@ -86,16 +78,13 @@ async def __get_last_pipeline_run_ids( return [pipeline["id"] for pipeline in await response.json()] async def __get_job_id_for_job_name( - self, - project_id: str, - pipeline_id: str, - job_name: str, + self, pipeline_id: str, job_name: str ) -> tuple[str, str] | None: """Search for a job by name in a pipeline""" async with aiohttp.ClientSession() as session: async with session.get( - f"{self.git_instance.api_url}/projects/{project_id}/pipelines/{pipeline_id}/jobs", - headers={"PRIVATE-TOKEN": self.git_model.password}, + f"{self.api_url}/projects/{self.project_id}/pipelines/{pipeline_id}/jobs", + headers={"PRIVATE-TOKEN": self.password}, timeout=config.requests.timeout, ) as response: response.raise_for_status() @@ -111,54 +100,25 @@ async def __get_job_id_for_job_name( return None - def get_artifact_from_job_as_json( - self, - project_id: str, - job_id: str, - trusted_path_to_artifact: str, - ) -> dict: - return self.get_artifact_from_job( - project_id, - job_id, - trusted_path_to_artifact, - ).json() - def get_artifact_from_job_as_content( - self, - project_id: str, - job_id: str, - trusted_path_to_artifact: str, + self, job_id: str, trusted_path_to_artifact: str ) -> bytes: - return self.get_artifact_from_job( - project_id, - job_id, - trusted_path_to_artifact, - ).content - - def get_artifact_from_job( - self, - project_id: str, - job_id: str, - trusted_path_to_artifact: str, - ) -> requests.Response: response = requests.get( - f"{self.git_instance.api_url}/projects/{project_id}/jobs/{job_id}/artifacts/{trusted_path_to_artifact}", - headers={"PRIVATE-TOKEN": self.git_model.password}, + f"{self.api_url}/projects/{self.project_id}/jobs/{job_id}/artifacts/{trusted_path_to_artifact}", + headers={"PRIVATE-TOKEN": self.password}, timeout=config.requests.timeout, ) response.raise_for_status() - return response - async def get_file_from_repository( - self, - project_id: str, - trusted_file_path: str, - revision: str | None = None, + return response.content + + def get_file_from_repository( + self, trusted_file_path: str, revision: str | None = None ) -> bytes: - branch = revision if revision else self.git_model.revision + branch = revision if revision else self.revision response = requests.get( - f"{self.git_instance.api_url}/projects/{project_id}/repository/files/{parse.quote(trusted_file_path, safe='')}?ref={parse.quote(branch, safe='')}", - headers={"PRIVATE-TOKEN": self.git_model.password}, + f"{self.api_url}/projects/{self.project_id}/repository/files/{parse.quote(trusted_file_path, safe='')}?ref={parse.quote(branch, safe='')}", + headers={"PRIVATE-TOKEN": self.password}, timeout=config.requests.timeout, ) diff --git a/backend/capellacollab/projects/toolmodels/modelsources/git/handler/exceptions.py b/backend/capellacollab/projects/toolmodels/modelsources/git/handler/exceptions.py index 6f3a35b140..fcabd7fc64 100644 --- a/backend/capellacollab/projects/toolmodels/modelsources/git/handler/exceptions.py +++ b/backend/capellacollab/projects/toolmodels/modelsources/git/handler/exceptions.py @@ -27,3 +27,16 @@ def __init__(self): ), err_code="NO_MATCHING_GIT_INSTANCE", ) + + +class GitInstanceAPIEndpointNotFoundError(core_exceptions.BaseError): + def __init__(self): + super().__init__( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + title="Git instance API endpoint not found", + reason=( + "The used Git instance has no API endpoint defined. " + "Please contact your administrator." + ), + err_code="GIT_INSTANCE_NO_API_ENDPOINT_DEFINED", + ) diff --git a/backend/capellacollab/projects/toolmodels/modelsources/git/handler/factory.py b/backend/capellacollab/projects/toolmodels/modelsources/git/handler/factory.py index 7a161506ca..720707afce 100644 --- a/backend/capellacollab/projects/toolmodels/modelsources/git/handler/factory.py +++ b/backend/capellacollab/projects/toolmodels/modelsources/git/handler/factory.py @@ -14,22 +14,48 @@ class GitHandlerFactory: + project_id_cache: dict[str, str] = {} + @staticmethod - def create_git_handler( + async def create_git_handler( db: orm.Session, git_model: git_models.DatabaseGitModel ) -> handler.GitHandler: + """ + Create a git handler for the given git model. + + Args: + db (orm.Session): Database session. + git_model (git_models.DatabaseGitModel): The git model instance. + + Returns: + handler.GitHandler: An instance of GitHandler. + + Raises: + GitInstanceAPIEndpointNotFoundError: If the git instance API endpoint is not found. + GitInstanceUnsupportedError: If the git instance type is unsupported. + """ git_instance = GitHandlerFactory.get_git_instance_for_git_model( db, git_model ) - match git_instance.type: - case settings_git_models.GitType.GITLAB: - return gitlab_handler.GitlabHandler(git_model, git_instance) - case settings_git_models.GitType.GITHUB: - return github_handler.GithubHandler(git_model, git_instance) - case _: - raise exceptions.GitInstanceUnsupportedError( - instance_name=str(git_instance.type) - ) + + if not git_instance.api_url: + raise exceptions.GitInstanceAPIEndpointNotFoundError() + + project_id = GitHandlerFactory.project_id_cache.get( + f"{git_model.path}-{str(git_instance.type)}", None + ) + + if project_id is None: + project_id = await GitHandlerFactory._get_project_id( + git_model, git_instance.type, git_instance.api_url + ) + GitHandlerFactory.project_id_cache[ + f"{git_model.path}-{str(git_instance.type)}" + ] = project_id + + return GitHandlerFactory._create_specific_git_handler( + git_model, git_instance.type, git_instance.api_url, project_id + ) @staticmethod def get_git_instance_for_git_model( @@ -48,3 +74,52 @@ def get_git_instance_for_git_model( if git_model.path.startswith(instance.url): return instance raise exceptions.NoMatchingGitInstanceError + + @staticmethod + async def _get_project_id( + git_model: git_models.DatabaseGitModel, + git_instance_type: settings_git_models.GitType, + api_url: str, + ) -> str: + match git_instance_type: + case settings_git_models.GitType.GITLAB: + return await gitlab_handler.GitlabHandler.get_project_id_by_git_url( + git_model.path, git_model.password, api_url + ) + case settings_git_models.GitType.GITHUB: + return await github_handler.GithubHandler.get_project_id_by_git_url( + git_model.path, git_model.password, api_url + ) + case _: + raise exceptions.GitInstanceUnsupportedError( + instance_name=str(git_instance_type) + ) + + @staticmethod + def _create_specific_git_handler( + git_model: git_models.DatabaseGitModel, + git_instance_type: settings_git_models.GitType, + api_url: str, + project_id: str, + ) -> handler.GitHandler: + match git_instance_type: + case settings_git_models.GitType.GITLAB: + return gitlab_handler.GitlabHandler( + git_model.path, + git_model.revision, + git_model.password, + api_url, + project_id, + ) + case settings_git_models.GitType.GITHUB: + return github_handler.GithubHandler( + git_model.path, + git_model.revision, + git_model.password, + api_url, + project_id, + ) + case _: + raise exceptions.GitInstanceUnsupportedError( + instance_name=str(git_instance_type) + ) diff --git a/backend/capellacollab/projects/toolmodels/modelsources/git/handler/handler.py b/backend/capellacollab/projects/toolmodels/modelsources/git/handler/handler.py index e09a62b5a5..e8eb0cc894 100644 --- a/backend/capellacollab/projects/toolmodels/modelsources/git/handler/handler.py +++ b/backend/capellacollab/projects/toolmodels/modelsources/git/handler/handler.py @@ -10,8 +10,7 @@ import requests -import capellacollab.projects.toolmodels.modelsources.git.models as git_models -import capellacollab.settings.modelsources.git.models as settings_git_models +from capellacollab.core import cache from .. import exceptions @@ -20,59 +19,48 @@ class GitHandler: + cache: cache.Cache = cache.InMemoryCache() + def __init__( self, - git_model: git_models.DatabaseGitModel, - git_instance: settings_git_models.DatabaseGitInstance, + path: str, + revision: str, + password: str, + api_url: str, + project_id: str, ) -> None: - self.git_model = git_model - self.git_instance = git_instance - self.check_git_instance_has_api_url() - - def check_git_instance_has_api_url(self): - if not self.git_instance.api_url: - raise exceptions.GitInstanceAPIEndpointNotFoundError() - - @abc.abstractmethod - async def get_project_id_by_git_url(self) -> str: - pass + self.path = path + self.revision = revision + self.password = password + self.api_url = api_url + self.project_id = project_id + @classmethod @abc.abstractmethod - async def get_last_job_run_id_for_git_model( - self, job_name: str, project_id: str | None = None - ) -> tuple[str, str]: + async def get_project_id_by_git_url( + cls, path: str, password: str, api_url: str + ) -> str: pass @abc.abstractmethod - def get_artifact_from_job_as_json( - self, - project_id: str, - job_id: str, - trusted_path_to_artifact: str, - ) -> dict: + async def get_last_job_run_id(self, job_name: str) -> tuple[str, str]: pass @abc.abstractmethod def get_artifact_from_job_as_content( - self, - project_id: str, - job_id: str, - trusted_path_to_artifact: str, + self, job_id: str, trusted_path_to_artifact: str ) -> bytes: pass @abc.abstractmethod - async def get_file_from_repository( - self, - project_id: str, - trusted_file_path: str, - revision: str | None = None, + def get_file_from_repository( + self, trusted_file_path: str, revision: str | None = None ) -> bytes: pass @abc.abstractmethod def get_last_updated_for_file_path( - self, project_id: str, file_path: str, revision: str | None + self, file_path: str, revision: str | None ) -> datetime.datetime | None: pass @@ -96,27 +84,62 @@ async def get_file_from_repository_or_artifacts( job_name: str, revision: str | None = None, ) -> tuple[t.Any, bytes]: - project_id = await self.get_project_id_by_git_url() try: - return ( - self.get_last_updated_for_file_path( - project_id, - trusted_file_path, - revision=revision, - ), - await self.get_file_from_repository( - project_id, trusted_file_path, revision - ), + f_last_updated = self.get_last_updated_for_file_path( + trusted_file_path, revision + ) + + f_content = self._get_content_from_cache( + trusted_file_path, "file", revision ) + + if not f_content: + f_content = self.get_file_from_repository( + trusted_file_path, revision + ) + self._store_content_in_cache( + trusted_file_path, f_content, "file", revision + ) + return (f_last_updated, f_content) except (requests.HTTPError, exceptions.GitRepositoryFileNotFoundError): pass - job_id, last_updated = await self.get_last_job_run_id_for_git_model( - job_name, project_id - ) - return ( - last_updated, - self.get_artifact_from_job_as_content( - project_id, job_id, trusted_file_path - ), + job_id, a_last_updated = await self.get_last_job_run_id(job_name) + + a_content = self._get_content_from_cache( + f"{job_id}-{trusted_file_path}", "artifact", revision ) + if not a_content: + a_content = self.get_artifact_from_job_as_content( + job_id, trusted_file_path + ) + self._store_content_in_cache( + f"{job_id}-{trusted_file_path}", + a_content, + "artifact", + revision, + ) + + return (a_last_updated, a_content) + + def _get_content_from_cache( + self, content_id: str, prefix: str, revision: str | None = None + ) -> bytes | None: + revision = revision if revision else self.revision + + key = f"{prefix}-{self.project_id}-{content_id}-{revision}" + + return GitHandler.cache.get(key) + + def _store_content_in_cache( + self, + content_id: str, + content: bytes, + prefix: str, + revision: str | None = None, + ) -> None: + revision = revision if revision else self.revision + + key = f"{prefix}-{self.project_id}-{content_id}-{revision}" + + GitHandler.cache.set(key, content) diff --git a/backend/capellacollab/projects/toolmodels/modelsources/git/injectables.py b/backend/capellacollab/projects/toolmodels/modelsources/git/injectables.py index 97b3b233c9..685e0acf3b 100644 --- a/backend/capellacollab/projects/toolmodels/modelsources/git/injectables.py +++ b/backend/capellacollab/projects/toolmodels/modelsources/git/injectables.py @@ -49,10 +49,10 @@ def get_existing_primary_git_model( raise exceptions.NoGitRepositoryAssignedToModelError(tool_model.slug) -def get_git_handler( +async def get_git_handler( git_model: git_models.DatabaseGitModel = fastapi.Depends( get_existing_primary_git_model ), db: orm.Session = fastapi.Depends(database.get_db), ) -> handler.GitHandler: - return factory.GitHandlerFactory.create_git_handler(db, git_model) + return await factory.GitHandlerFactory.create_git_handler(db, git_model) diff --git a/backend/capellacollab/projects/toolmodels/modelsources/git/validation.py b/backend/capellacollab/projects/toolmodels/modelsources/git/validation.py index 9e599cfece..9f8b4aff19 100644 --- a/backend/capellacollab/projects/toolmodels/modelsources/git/validation.py +++ b/backend/capellacollab/projects/toolmodels/modelsources/git/validation.py @@ -54,10 +54,10 @@ async def check_pipeline_health( return models.ModelArtifactStatus.UNCONFIGURED try: - git_handler = factory.GitHandlerFactory.create_git_handler( + git_handler = await factory.GitHandlerFactory.create_git_handler( db, primary_git_model ) - await git_handler.get_last_job_run_id_for_git_model(job_name) + await git_handler.get_last_job_run_id(job_name) except exceptions.GitPipelineJobNotFoundError: return models.ModelArtifactStatus.UNCONFIGURED except handler_exceptions.GitInstanceUnsupportedError: diff --git a/backend/tests/projects/toolmodels/conftest.py b/backend/tests/projects/toolmodels/conftest.py index 9fec519b1a..baea5c1c51 100644 --- a/backend/tests/projects/toolmodels/conftest.py +++ b/backend/tests/projects/toolmodels/conftest.py @@ -43,7 +43,7 @@ def fixture_git_instance_api_url( def fixture_git_instance( db: orm.Session, git_type: git_models.GitType, git_instance_api_url: str ) -> git_models.DatabaseGitInstance: - git_instance = git_models.DatabaseGitInstance( + git_instance = git_models.PostGitInstance( name="test", url="https://example.com/test/project", api_url=git_instance_api_url,