diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 776536e47..9abd03d09 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,6 +44,18 @@ repos: - id: isort entry: bash -c "cd backend && isort ." types: [python] + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.5.1 + hooks: + - id: mypy + types_or: [python, spec] + files: "^backend/capellacollab" + exclude: "^backend/capellacollab/alembic/" + args: [--config-file=./backend/pyproject.toml] + additional_dependencies: + - fastapi + - pydantic + - sqlalchemy - repo: local hooks: - id: pylint diff --git a/backend/capellacollab/__main__.py b/backend/capellacollab/__main__.py index eaab432d6..4f310aafa 100644 --- a/backend/capellacollab/__main__.py +++ b/backend/capellacollab/__main__.py @@ -21,6 +21,9 @@ from capellacollab.core import logging as core_logging from capellacollab.core.database import engine, migration from capellacollab.core.logging import exceptions as logging_exceptions +from capellacollab.projects.toolmodels import ( + exceptions as toolmodels_exceptions, +) from capellacollab.projects.toolmodels.backups import ( exceptions as backups_exceptions, ) @@ -133,6 +136,7 @@ async def healthcheck(): def register_exceptions(): tools_exceptions.register_exceptions(app) + toolmodels_exceptions.register_exceptions(app) git_exceptions.register_exceptions(app) gitlab_exceptions.register_exceptions(app) git_handler_exceptions.register_exceptions(app) diff --git a/backend/capellacollab/alembic/versions/1a4208c18909_make_tool_name_required.py b/backend/capellacollab/alembic/versions/1a4208c18909_make_tool_name_required.py new file mode 100644 index 000000000..e76ea666e --- /dev/null +++ b/backend/capellacollab/alembic/versions/1a4208c18909_make_tool_name_required.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Make tool name required + +Revision ID: 1a4208c18909 +Revises: d8cf851562cd +Create Date: 2023-09-19 11:25:16.343948 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "1a4208c18909" +down_revision = "d8cf851562cd" +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column( + "tools", + "name", + existing_type=sa.String(), + nullable=False, + ) diff --git a/backend/capellacollab/core/authentication/jwt_bearer.py b/backend/capellacollab/core/authentication/jwt_bearer.py index 40dd2b523..b9b205d38 100644 --- a/backend/capellacollab/core/authentication/jwt_bearer.py +++ b/backend/capellacollab/core/authentication/jwt_bearer.py @@ -25,7 +25,7 @@ class JWTBearer(security.HTTPBearer): def __init__(self, auto_error: bool = True): super().__init__(auto_error=auto_error) - async def __call__( + async def __call__( # type: ignore self, request: fastapi.Request ) -> dict[str, t.Any] | None: credentials: security.HTTPAuthorizationCredentials | None = ( diff --git a/backend/capellacollab/core/authentication/provider/azure/__main__.py b/backend/capellacollab/core/authentication/provider/azure/__main__.py index dc70ee2db..6fc912111 100644 --- a/backend/capellacollab/core/authentication/provider/azure/__main__.py +++ b/backend/capellacollab/core/authentication/provider/azure/__main__.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors # SPDX-License-Identifier: Apache-2.0 +import typing as t from capellacollab.config import config @@ -13,8 +14,8 @@ cfg = config["authentication"]["azure"] -def get_jwk_cfg(token: str) -> dict[str, any]: +def get_jwk_cfg(token: str) -> dict[str, t.Any]: return { "audience": cfg["client"]["id"], - "key": KeyStore.key_for_token(token).dict(), + "key": KeyStore.key_for_token(token).model_dump(), } diff --git a/backend/capellacollab/core/authentication/provider/azure/keystore.py b/backend/capellacollab/core/authentication/provider/azure/keystore.py index 65c400e17..a8945b565 100644 --- a/backend/capellacollab/core/authentication/provider/azure/keystore.py +++ b/backend/capellacollab/core/authentication/provider/azure/keystore.py @@ -12,11 +12,8 @@ from jose import jwt from capellacollab.config import config -from capellacollab.core.authentication.provider.models import ( - InvalidTokenError, - JSONWebKeySet, - KeyIDNotFoundError, -) + +from .. import models as provider_models log = logging.getLogger(__name__) cfg = config["authentication"]["azure"] @@ -38,7 +35,7 @@ def __init__( self.jwks_uri = jwks_uri self.algorithms = algorithms - self.public_keys: dict[t.Any, t.Any] = {} + self.public_keys: dict[str, provider_models.JSONWebKey] = {} self.key_refresh_interval = key_refresh_interval self.public_keys_last_refreshed: float = 0 self.refresh_keys() @@ -56,7 +53,7 @@ def refresh_keys(self) -> None: except Exception: log.error("Could not retrieve JWKS data from %s", self.jwks_uri) return - jwks = JSONWebKeySet.parse_raw(resp.text) + jwks = provider_models.JSONWebKeySet.parse_raw(resp.text) self.public_keys_last_refreshed = time.time() self.public_keys.clear() for key in jwks.keys: @@ -64,7 +61,7 @@ def refresh_keys(self) -> None: def key_for_token( self, token: str, *, in_retry: int = 0 - ) -> dict[str, t.Any]: + ) -> provider_models.JSONWebKey: # Before we do anything, the validation keys may need to be refreshed. # If so, refresh them. if self.keys_need_refresh(): @@ -75,7 +72,9 @@ def key_for_token( try: unverified_claims = jwt.get_unverified_header(token) except Exception: - raise InvalidTokenError("Unable to parse key ID from token") + raise provider_models.InvalidTokenError( + "Unable to parse key ID from token" + ) # See if we have the key identified by this key ID. try: @@ -85,7 +84,7 @@ def key_for_token( # haven't refreshed keys yet), then try to refresh the keys and try # again. if in_retry: - raise KeyIDNotFoundError() + raise provider_models.KeyIDNotFoundError() self.refresh_keys() return self.key_for_token(token, in_retry=1) diff --git a/backend/capellacollab/core/authentication/provider/azure/routes.py b/backend/capellacollab/core/authentication/provider/azure/routes.py index aa055e458..6cf6b9abb 100644 --- a/backend/capellacollab/core/authentication/provider/azure/routes.py +++ b/backend/capellacollab/core/authentication/provider/azure/routes.py @@ -62,7 +62,10 @@ async def api_get_token( ) access_token = token["id_token"] - username = get_username(JWTBearer().validate_token(access_token)) + validated_token = JWTBearer().validate_token(access_token) + assert validated_token + + username = get_username(validated_token) if user := users_crud.get_user_by_name(db, username): users_crud.update_last_login(db, user) diff --git a/backend/capellacollab/core/authentication/provider/oauth/__main__.py b/backend/capellacollab/core/authentication/provider/oauth/__main__.py index 3cbeefed8..803679fa3 100644 --- a/backend/capellacollab/core/authentication/provider/oauth/__main__.py +++ b/backend/capellacollab/core/authentication/provider/oauth/__main__.py @@ -15,5 +15,5 @@ def get_jwk_cfg(token: str) -> dict[str, t.Any]: return { "algorithms": ["RS256"], "audience": cfg["audience"] or cfg["client"]["id"], - "key": KeyStore.key_for_token(token).dict(), + "key": KeyStore.key_for_token(token).model_dump(), } diff --git a/backend/capellacollab/core/authentication/provider/oauth/keystore.py b/backend/capellacollab/core/authentication/provider/oauth/keystore.py index 9a01a680f..9480ced3c 100644 --- a/backend/capellacollab/core/authentication/provider/oauth/keystore.py +++ b/backend/capellacollab/core/authentication/provider/oauth/keystore.py @@ -14,6 +14,8 @@ from capellacollab.config import config from capellacollab.core.authentication.provider import models +from .. import models as provider_models + log = logging.getLogger(__name__) cfg = config["authentication"]["oauth"] @@ -35,7 +37,7 @@ def __init__( self.get_jwks_uri = get_jwks_uri self.jwks_uri = "" self.algorithms = algorithms - self.public_keys: dict[t.Any, t.Any] = {} + self.public_keys: dict[str, provider_models.JSONWebKey] = {} self.key_refresh_interval = key_refresh_interval self.public_keys_last_refreshed: float = 0 @@ -62,7 +64,7 @@ def refresh_keys(self) -> None: def key_for_token( self, token: str, *, in_retry: int = 0 - ) -> dict[str, t.Any]: + ) -> provider_models.JSONWebKey: # Before we do anything, the validation keys may need to be refreshed. # If so, refresh them. if self.keys_need_refresh(): diff --git a/backend/capellacollab/core/authentication/provider/oauth/routes.py b/backend/capellacollab/core/authentication/provider/oauth/routes.py index c368c6ec3..4f863b2fa 100644 --- a/backend/capellacollab/core/authentication/provider/oauth/routes.py +++ b/backend/capellacollab/core/authentication/provider/oauth/routes.py @@ -30,8 +30,12 @@ async def api_get_token( body: TokenRequest, db: orm.Session = fastapi.Depends(database.get_db) ): token = get_token(body.code) + access_token = token["id_token"] - username = get_username(JWTBearer().validate_token(token["access_token"])) + validated_token = JWTBearer().validate_token(access_token) + assert validated_token + + username = get_username(validated_token) if user := users_crud.get_user_by_name(db, username): users_crud.update_last_login(db, user) diff --git a/backend/capellacollab/core/database/__init__.py b/backend/capellacollab/core/database/__init__.py index 7520238da..12d6a7b91 100644 --- a/backend/capellacollab/core/database/__init__.py +++ b/backend/capellacollab/core/database/__init__.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors # SPDX-License-Identifier: Apache-2.0 +import typing as t import pydantic import sqlalchemy as sa @@ -23,7 +24,7 @@ class Base(orm.DeclarativeBase): from . import models # isort:skip # pylint: disable=unused-import -def get_db() -> orm.Session: +def get_db() -> t.Iterator[orm.Session]: with SessionLocal() as session: yield session diff --git a/backend/capellacollab/core/database/migration.py b/backend/capellacollab/core/database/migration.py index 4c6ed85f7..d3d78c1bd 100644 --- a/backend/capellacollab/core/database/migration.py +++ b/backend/capellacollab/core/database/migration.py @@ -162,6 +162,8 @@ def create_tools(db): def create_t4c_instance_and_repositories(db): tool = tools_crud.get_tool_by_name(db, "Capella") + assert tool + version = tools_crud.get_version_by_tool_id_version_name( db, tool.id, "5.2.0" ) @@ -189,13 +191,17 @@ def create_t4c_instance_and_repositories(db): def create_models(db: orm.Session): capella_tool = tools_crud.get_tool_by_name(db, "Capella") + assert capella_tool + + default_project = projects_crud.get_project_by_slug(db, "default") + assert default_project for version in ["5.0.0", "5.2.0", "6.0.0"]: capella_model = toolmodels_crud.create_model( db=db, - project=projects_crud.get_project_by_slug(db, "default"), + project=default_project, post_model=toolmodels_models.PostCapellaModel( - name=f"Meldody Model Test {version}", + name=f"Melody Model Test {version}", description="", tool_id=capella_tool.id, ), diff --git a/backend/capellacollab/projects/routes.py b/backend/capellacollab/projects/routes.py index ee78f1ee9..49a01f383 100644 --- a/backend/capellacollab/projects/routes.py +++ b/backend/capellacollab/projects/routes.py @@ -3,7 +3,6 @@ import logging -from collections import abc import fastapi import slugify @@ -39,7 +38,7 @@ ) -@router.get("", response_model=abc.Sequence[models.Project], tags=["Projects"]) +@router.get("", response_model=list[models.Project], tags=["Projects"]) def get_projects( user: users_models.DatabaseUser = fastapi.Depends( users_injectables.get_own_user @@ -49,18 +48,19 @@ def get_projects( log: logging.LoggerAdapter = fastapi.Depends( core_logging.get_request_logger ), -) -> abc.Sequence[models.DatabaseProject]: +) -> list[models.DatabaseProject]: if auth_injectables.RoleVerification( required_role=users_models.Role.ADMIN, verify=False )(token, db): log.debug("Fetching all projects") - return crud.get_projects(db) + return list(crud.get_projects(db)) projects = [ association.project for association in user.projects if not association.project.visibility == models.Visibility.INTERNAL - ] + crud.get_internal_projects(db) + ] + projects.extend(crud.get_internal_projects(db)) log.debug("Fetching the following projects: %s", projects) return projects @@ -89,7 +89,7 @@ def update_project( if crud.get_project_by_slug(db, new_slug) and project.slug != new_slug: raise fastapi.HTTPException( status_code=status.HTTP_409_CONFLICT, - details={ + detail={ "reason": "A project with a similar name already exists.", "technical": "Slug already used", }, @@ -137,7 +137,7 @@ def create_project( project = crud.create_project( db, post_project.name, - post_project.description, + post_project.description or "", post_project.visibility, ) diff --git a/backend/capellacollab/projects/toolmodels/backups/routes.py b/backend/capellacollab/projects/toolmodels/backups/routes.py index 660c3531d..4db172b4b 100644 --- a/backend/capellacollab/projects/toolmodels/backups/routes.py +++ b/backend/capellacollab/projects/toolmodels/backups/routes.py @@ -29,6 +29,7 @@ from capellacollab.tools import crud as tools_crud from capellacollab.users import models as users_models +from .. import exceptions as toolmodels_exceptions from . import core, crud, exceptions, injectables, models from .runs import routes as runs_routes @@ -100,6 +101,9 @@ def create_backup( ) if body.run_nightly: + if not capella_model.version_id: + raise toolmodels_exceptions.VersionIdNotSetError(capella_model.id) + reference = operators.get_operator().create_cronjob( image=tools_crud.get_backup_image_for_tool_version( db, capella_model.version_id diff --git a/backend/capellacollab/projects/toolmodels/backups/runs/interface.py b/backend/capellacollab/projects/toolmodels/backups/runs/interface.py index 3ff14f7ca..1f227250d 100644 --- a/backend/capellacollab/projects/toolmodels/backups/runs/interface.py +++ b/backend/capellacollab/projects/toolmodels/backups/runs/interface.py @@ -12,6 +12,9 @@ from capellacollab.config import config from capellacollab.core import database from capellacollab.core.logging import loki +from capellacollab.projects.toolmodels import ( + exceptions as toolmodels_exceptions, +) from capellacollab.projects.toolmodels.backups import core as backups_core from capellacollab.sessions import operators from capellacollab.tools import crud as tools_crud @@ -54,20 +57,21 @@ def _schedule_pending_jobs(): pending_run.pipeline.model.slug, ) try: + model = pending_run.pipeline.model + + if not model.version_id: + raise toolmodels_exceptions.VersionIdNotSetError(model.id) + job_name = operators.get_operator().create_job( image=tools_crud.get_backup_image_for_tool_version( - db, pending_run.pipeline.model.version_id + db, model.version_id ), command="backup", labels={ - "app.capellacollab/projectSlug": pending_run.pipeline.model.project.slug, - "app.capellacollab/projectID": str( - pending_run.pipeline.model.project.id - ), - "app.capellacollab/modelSlug": pending_run.pipeline.model.slug, - "app.capellacollab/modelID": str( - pending_run.pipeline.model.id - ), + "app.capellacollab/projectSlug": model.project.slug, + "app.capellacollab/projectID": str(model.project.id), + "app.capellacollab/modelSlug": model.slug, + "app.capellacollab/modelID": str(model.id), "app.capellacollab/pipelineID": str( pending_run.pipeline.id ), diff --git a/backend/capellacollab/projects/toolmodels/backups/validation.py b/backend/capellacollab/projects/toolmodels/backups/validation.py index 0f1034683..ae1d0c40f 100644 --- a/backend/capellacollab/projects/toolmodels/backups/validation.py +++ b/backend/capellacollab/projects/toolmodels/backups/validation.py @@ -11,7 +11,7 @@ def check_last_pipeline_run_status( db: orm.Session, model: toolmodel_models.DatabaseCapellaModel -) -> runs_models.PipelineRunStatus: +) -> runs_models.PipelineRunStatus | None: if pipeline := crud.get_first_pipeline_for_tool_model(db, model): # Only consider first pipeline for monitoring, usually there is only one pipeline. if pipeline_run := runs_crud.get_last_pipeline_run_of_pipeline( diff --git a/backend/capellacollab/projects/toolmodels/diagrams/models.py b/backend/capellacollab/projects/toolmodels/diagrams/models.py index c17d8943a..7ab752382 100644 --- a/backend/capellacollab/projects/toolmodels/diagrams/models.py +++ b/backend/capellacollab/projects/toolmodels/diagrams/models.py @@ -10,16 +10,12 @@ class DiagramMetadata(pydantic.BaseModel): - model_config = pydantic.ConfigDict(from_attributes=True) - name: str uuid: str success: bool class DiagramCacheMetadata(pydantic.BaseModel): - model_config = pydantic.ConfigDict(from_attributes=True) - diagrams: list[DiagramMetadata] last_updated: datetime.datetime diff --git a/backend/capellacollab/projects/toolmodels/diagrams/routes.py b/backend/capellacollab/projects/toolmodels/diagrams/routes.py index 835516306..148cd6aec 100644 --- a/backend/capellacollab/projects/toolmodels/diagrams/routes.py +++ b/backend/capellacollab/projects/toolmodels/diagrams/routes.py @@ -40,7 +40,7 @@ async def get_diagram_metadata( try: ( last_updated, - diagrams, + diagram_metadata_entries, ) = await handler.get_file_from_repository_or_artifacts_as_json( "diagram_cache/index.json", "update_capella_diagram_cache", @@ -57,8 +57,13 @@ async def get_diagram_metadata( ), }, ) + return models.DiagramCacheMetadata( - diagrams=diagrams, last_updated=last_updated + diagrams=[ + models.DiagramMetadata.model_validate(diagram_metadata) + for diagram_metadata in diagram_metadata_entries + ], + last_updated=last_updated, ) diff --git a/backend/capellacollab/projects/toolmodels/exceptions.py b/backend/capellacollab/projects/toolmodels/exceptions.py new file mode 100644 index 000000000..f07233e60 --- /dev/null +++ b/backend/capellacollab/projects/toolmodels/exceptions.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors +# SPDX-License-Identifier: Apache-2.0 + +import fastapi +from fastapi import exception_handlers, status + + +class VersionIdNotSetError(Exception): + def __init__(self, toolmodel_id: int): + self.toolmodel_id = toolmodel_id + + +async def version_id_not_set_exception_handler( + request: fastapi.Request, exc: VersionIdNotSetError +) -> fastapi.Response: + return await exception_handlers.http_exception_handler( + request, + fastapi.HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "reason": f"The toolmodel with id {exc.toolmodel_id} does not have a version set." + }, + ), + ) + + +def register_exceptions(app: fastapi.FastAPI): + app.add_exception_handler( + VersionIdNotSetError, version_id_not_set_exception_handler + ) diff --git a/backend/capellacollab/projects/toolmodels/modelsources/git/exceptions.py b/backend/capellacollab/projects/toolmodels/modelsources/git/exceptions.py index 906d24ebf..0837c0513 100644 --- a/backend/capellacollab/projects/toolmodels/modelsources/git/exceptions.py +++ b/backend/capellacollab/projects/toolmodels/modelsources/git/exceptions.py @@ -34,6 +34,12 @@ class GitPipelineJobFailedError(Exception): job_name: str +@dataclasses.dataclass +class GitPipelineJobUnknownStateError(Exception): + job_name: str + state: str + + class GithubArtifactExpiredError(Exception): pass @@ -110,6 +116,24 @@ async def git_pipeline_job_failed_handler( ) +async def unknown_state_handler( + request: fastapi.Request, exc: GitPipelineJobUnknownStateError +) -> fastapi.Response: + return await exception_handlers.http_exception_handler( + request, + fastapi.HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "err_code": "UNKNOWN_STATE_ERROR", + "reason": ( + f"Job '{exc.job_name}' has an unhandled or unknown state: '{exc.state}'", + "Please contact your administrator.", + ), + }, + ), + ) + + async def github_artifact_expired_handler( request: fastapi.Request, _: GithubArtifactExpiredError ) -> fastapi.Response: @@ -141,6 +165,9 @@ def register_exceptions(app: fastapi.FastAPI): GitPipelineJobNotFoundError, git_pipeline_job_not_found_handler, ) + app.add_exception_handler( + GitPipelineJobUnknownStateError, unknown_state_handler + ) app.add_exception_handler( GitPipelineJobFailedError, git_pipeline_job_failed_handler, diff --git a/backend/capellacollab/projects/toolmodels/modelsources/git/github/handler.py b/backend/capellacollab/projects/toolmodels/modelsources/git/github/handler.py index 37b2c0cb3..77fce82fd 100644 --- a/backend/capellacollab/projects/toolmodels/modelsources/git/github/handler.py +++ b/backend/capellacollab/projects/toolmodels/modelsources/git/github/handler.py @@ -61,7 +61,7 @@ def __get_file_from_repository( self, project_id: str, trusted_file_path: str, - revision: str | None = None, + revision: str, headers: dict[str, str] | None = None, ) -> requests.Response: return requests.get( @@ -179,7 +179,9 @@ def __get_latest_successful_job(self, jobs: list, job_name: str) -> dict: ): raise git_exceptions.GitPipelineJobFailedError(job_name) - return None + raise git_exceptions.GitPipelineJobUnknownStateError( + job_name, matched_jobs[0]["conclusion"] + ) def __get_latest_artifact_metadata(self, project_id: str, job_id: str): response = requests.get( diff --git a/backend/capellacollab/projects/toolmodels/modelsources/git/handler/factory.py b/backend/capellacollab/projects/toolmodels/modelsources/git/handler/factory.py index d52620a25..3ef107284 100644 --- a/backend/capellacollab/projects/toolmodels/modelsources/git/handler/factory.py +++ b/backend/capellacollab/projects/toolmodels/modelsources/git/handler/factory.py @@ -28,7 +28,7 @@ def create_git_handler( return github_handler.GithubHandler(git_model, git_instance) case _: raise exceptions.GitInstanceUnsupportedError( - instance_name=git_instance.type + instance_name=str(git_instance.type) ) @staticmethod diff --git a/backend/capellacollab/projects/toolmodels/modelsources/git/handler/handler.py b/backend/capellacollab/projects/toolmodels/modelsources/git/handler/handler.py index 75ebe750c..06f9ac632 100644 --- a/backend/capellacollab/projects/toolmodels/modelsources/git/handler/handler.py +++ b/backend/capellacollab/projects/toolmodels/modelsources/git/handler/handler.py @@ -7,6 +7,7 @@ import typing as t import requests +from capellambse import diagram_cache import capellacollab.projects.toolmodels.modelsources.git.models as git_models import capellacollab.settings.modelsources.git.models as settings_git_models @@ -34,7 +35,7 @@ async def get_project_id_by_git_url(self) -> str: @abc.abstractmethod async def get_last_job_run_id_for_git_model( - self, job_name: str, project_id: str | None + self, job_name: str, project_id: str | None = None ) -> tuple[str, str]: pass @@ -76,7 +77,7 @@ async def get_file_from_repository_or_artifacts_as_json( trusted_file_path: str, job_name: str, revision: str | None = None, - ) -> tuple[datetime.datetime, dict[str, t.Any]]: + ) -> tuple[datetime.datetime, list[diagram_cache.IndexEntry]]: ( last_updated, result, diff --git a/backend/capellacollab/projects/users/routes.py b/backend/capellacollab/projects/users/routes.py index 08560add3..c12e7d56f 100644 --- a/backend/capellacollab/projects/users/routes.py +++ b/backend/capellacollab/projects/users/routes.py @@ -60,7 +60,7 @@ def get_project_user_association_or_raise( return models.ProjectUser( role=models.ProjectUserRole.USER, permission=models.ProjectUserPermission.READ, - user=user, + user=users_models.User.model_validate(user), ) raise fastapi.HTTPException( @@ -88,7 +88,7 @@ def get_current_user( return models.ProjectUser( role=models.ProjectUserRole.ADMIN, permission=models.ProjectUserPermission.WRITE, - user=user, + user=users_models.User.model_validate(user), ) return get_project_user_association_or_raise(db, project, user) diff --git a/backend/capellacollab/sessions/files/routes.py b/backend/capellacollab/sessions/files/routes.py index 41fc4a20d..a0c6709cc 100644 --- a/backend/capellacollab/sessions/files/routes.py +++ b/backend/capellacollab/sessions/files/routes.py @@ -63,6 +63,8 @@ def upload_files( for file in files: file.file.seek(0) + + assert file.filename file.filename = file.filename.replace(" ", "_") tar.addfile( tar.gettarinfo(arcname=file.filename, fileobj=file.file), diff --git a/backend/capellacollab/sessions/hooks/jupyter.py b/backend/capellacollab/sessions/hooks/jupyter.py index 78dd23d11..3bc359b2d 100644 --- a/backend/capellacollab/sessions/hooks/jupyter.py +++ b/backend/capellacollab/sessions/hooks/jupyter.py @@ -41,7 +41,7 @@ def __init__(self): ) self._general_conf = config["general"] - def configuration_hook( + def configuration_hook( # type: ignore[override] self, db: orm.Session, user: users_models.DatabaseUser, @@ -65,7 +65,7 @@ def configuration_hook( volumes = self._get_project_share_volume_mounts(db, token, tool) - return environment, volumes, [] + return environment, volumes, [] # type: ignore[return-value] def post_session_creation_hook( self, @@ -76,12 +76,12 @@ def post_session_creation_hook( ): operator.create_public_route( session_id=session_id, - host=self._jupyter_public_uri.hostname, + host=self._jupyter_public_uri.hostname or "", path=self._determine_base_url(user.name), port=8888, ) - def pre_session_termination_hook( + def pre_session_termination_hook( # type: ignore self, operator: operators.KubernetesOperator, session: sessions_models.DatabaseSession, diff --git a/backend/capellacollab/sessions/hooks/persistent_workspace.py b/backend/capellacollab/sessions/hooks/persistent_workspace.py index a54f120bb..6613c04b3 100644 --- a/backend/capellacollab/sessions/hooks/persistent_workspace.py +++ b/backend/capellacollab/sessions/hooks/persistent_workspace.py @@ -22,7 +22,7 @@ class PersistentWorkspaceHook(interface.HookRegistration): Is responsible for mounting the persistent workspace into persistent sessions. """ - def configuration_hook( + def configuration_hook( # type: ignore self, operator: operators.KubernetesOperator, user: users_models.DatabaseUser, diff --git a/backend/capellacollab/sessions/hooks/pure_variants.py b/backend/capellacollab/sessions/hooks/pure_variants.py index e5d648598..1a02d40f9 100644 --- a/backend/capellacollab/sessions/hooks/pure_variants.py +++ b/backend/capellacollab/sessions/hooks/pure_variants.py @@ -25,7 +25,7 @@ class PureVariantsConfigEnvironment(t.TypedDict): class PureVariantsIntegration(interface.HookRegistration): - def configuration_hook( + def configuration_hook( # type: ignore self, db: orm.Session, user: users_models.DatabaseUser, diff --git a/backend/capellacollab/sessions/hooks/t4c.py b/backend/capellacollab/sessions/hooks/t4c.py index 03011f775..ce094b858 100644 --- a/backend/capellacollab/sessions/hooks/t4c.py +++ b/backend/capellacollab/sessions/hooks/t4c.py @@ -35,7 +35,7 @@ class T4CConfigEnvironment(t.TypedDict): class T4CIntegration(interface.HookRegistration): - def configuration_hook( + def configuration_hook( # type: ignore self, db: orm.Session, user: users_models.DatabaseUser, @@ -47,7 +47,6 @@ def configuration_hook( list[operators_models.Volume], list[core_models.Message], ]: - environment = {} warnings: list[core_models.Message] = [] # When using a different tool with TeamForCapella support (e.g., Capella + pure::variants), @@ -57,7 +56,7 @@ def configuration_hook( db, tool_version.name, user ) - environment["T4C_JSON"] = json.dumps( + t4c_json = json.dumps( [ { "repository": repository.name, @@ -72,12 +71,16 @@ def configuration_hook( ] ) - environment["T4C_LICENCE_SECRET"] = ( - t4c_repositories[0].instance.license if t4c_repositories else None + t4c_licence_secret = ( + t4c_repositories[0].instance.license if t4c_repositories else "" ) - environment["T4C_USERNAME"] = user.name - environment["T4C_PASSWORD"] = credentials.generate_password() + environment = T4CConfigEnvironment( + T4C_LICENCE_SECRET=t4c_licence_secret, + T4C_JSON=t4c_json, + T4C_USERNAME=user.name, + T4C_PASSWORD=credentials.generate_password(), + ) for repository in t4c_repositories: try: @@ -109,7 +112,7 @@ def configuration_hook( return environment, [], warnings - def pre_session_termination_hook( + def pre_session_termination_hook( # type: ignore self, db: orm.Session, session: sessions_models.DatabaseSession, diff --git a/backend/capellacollab/sessions/models.py b/backend/capellacollab/sessions/models.py index 5413f9ce4..c8088bcc3 100644 --- a/backend/capellacollab/sessions/models.py +++ b/backend/capellacollab/sessions/models.py @@ -29,18 +29,17 @@ class WorkspaceType(enum.Enum): READONLY = "readonly" -class GetSessionsResponse(pydantic.BaseModel): +class Session(pydantic.BaseModel): model_config = pydantic.ConfigDict(from_attributes=True) id: str type: WorkspaceType created_at: datetime.datetime owner: users_models.BaseUser - state: str + guacamole_username: str | None = None guacamole_connection_id: str | None = None - warnings: list[core_models.Message] | None = None - last_seen: str + project: projects_models.Project | None = None version: tools_models.ToolVersionWithTool | None = None @@ -49,7 +48,11 @@ class GetSessionsResponse(pydantic.BaseModel): ) -class OwnSessionResponse(GetSessionsResponse): +class GetSessionsResponse(Session): + state: str + warnings: list[core_models.Message] | None = None + last_seen: str + t4c_password: str | None = None jupyter_uri: str | None = None diff --git a/backend/capellacollab/sessions/operators/k8s.py b/backend/capellacollab/sessions/operators/k8s.py index 439204cd2..8b664e5c3 100644 --- a/backend/capellacollab/sessions/operators/k8s.py +++ b/backend/capellacollab/sessions/operators/k8s.py @@ -150,7 +150,7 @@ def start_session( session_type: str, tool_name: str, version_name: str, - environment: dict[str, str | None], + environment: dict[str, str], ports: dict[str, int], volumes: list[models.Volume], prometheus_path="/metrics", @@ -441,7 +441,7 @@ def _create_deployment( self, image: str, name: str, - environment: dict[str, str | None], + environment: dict[str, str], ports: dict[str, int], volumes: list[models.Volume], limits: str = "high", @@ -745,7 +745,7 @@ def _create_openshift_route( return v1_routes.create(body=route_dict, namespace=namespace) def create_persistent_volume( - self, name: str, size: str, labels: dict[str, str] = None + self, name: str, size: str, labels: dict[str, str] | None = None ): pvc: client.V1PersistentVolumeClaim = client.V1PersistentVolumeClaim( kind="PersistentVolumeClaim", @@ -950,7 +950,7 @@ def print_file_tree_as_json(): print("Using CLI arguments: " + str(sys.argv[1:]), file=sys.stderr) - def get_files(dir: pathlib.PosixPath, show_hidden: bool): + def get_files(dir: pathlib.Path, show_hidden: bool): file = { "path": str(dir.absolute()), "name": dir.name, diff --git a/backend/capellacollab/sessions/routes.py b/backend/capellacollab/sessions/routes.py index 36aa16e81..8021ae91f 100644 --- a/backend/capellacollab/sessions/routes.py +++ b/backend/capellacollab/sessions/routes.py @@ -263,7 +263,7 @@ def models_as_json( def git_model_as_json( git_model: git_models.DatabaseGitModel, deep_clone: bool ) -> dict[str, str | int]: - d = { + d: dict[str, str | int] = { "url": git_model.path, "revision": git_model.revision, "depth": 0 if deep_clone else 1, @@ -474,21 +474,27 @@ def create_database_session( version: tools_models.DatabaseVersion, project: projects_models.DatabaseProject | None, **kwargs, -) -> models.DatabaseSession: - database_model = models.DatabaseSession( - tool=tool, - version=version, - owner_name=owner, - project=project, - type=type, - **session, - **kwargs, +) -> models.GetSessionsResponse: + db_session = crud.create_session( + db, + models.DatabaseSession( + tool=tool, + version=version, + owner_name=owner, + project=project, + type=type, + **session, + **kwargs, + ), ) - response = crud.create_session(db=db, session=database_model) - response.state = "New" - response.last_seen = "UNKNOWN" - response.warnings = [] - return response + + session_dict = models.Session.model_validate(db_session).model_dump() + + session_dict["state"] = "New" + session_dict["last_seen"] = "UNKNOWN" + session_dict["warnings"] = [] + + return models.GetSessionsResponse.model_validate(session_dict) def create_database_and_guacamole_session( @@ -501,7 +507,7 @@ def create_database_and_guacamole_session( version: tools_models.DatabaseVersion, project: projects_models.DatabaseProject | None, environment: dict[str, str], -) -> models.DatabaseSession: +) -> models.GetSessionsResponse: guacamole_username = credentials.generate_password() guacamole_password = credentials.generate_password(length=64) @@ -568,6 +574,14 @@ def create_guacamole_token( }, ) + if not (session.guacamole_username and session.guacamole_password): + raise fastapi.HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "reason": "The session does not contain a guacamole username or password" + }, + ) + token = guacamole.get_token( session.guacamole_username, session.guacamole_password ) @@ -581,7 +595,7 @@ def create_guacamole_token( @users_router.get( - "/{user_id}/sessions", response_model=list[models.OwnSessionResponse] + "/{user_id}/sessions", response_model=list[models.GetSessionsResponse] ) def get_sessions_for_user( user: users_models.DatabaseUser = fastapi.Depends( diff --git a/backend/capellacollab/sessions/sessions.py b/backend/capellacollab/sessions/sessions.py index e1775dddd..b58130da1 100644 --- a/backend/capellacollab/sessions/sessions.py +++ b/backend/capellacollab/sessions/sessions.py @@ -1,10 +1,9 @@ # SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors # SPDX-License-Identifier: Apache-2.0 - import logging import re -import typing as t +from collections import abc import requests @@ -16,15 +15,26 @@ def inject_attrs_in_sessions( - db_sessions: list[models.DatabaseSession], -) -> list[dict[str, t.Any]]: + db_sessions: abc.Sequence[models.DatabaseSession], +) -> list[models.GetSessionsResponse]: sessions_list = [] for session in db_sessions: - session.state = _determine_session_state(session) - session.last_seen = get_last_seen(session.id) - session.jupyter_uri = session.environment.get("JUPYTER_URI") - session.t4c_password = session.environment.get("T4C_PASSWORD") - sessions_list.append(session) + session_dict = models.Session.model_validate(session).model_dump() + + session_dict["state"] = _determine_session_state(session) + session_dict["last_seen"] = get_last_seen(session.id) + + if session.environment: + session_dict["jupyter_uri"] = session.environment.get( + "JUPYTER_URI" + ) + session_dict["t4c_password"] = session.environment.get( + "T4C_PASSWORD" + ) + + sessions_list.append( + models.GetSessionsResponse.model_validate(session_dict) + ) return sessions_list diff --git a/backend/capellacollab/settings/integrations/purevariants/crud.py b/backend/capellacollab/settings/integrations/purevariants/crud.py index 06d7964db..85fcb0d54 100644 --- a/backend/capellacollab/settings/integrations/purevariants/crud.py +++ b/backend/capellacollab/settings/integrations/purevariants/crud.py @@ -17,7 +17,7 @@ def get_pure_variants_configuration( def set_license_server_configuration( - db: orm.Session, value: str + db: orm.Session, value: str | None ) -> models.DatabasePureVariantsLicenses: if pv_license := get_pure_variants_configuration(db): pv_license.license_server_url = value diff --git a/backend/capellacollab/tools/models.py b/backend/capellacollab/tools/models.py index c5eee635f..d503de71c 100644 --- a/backend/capellacollab/tools/models.py +++ b/backend/capellacollab/tools/models.py @@ -22,7 +22,7 @@ class DatabaseTool(database.Base): id: orm.Mapped[int] = orm.mapped_column(primary_key=True) - name: orm.Mapped[str | None] + name: orm.Mapped[str] docker_image_template: orm.Mapped[str] docker_image_backup_template: orm.Mapped[str | None] readonly_docker_image_template: orm.Mapped[str | None] diff --git a/backend/capellacollab/tools/routes.py b/backend/capellacollab/tools/routes.py index 7d686d9d3..c36d12e8e 100644 --- a/backend/capellacollab/tools/routes.py +++ b/backend/capellacollab/tools/routes.py @@ -282,7 +282,12 @@ def raise_when_tool_dependency_exist( for model in tool_models ) - raise_if_dependencies_exist(dependencies, tool.name, "tool") + if dependencies: + raise core_exceptions.ExistingDependenciesError( + entity_name=tool.name, + entity_type="tool", + dependencies=dependencies, + ) def raise_when_tool_version_dependency_exist( @@ -314,11 +319,12 @@ def raise_when_tool_version_dependency_exist( for model in version_models ) - raise core_exceptions.ExistingDependenciesError( - entity_name=version.name, - entity_type="version", - dependencies=dependencies, - ) + if dependencies: + raise core_exceptions.ExistingDependenciesError( + entity_name=version.name, + entity_type="version", + dependencies=dependencies, + ) def raise_when_tool_nature_dependency_exist( @@ -341,8 +347,9 @@ def raise_when_tool_nature_dependency_exist( for model in nature_models ) - raise core_exceptions.ExistingDependenciesError( - entity_name=nature.name, - entity_type="nature", - dependencies=dependencies, - ) + if dependencies: + raise core_exceptions.ExistingDependenciesError( + entity_name=nature.name, + entity_type="nature", + dependencies=dependencies, + ) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index f165432c8..7d24ea5b9 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "alembic==1.10.4", "appdirs", "cachetools", + "capellambse==0.5.35.dev4", "fastapi>=0.101.0", "kubernetes", "msal", @@ -100,6 +101,7 @@ show_error_codes = true warn_redundant_casts = true warn_unreachable = true python_version = "3.11" +exclude = "capellacollab/alembic" [[tool.mypy.overrides]] module = ["tests.*"] @@ -109,7 +111,26 @@ allow_untyped_defs = true [[tool.mypy.overrides]] # Untyped third party libraries module = [ - # ... + "capellambse.*", + "kubernetes.*", + "prometheus_client.*", + "openshift.*", + "deepdiff.*", + "appdirs.*", + "jsonschema.*", + "requests.*", + "jose.*", + "slugify.*", + "cachetools.*", + "requests_oauthlib.*", + "msal.*", + "yaml.*", + "fastapi_pagination.*", + "aiohttp.*", + "starlette_prometheus.*", + "uvicorn.*", + "alembic.*", + "jwt.*", ] ignore_missing_imports = true