From 81f0ba0c1448caeff46efaa564e0f825788e90bc Mon Sep 17 00:00:00 2001 From: dominik003 Date: Tue, 27 Jun 2023 14:42:03 +0200 Subject: [PATCH 1/5] fix: Add missing `/repositories` to url factory --- .../t4c-settings/service/t4c-repos/t4c-repo.service.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/settings/modelsources/t4c-settings/service/t4c-repos/t4c-repo.service.ts b/frontend/src/app/settings/modelsources/t4c-settings/service/t4c-repos/t4c-repo.service.ts index 132405636..c28021ed5 100644 --- a/frontend/src/app/settings/modelsources/t4c-settings/service/t4c-repos/t4c-repo.service.ts +++ b/frontend/src/app/settings/modelsources/t4c-settings/service/t4c-repos/t4c-repo.service.ts @@ -32,7 +32,9 @@ export class T4CRepoService { readonly repositories = this._repositories.asObservable(); urlFactory(instanceId: number, repositoryId: number): string { - return `${this.t4cInstanceService.urlFactory(instanceId)}/${repositoryId}`; + return `${this.t4cInstanceService.urlFactory( + instanceId + )}/repositories/${repositoryId}`; } loadRepositories(instanceId: number): void { From 6b4146e352825dec299f4da181b6e9ba0f6ba51d Mon Sep 17 00:00:00 2001 From: dominik003 Date: Tue, 27 Jun 2023 15:12:47 +0200 Subject: [PATCH 2/5] feat: Add http port to t4c instances --- ...2c1b545e3_add_http_port_to_t4c_instance.py | 24 ++++++++++++ .../projects/toolmodels/backups/core.py | 37 ++++++++++++------- .../toolmodels/backups/runs/routes.py | 5 ++- backend/capellacollab/sessions/routes.py | 4 +- .../settings/modelsources/t4c/models.py | 13 +++++++ .../services/settings/t4c-instance.service.ts | 1 + .../edit-t4c-instance.component.css | 24 ------------ .../edit-t4c-instance.component.html | 31 ++++++++++++++-- .../edit-t4c-instance.component.ts | 27 ++++++++------ 9 files changed, 111 insertions(+), 55 deletions(-) create mode 100644 backend/capellacollab/alembic/versions/3442c1b545e3_add_http_port_to_t4c_instance.py diff --git a/backend/capellacollab/alembic/versions/3442c1b545e3_add_http_port_to_t4c_instance.py b/backend/capellacollab/alembic/versions/3442c1b545e3_add_http_port_to_t4c_instance.py new file mode 100644 index 000000000..7ba6d9533 --- /dev/null +++ b/backend/capellacollab/alembic/versions/3442c1b545e3_add_http_port_to_t4c_instance.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Add http port to t4c instance + +Revision ID: 3442c1b545e3 +Revises: c6d27bd8cf6e +Create Date: 2023-06-26 17:04:34.613373 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "3442c1b545e3" +down_revision = "c6d27bd8cf6e" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "t4c_instances", sa.Column("http_port", sa.Integer(), nullable=True) + ) diff --git a/backend/capellacollab/projects/toolmodels/backups/core.py b/backend/capellacollab/projects/toolmodels/backups/core.py index 3d010211b..0d63ec039 100644 --- a/backend/capellacollab/projects/toolmodels/backups/core.py +++ b/backend/capellacollab/projects/toolmodels/backups/core.py @@ -13,25 +13,36 @@ def get_environment( - gitmodel: git_models.DatabaseGitModel, - t4cmodel: t4c_models.DatabaseT4CModel, + git_model: git_models.DatabaseGitModel, + t4c_model: t4c_models.DatabaseT4CModel, t4c_username: str, t4c_password: str, include_commit_history: bool = False, ) -> dict[str, str]: - return { - "GIT_REPO_URL": gitmodel.path, - "GIT_REPO_BRANCH": gitmodel.revision, - "GIT_USERNAME": gitmodel.username, - "GIT_PASSWORD": gitmodel.password, - "T4C_REPO_HOST": t4cmodel.repository.instance.host, - "T4C_REPO_PORT": str(t4cmodel.repository.instance.port), - "T4C_CDO_PORT": str(t4cmodel.repository.instance.cdo_port), - "T4C_REPO_NAME": t4cmodel.repository.name, - "T4C_PROJECT_NAME": t4cmodel.name, + env = { + "GIT_REPO_URL": git_model.path, + "GIT_REPO_BRANCH": git_model.revision, + "GIT_USERNAME": git_model.username, + "GIT_PASSWORD": git_model.password, + "T4C_REPO_HOST": t4c_model.repository.instance.host, + "T4C_REPO_PORT": str(t4c_model.repository.instance.port), + "T4C_REPO_NAME": t4c_model.repository.name, + "T4C_PROJECT_NAME": t4c_model.name, "T4C_USERNAME": t4c_username, "T4C_PASSWORD": t4c_password, "LOG_LEVEL": "INFO", "INCLUDE_COMMIT_HISTORY": json.dumps(include_commit_history), - "CONNECTION_TYPE": "telnet", } + + if http_port := t4c_model.repository.instance.http_port: + env = env | { + "HTTP_LOGIN": t4c_model.repository.instance.username, + "HTTP_PASSWORD": t4c_model.repository.instance.password, + "HTTP_PORT": str(http_port), + } + else: + env = env | { + "T4C_CDO_PORT": str(t4c_model.repository.instance.cdo_port) + } + + return env diff --git a/backend/capellacollab/projects/toolmodels/backups/runs/routes.py b/backend/capellacollab/projects/toolmodels/backups/runs/routes.py index 79376de6f..38074f730 100644 --- a/backend/capellacollab/projects/toolmodels/backups/runs/routes.py +++ b/backend/capellacollab/projects/toolmodels/backups/runs/routes.py @@ -155,7 +155,10 @@ def get_logs( ] ) - masked_values = [pipeline_run.pipeline.t4c_password] + masked_values = [ + pipeline_run.pipeline.t4c_password, + pipeline_run.pipeline.t4c_model.repository.instance.password, + ] masked_values_generated = [] # Also mask derivated, e.g. base64 encoded credentials diff --git a/backend/capellacollab/sessions/routes.py b/backend/capellacollab/sessions/routes.py index 2c93a0392..4ba0a3886 100644 --- a/backend/capellacollab/sessions/routes.py +++ b/backend/capellacollab/sessions/routes.py @@ -377,7 +377,9 @@ def start_persistent_guacamole_session( { "repository": repository.name, "protocol": repository.instance.protocol, - "port": repository.instance.port, + "port": repository.instance.http_port + if repository.instance.protocol == "ws" + else repository.instance.port, "host": repository.instance.host, "instance": repository.instance.name, } diff --git a/backend/capellacollab/settings/modelsources/t4c/models.py b/backend/capellacollab/settings/modelsources/t4c/models.py index 2b880d83f..f2581a909 100644 --- a/backend/capellacollab/settings/modelsources/t4c/models.py +++ b/backend/capellacollab/settings/modelsources/t4c/models.py @@ -60,6 +60,9 @@ class DatabaseT4CInstance(database.Base): sa.CheckConstraint("cdo_port >= 0 AND cdo_port <= 65535"), default=12036, ) + http_port: orm.Mapped[int | None] = orm.mapped_column( + sa.CheckConstraint("http_port >= 0 AND http_port <= 65535"), + ) usage_api: orm.Mapped[str] rest_api: orm.Mapped[str] username: orm.Mapped[str] @@ -89,6 +92,7 @@ class T4CInstanceBase(pydantic.BaseModel): host: str port: int cdo_port: int + http_port: int | None usage_api: str rest_api: str username: str @@ -107,6 +111,10 @@ class T4CInstanceBase(pydantic.BaseModel): port_validator ) + _validate_http_port = pydantic.validator("http_port", allow_reuse=True)( + port_validator + ) + class Config: orm_mode = True @@ -117,6 +125,7 @@ class PatchT4CInstance(pydantic.BaseModel): host: str | None port: int | None cdo_port: int | None + http_port: int | None usage_api: str | None rest_api: str | None username: str | None @@ -137,6 +146,10 @@ class PatchT4CInstance(pydantic.BaseModel): port_validator ) + _validate_http_port = pydantic.validator("http_port", allow_reuse=True)( + port_validator + ) + class Config: orm_mode = True diff --git a/frontend/src/app/services/settings/t4c-instance.service.ts b/frontend/src/app/services/settings/t4c-instance.service.ts index ccbbbbb34..96514461d 100644 --- a/frontend/src/app/services/settings/t4c-instance.service.ts +++ b/frontend/src/app/services/settings/t4c-instance.service.ts @@ -16,6 +16,7 @@ export type BaseT4CInstance = { host: string; port: number; cdo_port: number; + http_port?: number; usage_api: string; rest_api: string; username: string; diff --git a/frontend/src/app/settings/modelsources/t4c-settings/edit-t4c-instance/edit-t4c-instance.component.css b/frontend/src/app/settings/modelsources/t4c-settings/edit-t4c-instance/edit-t4c-instance.component.css index 1ca5c4293..d49deaffd 100644 --- a/frontend/src/app/settings/modelsources/t4c-settings/edit-t4c-instance/edit-t4c-instance.component.css +++ b/frontend/src/app/settings/modelsources/t4c-settings/edit-t4c-instance/edit-t4c-instance.component.css @@ -2,27 +2,3 @@ * SPDX-FileCopyrightText: Copyright DB Netz AG and the capella-collab-manager contributors * SPDX-License-Identifier: Apache-2.0 */ - -#submit-button-wrapper { - text-align: right; -} - -.space-evenly { - display: flex; - justify-content: space-evenly; -} - -#protocol { - width: 6em; - max-width: 85vw; -} - -#host { - width: 15em; - max-width: 85vw; -} - -#port { - width: 8em; - max-width: 85vw; -} diff --git a/frontend/src/app/settings/modelsources/t4c-settings/edit-t4c-instance/edit-t4c-instance.component.html b/frontend/src/app/settings/modelsources/t4c-settings/edit-t4c-instance/edit-t4c-instance.component.html index fc63e6f78..813c44691 100644 --- a/frontend/src/app/settings/modelsources/t4c-settings/edit-t4c-instance/edit-t4c-instance.component.html +++ b/frontend/src/app/settings/modelsources/t4c-settings/edit-t4c-instance/edit-t4c-instance.component.html @@ -64,8 +64,9 @@

Add a Team4Capella instance

+
- + Port Add a Team4Capella instance The port is required. - + + CDO port Add a Team4Capella instance The CDO port is required. + + + Http Port + + + Valid ports are between 0 and 65535. + + + We only support numerical ports :( + +
@@ -148,7 +166,12 @@

Add a Team4Capella instance
- @@ -162,7 +185,7 @@

Add a Team4Capella instance

Edit
-
+
-
- - Repository name - - Please enter a name - Name already used - The following characters are allowed: A-Z, a-z, 0-9, _, - - - -
- -
- - - sync - cloud_off - cloud_queue - block - cloud_upload - sync_problem -
-
{{ repository.name }}
-
Repository ID: {{ repository.id }}
-
status: {{ repository.status }}
+ + +
+
+ + Repository name + + Please enter a name + Name already used + The following characters are allowed: A-Z, a-z, 0-9, _, - + + +
+
- - +
-
-
-
- - - -
+ +
+
- Create on instance {{ instance!.name }} - +
+
+ sync + cloud_off + cloud_queue + block + cloud_upload + sync_problem +
+ +
+
+ {{ repository.name }} +
+
+ Repository ID: {{ repository.id }}
+ status: {{ repository.status }} +
+
+
+
+ + + + + + + +
+
-
- + +
diff --git a/frontend/src/app/settings/modelsources/t4c-settings/t4c-instance-settings/t4c-instance-settings.component.ts b/frontend/src/app/settings/modelsources/t4c-settings/t4c-instance-settings/t4c-instance-settings.component.ts index fdaf9f507..91727c63e 100644 --- a/frontend/src/app/settings/modelsources/t4c-settings/t4c-instance-settings/t4c-instance-settings.component.ts +++ b/frontend/src/app/settings/modelsources/t4c-settings/t4c-instance-settings/t4c-instance-settings.component.ts @@ -31,6 +31,8 @@ import { T4CRepoDeletionDialogComponent } from './t4c-repo-deletion-dialog/t4c-r export class T4CInstanceSettingsComponent implements OnChanges, OnDestroy { @Input() instance?: T4CInstance; + search = ''; + constructor( public t4cRepoService: T4CRepoService, private dialog: MatDialog @@ -55,6 +57,18 @@ export class T4CInstanceSettingsComponent implements OnChanges, OnDestroy { }), }); + getFilteredRepositories( + repositories: T4CServerRepository[] | undefined | null + ): T4CServerRepository[] | undefined { + if (repositories === undefined || repositories === null) { + return undefined; + } + + return repositories.filter((repository) => + repository.name.toLowerCase().includes(this.search.toLowerCase()) + ); + } + createRepository(): void { if (this.form.valid) { this.t4cRepoService @@ -73,27 +87,20 @@ export class T4CInstanceSettingsComponent implements OnChanges, OnDestroy { } startRepository(repository: T4CServerRepository): void { - this.repositoryList.selectedOptions.clear(); this.t4cRepoService .startRepository(repository.instance.id, repository.id) .subscribe(); } stopRepository(repository: T4CServerRepository): void { - this.repositoryList.selectedOptions.clear(); this.t4cRepoService .stopRepository(repository.instance.id, repository.id) .subscribe(); } recreateRepository(repository: T4CServerRepository): void { - this.repositoryList.selectedOptions.clear(); this.t4cRepoService .recreateRepository(repository.instance.id, repository.id) .subscribe(); } - - get selectedRepository(): T4CRepository & T4CServerRepository { - return this.repositoryList.selectedOptions.selected[0].value; - } } diff --git a/frontend/src/app/settings/modelsources/t4c-settings/t4c-settings.component.html b/frontend/src/app/settings/modelsources/t4c-settings/t4c-settings.component.html index 1a6782448..24011adf0 100644 --- a/frontend/src/app/settings/modelsources/t4c-settings/t4c-settings.component.html +++ b/frontend/src/app/settings/modelsources/t4c-settings/t4c-settings.component.html @@ -3,11 +3,6 @@ ~ SPDX-License-Identifier: Apache-2.0 --> - -
From 8a878e89575e5b386fbfe517e022d0a1229b8df5 Mon Sep 17 00:00:00 2001 From: dominik003 Date: Mon, 3 Jul 2023 10:34:22 +0200 Subject: [PATCH 4/5] test: Validate that log secrets are masked --- .gitignore | 1 - backend/.gitignore | 1 + .../toolmodels/backups/runs/injectables.py | 4 +- .../toolmodels/backups/runs/routes.py | 5 +- .../pipeline-runs/test_pipeline_runs.py | 77 +++++++++++++++++++ 5 files changed, 83 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index f5d18e695..ff70b131f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,5 @@ certs/* /node_modules .idea .vscode -/backend/log/* /helm/charts/* /logs/* diff --git a/backend/.gitignore b/backend/.gitignore index 292acd219..1330581c9 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -12,6 +12,7 @@ config/* build .git_archival.txt logs +backend.log* htmlcov .coverage diff --git a/backend/capellacollab/projects/toolmodels/backups/runs/injectables.py b/backend/capellacollab/projects/toolmodels/backups/runs/injectables.py index 9b4e0a8bc..323a5bec4 100644 --- a/backend/capellacollab/projects/toolmodels/backups/runs/injectables.py +++ b/backend/capellacollab/projects/toolmodels/backups/runs/injectables.py @@ -11,7 +11,7 @@ ) from capellacollab.projects.toolmodels.backups import models as backups_models -from . import crud +from . import crud, models def get_existing_pipeline_run( @@ -20,7 +20,7 @@ def get_existing_pipeline_run( backups_injectables.get_existing_pipeline ), db: orm.Session = fastapi.Depends(database.get_db), -) -> backups_models.DatabaseBackup: +) -> models.DatabasePipelineRun: if pipeline_run := crud.get_pipeline_run_by_id(db, pipeline_run_id): if pipeline_run.pipeline.id != pipeline.id: raise fastapi.HTTPException( diff --git a/backend/capellacollab/projects/toolmodels/backups/runs/routes.py b/backend/capellacollab/projects/toolmodels/backups/runs/routes.py index 38074f730..75b43634c 100644 --- a/backend/capellacollab/projects/toolmodels/backups/runs/routes.py +++ b/backend/capellacollab/projects/toolmodels/backups/runs/routes.py @@ -81,10 +81,11 @@ def get_pipeline_runs( response_model=models.PipelineRun, ) def get_pipeline_run( - pipeline_run: pipeline_models.DatabaseBackup = fastapi.Depends( + pipeline_run: models.DatabasePipelineRun = fastapi.Depends( injectables.get_existing_pipeline_run ), -) -> list[models.DatabasePipelineRun]: +) -> models.DatabasePipelineRun: + print(pipeline_run.status) return pipeline_run diff --git a/backend/tests/projects/toolmodels/pipelines/pipeline-runs/test_pipeline_runs.py b/backend/tests/projects/toolmodels/pipelines/pipeline-runs/test_pipeline_runs.py index 347b22563..94ca51825 100644 --- a/backend/tests/projects/toolmodels/pipelines/pipeline-runs/test_pipeline_runs.py +++ b/backend/tests/projects/toolmodels/pipelines/pipeline-runs/test_pipeline_runs.py @@ -3,6 +3,7 @@ import datetime import time +from unittest import mock import pytest from fastapi import testclient @@ -13,7 +14,16 @@ import capellacollab.projects.toolmodels.backups.runs.crud as pipeline_runs_crud import capellacollab.projects.toolmodels.backups.runs.models as pipeline_runs_models import capellacollab.projects.toolmodels.models as toolmodels_models +from capellacollab.__main__ import app from capellacollab.core.logging import loki +from capellacollab.projects.toolmodels.backups.runs import ( + injectables as runs_injectables, +) +from capellacollab.projects.toolmodels.backups.runs import ( + models as runs_models, +) +from capellacollab.users import crud as users_crud +from capellacollab.users import models as users_models @pytest.fixture(name="unix_time_in_ns") @@ -137,3 +147,70 @@ def def_get_logs( assert response.status_code == 200 assert b"test3" in response.content + + +@pytest.fixture(name="mock_pipeline_run") +def fixture_mock_pipeline_run(): + mock_pipeline_run = mock.MagicMock(spec=runs_models.DatabasePipelineRun) + + # Assign the values you want the mock object to return + mock_pipeline_run.id = "mock_id" + mock_pipeline_run.reference_id = "mock_reference_id" + mock_pipeline_run.trigger_time = "mock_time" + + # These values will be masked + mock_pipeline_run.pipeline.t4c_password = "secret_pipeline_t4c_password" + mock_pipeline_run.pipeline.t4c_model.repository.instance.password = ( + "secret_t4c_instance_password" + ) + + return mock_pipeline_run + + +@pytest.fixture(name="override_get_existing_pipeline_run_dependency") +def fixture_override_get_existing_pipeline_run_dependency( + mock_pipeline_run: mock.Mock, +): + def get_mock_existing_pipeline_run() -> runs_models.DatabasePipelineRun: + return mock_pipeline_run + + app.dependency_overrides[ + runs_injectables.get_existing_pipeline_run + ] = get_mock_existing_pipeline_run + + yield + + app.dependency_overrides.pop( + runs_injectables.get_existing_pipeline_run, None + ) + + +@mock.patch("capellacollab.core.logging.loki.fetch_logs_from_loki") +@pytest.mark.usefixtures("override_get_existing_pipeline_run_dependency") +def test_mask_logs( + mock_fetch_logs: mock.Mock, + client: testclient.TestClient, + db: orm.Session, + executor_name: str, +): + users_crud.create_user(db, executor_name, users_models.Role.ADMIN) + + mock_fetch_logs.return_value = [ + { + "values": [ + ( + 1618181583458797952, + "This is a log entry containing a secret_pipeline_t4c_password and secret_t4c_instance_password", + ) + ] + } + ] + + response = client.get( + "/api/v1/projects/1/models/1/backups/pipelines/1/runs/1/logs" + ) + + logs = response.json() + + assert "secret_pipeline_t4c_password" not in logs + assert "secret_t4c_instance_password" not in logs From 202447bafd5f36a1408fd1afbb677f801cebcc71 Mon Sep 17 00:00:00 2001 From: dominik003 Date: Mon, 3 Jul 2023 10:43:42 +0200 Subject: [PATCH 5/5] feat: Apply PR comments for PR #802 --- .../projects/toolmodels/backups/core.py | 4 +- .../toolmodels/backups/runs/routes.py | 5 +- .../edit-t4c-instance.component.html | 2 +- .../edit-t4c-instance.component.ts | 27 ++++---- .../t4c-instance-settings.component.html | 66 +++++++++---------- 5 files changed, 49 insertions(+), 55 deletions(-) diff --git a/backend/capellacollab/projects/toolmodels/backups/core.py b/backend/capellacollab/projects/toolmodels/backups/core.py index 0d63ec039..dcdc3d34b 100644 --- a/backend/capellacollab/projects/toolmodels/backups/core.py +++ b/backend/capellacollab/projects/toolmodels/backups/core.py @@ -35,13 +35,13 @@ def get_environment( } if http_port := t4c_model.repository.instance.http_port: - env = env | { + env |= { "HTTP_LOGIN": t4c_model.repository.instance.username, "HTTP_PASSWORD": t4c_model.repository.instance.password, "HTTP_PORT": str(http_port), } else: - env = env | { + env |= env | { "T4C_CDO_PORT": str(t4c_model.repository.instance.cdo_port) } diff --git a/backend/capellacollab/projects/toolmodels/backups/runs/routes.py b/backend/capellacollab/projects/toolmodels/backups/runs/routes.py index 75b43634c..c8bd7d1c6 100644 --- a/backend/capellacollab/projects/toolmodels/backups/runs/routes.py +++ b/backend/capellacollab/projects/toolmodels/backups/runs/routes.py @@ -127,10 +127,7 @@ def _transform_unix_nanoseconds_to_human_readable_format( ) -@router.get( - "/{pipeline_run_id}/logs", - response_model=str, -) +@router.get("/{pipeline_run_id}/logs", response_model=str) def get_logs( pipeline_run: models.DatabasePipelineRun = fastapi.Depends( injectables.get_existing_pipeline_run diff --git a/frontend/src/app/settings/modelsources/t4c-settings/edit-t4c-instance/edit-t4c-instance.component.html b/frontend/src/app/settings/modelsources/t4c-settings/edit-t4c-instance/edit-t4c-instance.component.html index 813c44691..7cd7a4be6 100644 --- a/frontend/src/app/settings/modelsources/t4c-settings/edit-t4c-instance/edit-t4c-instance.component.html +++ b/frontend/src/app/settings/modelsources/t4c-settings/edit-t4c-instance/edit-t4c-instance.component.html @@ -104,7 +104,7 @@

Add a Team4Capella instance

- Http Port + HTTP port +

Repositories

- -
-
- - Repository name - - Please enter a name - Name already used - The following characters are allowed: A-Z, a-z, 0-9, _, - - - -
- -
-
- -
- + +
+ Search Repositories
-
+
{{ repository.name }}
@@ -115,7 +88,7 @@

Repositories

(click)="startRepository(repository)" matTooltip="Start repository" > - power_settings_new + power_settings_new
+ +
+
+ + Repository name + + Please enter a name + Repository already exists + The following characters are allowed: A-Z, a-z, 0-9, _, - + + + +
+