From 8f34a29a94bdb2ad93e229aa6dbdc075a96ed26e Mon Sep 17 00:00:00 2001 From: romeonicholas Date: Thu, 22 Feb 2024 13:22:41 +0100 Subject: [PATCH] feat: Convert config template to pydantic model --- backend/.gitignore | 1 - backend/capellacollab/__init__.py | 5 - backend/capellacollab/__main__.py | 6 +- backend/capellacollab/alembic/env.py | 4 +- backend/capellacollab/config/__init__.py | 18 +-- .../config/config.yaml} | 55 ++----- backend/capellacollab/config/diff.py | 15 +- backend/capellacollab/config/generate.py | 15 ++ backend/capellacollab/config/loader.py | 4 + backend/capellacollab/config/models.py | 143 ++++++++++++++++++ .../core/authentication/__init__.py | 5 +- .../core/authentication/jwt_bearer.py | 4 +- .../authentication/provider/oauth/__main__.py | 4 +- .../authentication/provider/oauth/flow.py | 30 ++-- .../authentication/provider/oauth/keystore.py | 10 +- .../capellacollab/core/database/__init__.py | 2 +- .../capellacollab/core/database/migration.py | 6 +- .../capellacollab/core/logging/__init__.py | 2 +- backend/capellacollab/core/logging/loki.py | 17 ++- backend/capellacollab/core/metadata.py | 17 ++- .../toolmodels/backups/runs/interface.py | 2 +- .../modelsources/git/github/handler.py | 10 +- .../modelsources/git/gitlab/handler.py | 12 +- backend/capellacollab/sessions/guacamole.py | 24 +-- .../capellacollab/sessions/hooks/jupyter.py | 13 +- backend/capellacollab/sessions/idletimeout.py | 6 +- .../capellacollab/sessions/operators/k8s.py | 47 +++--- backend/capellacollab/sessions/routes.py | 2 +- backend/capellacollab/sessions/sessions.py | 4 +- .../settings/modelsources/t4c/interface.py | 2 +- .../t4c/repositories/interface.py | 2 +- 31 files changed, 292 insertions(+), 195 deletions(-) rename backend/{config/config_template.yaml => capellacollab/config/config.yaml} (63%) create mode 100644 backend/capellacollab/config/generate.py create mode 100644 backend/capellacollab/config/models.py diff --git a/backend/.gitignore b/backend/.gitignore index 7e85607350..d55f338df9 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -3,7 +3,6 @@ .idea config/* -!config/config_template.yaml .history **/__pycache__/ .vscode diff --git a/backend/capellacollab/__init__.py b/backend/capellacollab/__init__.py index b05402fb25..3d9638f6f7 100644 --- a/backend/capellacollab/__init__.py +++ b/backend/capellacollab/__init__.py @@ -3,13 +3,8 @@ from importlib import metadata -from capellacollab import config as config_module - try: __version__ = metadata.version("capellacollab-backend") except metadata.PackageNotFoundError: __version__ = "0.0.0+unknown" del metadata - - -config_module.validate_schema() diff --git a/backend/capellacollab/__main__.py b/backend/capellacollab/__main__.py index 052758e476..db70182f0e 100644 --- a/backend/capellacollab/__main__.py +++ b/backend/capellacollab/__main__.py @@ -50,18 +50,18 @@ handlers: list[logging.Handler] = [ logging.StreamHandler(), core_logging.CustomTimedRotatingFileHandler( - str(config["logging"]["logPath"]) + "backend.log" + str(config.logging.logPath) + "backend.log" ), ] for handler in handlers: handler.setFormatter(core_logging.CustomFormatter()) -logging.basicConfig(level=config["logging"]["level"], handlers=handlers) +logging.basicConfig(level=config.logging.level, handlers=handlers) async def startup(): - migration.migrate_db(engine, config["database"]["url"]) + migration.migrate_db(engine, config.database.url) logging.info("Migrations done - Server is running") # This is needed to load the Kubernetes configuration at startup diff --git a/backend/capellacollab/alembic/env.py b/backend/capellacollab/alembic/env.py index 342c124210..4852e1352e 100644 --- a/backend/capellacollab/alembic/env.py +++ b/backend/capellacollab/alembic/env.py @@ -16,7 +16,7 @@ # access to the values within the .ini file in use. config = context.config -logging.basicConfig(level=cfg["logging"]["level"]) +logging.basicConfig(level=cfg.logging.level) if os.getenv("ALEMBIC_CONFIGURE_LOGGER", "true") != "false": logging.getLogger("capellacollab").setLevel("WARNING") @@ -25,7 +25,7 @@ # this will overwrite the ini-file sqlalchemy.url path # with the path given in the config of the main code if not config.get_main_option("sqlalchemy.url"): - config.set_main_option("sqlalchemy.url", cfg["database"]["url"]) + config.set_main_option("sqlalchemy.url", cfg.database.url) # Import models diff --git a/backend/capellacollab/config/__init__.py b/backend/capellacollab/config/__init__.py index d8d7031452..ef478ac9f7 100644 --- a/backend/capellacollab/config/__init__.py +++ b/backend/capellacollab/config/__init__.py @@ -1,23 +1,11 @@ # SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors # SPDX-License-Identifier: Apache-2.0 - import logging -import jsonschema -import jsonschema.exceptions - -from . import exceptions, loader +from . import loader, models log = logging.getLogger(__name__) -config = loader.load_yaml() - -def validate_schema(): - config_schema = loader.load_config_schema() - try: - jsonschema.validate(config, config_schema) - except jsonschema.exceptions.ValidationError as error: - raise exceptions.InvalidConfigurationError( - f"{error.__class__.__name__}: {error.message}", - ) from None +config_data = loader.load_yaml() +config = models.AppConfig(**config_data) diff --git a/backend/config/config_template.yaml b/backend/capellacollab/config/config.yaml similarity index 63% rename from backend/config/config_template.yaml rename to backend/capellacollab/config/config.yaml index 6c89c3f91a..ecc079fc28 100644 --- a/backend/config/config_template.yaml +++ b/backend/capellacollab/config/config.yaml @@ -4,18 +4,11 @@ docker: registry: k3d-myregistry.localhost:12345 externalRegistry: docker.io - k8s: - # Only required when using operator k8s - context: k3d-collab-cluster # Only required, if you'd like to use a local k3d environment + context: k3d-collab-cluster namespace: collab-sessions - - # apiURL: dummy # Only required when no kubectl context is available - # token: dummy # Only required when no kubectl context is available - storageClassName: local-path storageAccessMode: ReadWriteOnce - cluster: imagePullPolicy: Always podSecurityContext: @@ -23,80 +16,52 @@ k8s: runAsGroup: 1004370000 fsGroup: 1004370000 runAsNonRoot: true - promtail: - lokiEnabled: True + lokiEnabled: true lokiUrl: http://localhost:30001/loki/api/v1/push lokiUsername: localLokiUser lokiPassword: localLokiPassword serverPort: 3101 - + ingressClassName: traefik general: host: localhost port: 8000 scheme: http - wildcardHost: False - + wildcardHost: false extensions: guacamole: baseURI: http://localhost:8080/guacamole publicURI: http://localhost:8080/guacamole - username: guacadmin password: guacadmin - jupyter: publicURI: http://localhost:8080/jupyter - authentication: - provider: oauth # oauth | azure + provider: oauth jwt: - usernameClaim: sub # preferred_username - + usernameClaim: sub oauth: - # Only required when using provider oauth endpoints: wellKnown: http://localhost:8083/default/.well-known/openid-configuration - tokenIssuance: - authorization: - + tokenIssuance: null + authorization: null audience: default - scopes: - - openid - + - openid client: id: default - secret: - + secret: null redirectURI: http://localhost:4200/oauth2/callback - - # azure: - # # Only required when using provider azure - # authorizationEndpoint: http://tbd - - # client: - # id: tbd - # secret: tbd - - # audience: tbd - # redirectURI: http://localhost:4200/oauth2/callback - pipelines: timeout: 60 - database: url: postgresql://dev:dev@localhost:5432/dev - initial: admin: admin - logging: level: DEBUG logPath: logs/ - requests: timeout: 2 - prometheus: url: http://localhost:8080/prometheus/ diff --git a/backend/capellacollab/config/diff.py b/backend/capellacollab/config/diff.py index ee400663bb..a210f6c18f 100644 --- a/backend/capellacollab/config/diff.py +++ b/backend/capellacollab/config/diff.py @@ -3,12 +3,10 @@ # pylint: disable=bad-builtin -import pathlib - import deepdiff -import yaml from . import loader +from . import models as config_models class bcolors: @@ -22,16 +20,15 @@ class bcolors: print("Start comparison of configuration files") -config_template = yaml.safe_load( - ( - pathlib.Path(__file__).parents[2] / "config" / "config_template.yaml" - ).open() -) +pydantic_config = config_models.AppConfig() config = loader.load_yaml() diff = deepdiff.DeepDiff( - config, config_template, ignore_order=True, report_repetition=True + config, + pydantic_config.model_dump(), + ignore_order=True, + report_repetition=True, ) for key, value in ( diff --git a/backend/capellacollab/config/generate.py b/backend/capellacollab/config/generate.py new file mode 100644 index 0000000000..fdb6200208 --- /dev/null +++ b/backend/capellacollab/config/generate.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import yaml + +from . import models + +app_config = models.AppConfig() +config_dict = app_config.model_dump() +yaml_str = yaml.dump(config_dict, sort_keys=False) + +with open( + "backend/capellacollab/config/config.yaml", "w", encoding="utf-8" +) as yaml_file: + yaml_file.write(yaml_str) diff --git a/backend/capellacollab/config/loader.py b/backend/capellacollab/config/loader.py index f71d9dca2b..41c064010d 100644 --- a/backend/capellacollab/config/loader.py +++ b/backend/capellacollab/config/loader.py @@ -13,6 +13,7 @@ log = logging.getLogger(__name__) config_locations: list[pathlib.Path] = [ + pathlib.Path(__file__).parents[0] / "config.yaml", pathlib.Path(__file__).parents[2] / "config" / "config.yaml", pathlib.Path(appdirs.user_config_dir("capellacollab", "db")) / "config.yaml", @@ -56,6 +57,9 @@ def load_yaml() -> dict: ) return yaml.safe_load(loc.open()) + log.error( + "No configuration file found. Generate a default configuration with 'python -m capellacollab.config.generate'" + ) raise FileNotFoundError("config.yaml") diff --git a/backend/capellacollab/config/models.py b/backend/capellacollab/config/models.py new file mode 100644 index 0000000000..9cfb0e0279 --- /dev/null +++ b/backend/capellacollab/config/models.py @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import pydantic + + +class DockerConfig(pydantic.BaseModel): + registry: str = "k3d-myregistry.localhost:12345" + externalRegistry: str = "docker.io" + + +class K8sPodSecurityContext(pydantic.BaseModel): + runAsUser: int = 1004370000 + runAsGroup: int = 1004370000 + fsGroup: int = 1004370000 + runAsNonRoot: bool = True + + +class K8sClusterConfig(pydantic.BaseModel): + imagePullPolicy: str = "Always" + podSecurityContext: K8sPodSecurityContext = K8sPodSecurityContext() + + +class K8sPromtailConfig(pydantic.BaseModel): + lokiEnabled: bool = True + lokiUrl: str = "http://localhost:30001/loki/api/v1/push" + lokiUsername: str = "localLokiUser" + lokiPassword: str = "localLokiPassword" + serverPort: int = 3101 + + +# Only required when using operator k8s + + +class K8sConfig(pydantic.BaseModel): + # Only required, if you'd like to use a local k3d environment + context: str = "k3d-collab-cluster" + namespace: str = "collab-sessions" + storageClassName: str = "local-path" + storageAccessMode: str = "ReadWriteOnce" + cluster: K8sClusterConfig = K8sClusterConfig() + promtail: K8sPromtailConfig = K8sPromtailConfig() + ingressClassName: str = "traefik" + # Only required when no kubectl context is available + # apiURL: str | None = None + # token: str | None = None + + +class GeneralConfig(pydantic.BaseModel): + host: str = "localhost" + port: int = 8000 + scheme: str = "http" + wildcardHost: bool = False + + +class ExtensionGuacamoleConfig(pydantic.BaseModel): + baseURI: str = "http://localhost:8080/guacamole" + publicURI: str = "http://localhost:8080/guacamole" + username: str = "guacadmin" + password: str = "guacadmin" + + +class ExtensionJupyterConfig(pydantic.BaseModel): + publicURI: str = "http://localhost:8080/jupyter" + + +class ExtensionsConfig(pydantic.BaseModel): + guacamole: ExtensionGuacamoleConfig = ExtensionGuacamoleConfig() + jupyter: ExtensionJupyterConfig = ExtensionJupyterConfig() + + +class AuthOauthClientConfig(pydantic.BaseModel): + id: str = "default" + secret: str | None = None + + +# Only required when using provider oauth + + +class AuthOathEndpointsConfig(pydantic.BaseModel): + wellKnown: str = ( + "http://localhost:8083/default/.well-known/openid-configuration" + ) + tokenIssuance: str | None = None + authorization: str | None = None + + +class AuthOauthConfig(pydantic.BaseModel): + # Only required when using provider oauth + endpoints: AuthOathEndpointsConfig = AuthOathEndpointsConfig() + audience: str = "default" + scopes: list[str] = ["openid"] + client: AuthOauthClientConfig = AuthOauthClientConfig() + redirectURI: str = "http://localhost:4200/oauth2/callback" + + +class JWTConfig(pydantic.BaseModel): + usernameClaim: str = "sub" # preferred_username + + +class AuthenticationConfig(pydantic.BaseModel): + provider: str = "oauth" # oauth | azure + jwt: JWTConfig = JWTConfig() + oauth: AuthOauthConfig = AuthOauthConfig() + + +class PipelineConfig(pydantic.BaseModel): + timeout: int = 60 + + +class DatabaseConfig(pydantic.BaseModel): + url: str = "postgresql://dev:dev@localhost:5432/dev" + + +class InitialConfig(pydantic.BaseModel): + admin: str = "admin" + + +class LoggingConfig(pydantic.BaseModel): + level: str = "DEBUG" + logPath: str = "logs/" + + +class RequestsConfig(pydantic.BaseModel): + timeout: int = 2 + + +class PrometheusConfig(pydantic.BaseModel): + url: str = "http://localhost:8080/prometheus/" + + +class AppConfig(pydantic.BaseModel): + docker: DockerConfig = DockerConfig() + k8s: K8sConfig = K8sConfig() + general: GeneralConfig = GeneralConfig() + extensions: ExtensionsConfig = ExtensionsConfig() + authentication: AuthenticationConfig = AuthenticationConfig() + pipelines: PipelineConfig = PipelineConfig() + database: DatabaseConfig = DatabaseConfig() + initial: InitialConfig = InitialConfig() + logging: LoggingConfig = LoggingConfig() + requests: RequestsConfig = RequestsConfig() + prometheus: PrometheusConfig = PrometheusConfig() diff --git a/backend/capellacollab/core/authentication/__init__.py b/backend/capellacollab/core/authentication/__init__.py index 0665174b29..4e4d1ffb02 100644 --- a/backend/capellacollab/core/authentication/__init__.py +++ b/backend/capellacollab/core/authentication/__init__.py @@ -14,11 +14,10 @@ def get_authentication_entrypoint(): for i in metadata.entry_points()[ "capellacollab.authentication.providers" ] - if i.name == config["authentication"]["provider"] + if i.name == config.authentication.provider ) return ep except StopIteration: raise ValueError( - "Unknown authentication provider " - + config["authentication"]["provider"] + "Unknown authentication provider " + config.authentication.provider ) from None diff --git a/backend/capellacollab/core/authentication/jwt_bearer.py b/backend/capellacollab/core/authentication/jwt_bearer.py index 60ba2a5f1d..e9e40a6814 100644 --- a/backend/capellacollab/core/authentication/jwt_bearer.py +++ b/backend/capellacollab/core/authentication/jwt_bearer.py @@ -52,9 +52,7 @@ async def __call__( # type: ignore return None def get_username(self, token_decoded: dict[str, str]) -> str: - return token_decoded[ - config["authentication"]["jwt"]["usernameClaim"] - ].strip() + return token_decoded[config.authentication.jwt.usernameClaim].strip() def initialize_user(self, token_decoded: dict[str, str]): with database.SessionLocal() as session: diff --git a/backend/capellacollab/core/authentication/provider/oauth/__main__.py b/backend/capellacollab/core/authentication/provider/oauth/__main__.py index bbc0d66d41..bc9bae8c18 100644 --- a/backend/capellacollab/core/authentication/provider/oauth/__main__.py +++ b/backend/capellacollab/core/authentication/provider/oauth/__main__.py @@ -8,12 +8,12 @@ from .keystore import KeyStore -cfg = config["authentication"]["oauth"] +cfg = config.authentication.oauth def get_jwk_cfg(token: str) -> dict[str, t.Any]: return { "algorithms": ["RS256"], - "audience": cfg["audience"] or cfg["client"]["id"], + "audience": cfg.audience or cfg.client.id, "key": KeyStore.key_for_token(token).model_dump(), } diff --git a/backend/capellacollab/core/authentication/provider/oauth/flow.py b/backend/capellacollab/core/authentication/provider/oauth/flow.py index d51b0a6b31..80eb82abbf 100644 --- a/backend/capellacollab/core/authentication/provider/oauth/flow.py +++ b/backend/capellacollab/core/authentication/provider/oauth/flow.py @@ -12,16 +12,16 @@ from capellacollab.config import config -cfg = config["authentication"]["oauth"] +cfg = config.authentication.oauth logger = logging.getLogger(__name__) auth_args = {} -if cfg["scopes"]: - auth_args["scope"] = cfg["scopes"] +if cfg.scopes: + auth_args["scope"] = cfg.scopes auth_session = OAuth2Session( - cfg["client"]["id"], redirect_uri=cfg["redirectURI"], **auth_args + cfg.client.id, redirect_uri=cfg.redirectURI, **auth_args ) @@ -38,8 +38,8 @@ def get_token(code: str) -> dict[str, t.Any]: return auth_session.fetch_token( read_well_known()["token_endpoint"], code=code, - client_id=cfg["client"]["id"], - client_secret=cfg["client"]["secret"], + client_id=cfg.client.id, + client_secret=cfg.client.secret, ) @@ -48,8 +48,8 @@ def refresh_token(_refresh_token: str) -> dict[str, t.Any]: return auth_session.refresh_token( read_well_known()["token_endpoint"], refresh_token=_refresh_token, - client_id=cfg["client"]["id"], - client_secret=cfg["client"]["secret"], + client_id=cfg.client.id, + client_secret=cfg.client.secret, ) except Exception as e: logger.debug("Could not refresh token because of exception %s", str(e)) @@ -63,10 +63,10 @@ def refresh_token(_refresh_token: str) -> dict[str, t.Any]: def read_well_known() -> dict[str, t.Any]: - if cfg["endpoints"]["wellKnown"]: + if cfg.endpoints.wellKnown: r = requests.get( - cfg["endpoints"]["wellKnown"], - timeout=config["requests"]["timeout"], + cfg.endpoints.wellKnown, + timeout=config.requests.timeout, ) r.raise_for_status() @@ -75,11 +75,11 @@ def read_well_known() -> dict[str, t.Any]: authorization_endpoint = resp["authorization_endpoint"] token_endpoint = resp["token_endpoint"] - if cfg["endpoints"]["authorization"]: - authorization_endpoint = cfg["endpoints"]["authorization"] + if cfg.endpoints.authorization: + authorization_endpoint = cfg.endpoints.authorization - if cfg["endpoints"]["tokenIssuance"]: - token_endpoint = cfg["endpoints"]["tokenIssuance"] + if cfg.endpoints.tokenIssuance: + token_endpoint = cfg.endpoints.tokenIssuance return { "authorization_endpoint": authorization_endpoint, diff --git a/backend/capellacollab/core/authentication/provider/oauth/keystore.py b/backend/capellacollab/core/authentication/provider/oauth/keystore.py index 35ef7f8a73..7437fdece9 100644 --- a/backend/capellacollab/core/authentication/provider/oauth/keystore.py +++ b/backend/capellacollab/core/authentication/provider/oauth/keystore.py @@ -17,7 +17,7 @@ from .. import models as provider_models log = logging.getLogger(__name__) -cfg = config["authentication"]["oauth"] +cfg = config.authentication.oauth # Copied and adapted from https://github.com/marpaia/jwks/blob/master/jwks/jwks.py: @@ -50,9 +50,7 @@ def refresh_keys(self) -> None: if not self.jwks_uri: self.jwks_uri = self.get_jwks_uri() try: - resp = requests.get( - self.jwks_uri, timeout=config["requests"]["timeout"] - ) + resp = requests.get(self.jwks_uri, timeout=config.requests.timeout) except Exception: log.error("Could not retrieve JWKS data from %s", self.jwks_uri) return @@ -90,10 +88,10 @@ def key_for_token( return self.key_for_token(token, in_retry=1) -def _get_jwks_uri(wellknown_endpoint=cfg["endpoints"]["wellKnown"]): +def _get_jwks_uri(wellknown_endpoint=cfg.endpoints.wellKnown): openid_config = requests.get( wellknown_endpoint, - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, ).json() return openid_config["jwks_uri"] diff --git a/backend/capellacollab/core/database/__init__.py b/backend/capellacollab/core/database/__init__.py index 581e427b21..cc077f803e 100644 --- a/backend/capellacollab/core/database/__init__.py +++ b/backend/capellacollab/core/database/__init__.py @@ -11,7 +11,7 @@ from capellacollab.config import config engine = sa.create_engine( - config["database"]["url"], + config.database.url, connect_args={"connect_timeout": 5, "options": "-c timezone=utc"}, ) SessionLocal = orm.sessionmaker(autocommit=False, autoflush=False, bind=engine) diff --git a/backend/capellacollab/core/database/migration.py b/backend/capellacollab/core/database/migration.py index cf78b3f745..4ff3045d97 100644 --- a/backend/capellacollab/core/database/migration.py +++ b/backend/capellacollab/core/database/migration.py @@ -85,10 +85,10 @@ def migrate_db(engine, database_url: str): def initialize_admin_user(db: orm.Session): - LOGGER.info("Initialized adminuser %s", config["initial"]["admin"]) + LOGGER.info("Initialized adminuser %s", config.initial.admin) admin_user = users_crud.create_user( db=db, - username=config["initial"]["admin"], + username=config.initial.admin, role=users_models.Role.ADMIN, ) events_crud.create_user_creation_event(db, admin_user) @@ -116,7 +116,7 @@ def initialize_coffee_machine_project(db: orm.Session): def create_tools(db: orm.Session): LOGGER.info("Initialized tools") - registry = config["docker"]["registry"] + registry = config.docker.registry if os.getenv("DEVELOPMENT_MODE", "").lower() in ("1", "true", "t"): capella = tools_models.DatabaseTool( name="Capella", diff --git a/backend/capellacollab/core/logging/__init__.py b/backend/capellacollab/core/logging/__init__.py index 64e413d6ba..cd2a8631c5 100644 --- a/backend/capellacollab/core/logging/__init__.py +++ b/backend/capellacollab/core/logging/__init__.py @@ -16,7 +16,7 @@ from capellacollab import config from capellacollab.core.authentication import injectables as auth_injectables -LOGGING_LEVEL = config.config["logging"]["level"] +LOGGING_LEVEL = config.config.logging.level class CustomFormatter(logging.Formatter): diff --git a/backend/capellacollab/core/logging/loki.py b/backend/capellacollab/core/logging/loki.py index bef66ed356..42ff22ca8d 100644 --- a/backend/capellacollab/core/logging/loki.py +++ b/backend/capellacollab/core/logging/loki.py @@ -12,11 +12,12 @@ from requests import auth from capellacollab.config import config +from capellacollab.config import models as config_models from . import exceptions -LOGGING_LEVEL = config["logging"]["level"] -PROMTAIL_CONFIGURATION: dict[str, str] = config["k8s"]["promtail"] +LOGGING_LEVEL = config.logging.level +PROMTAIL_CONFIGURATION: config_models.K8sPromtailConfig = config.k8s.promtail class LogEntry(t.TypedDict): @@ -46,12 +47,12 @@ def push_logs_to_loki(entries: list[LogEntry], labels): # Send the log data to Loki try: response = requests.post( - PROMTAIL_CONFIGURATION["lokiUrl"] + "/push", + PROMTAIL_CONFIGURATION.lokiUrl + "/push", data=log_data, headers={"Content-Type": "application/json"}, auth=auth.HTTPBasicAuth( - PROMTAIL_CONFIGURATION["lokiUsername"], - PROMTAIL_CONFIGURATION["lokiPassword"], + PROMTAIL_CONFIGURATION.lokiUsername, + PROMTAIL_CONFIGURATION.lokiPassword, ), timeout=10, ) @@ -78,12 +79,12 @@ def fetch_logs_from_loki( # Send the query request to Loki try: response = requests.get( - PROMTAIL_CONFIGURATION["lokiUrl"] + "/query_range", + PROMTAIL_CONFIGURATION.lokiUrl + "/query_range", params=params, headers={"Content-Type": "application/json"}, auth=auth.HTTPBasicAuth( - PROMTAIL_CONFIGURATION["lokiUsername"], - PROMTAIL_CONFIGURATION["lokiPassword"], + PROMTAIL_CONFIGURATION.lokiUsername, + PROMTAIL_CONFIGURATION.lokiPassword, ), timeout=5, ) diff --git a/backend/capellacollab/core/metadata.py b/backend/capellacollab/core/metadata.py index a61cef9033..af40eb57f2 100644 --- a/backend/capellacollab/core/metadata.py +++ b/backend/capellacollab/core/metadata.py @@ -1,17 +1,18 @@ # SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors # SPDX-License-Identifier: Apache-2.0 -import typing as t - import fastapi import pydantic from sqlalchemy import orm import capellacollab from capellacollab.config import config +from capellacollab.config import models as config_models from capellacollab.core import database from capellacollab.settings.configuration import core as config_core -from capellacollab.settings.configuration import models as config_models +from capellacollab.settings.configuration import ( + models as settings_config_models, +) class Metadata(pydantic.BaseModel): @@ -31,7 +32,7 @@ class Metadata(pydantic.BaseModel): router = fastapi.APIRouter() -general_cfg: dict[str, t.Any] = config["general"] +general_cfg: config_models.GeneralConfig = config.general @router.get( @@ -40,14 +41,14 @@ class Metadata(pydantic.BaseModel): ) def get_metadata(db: orm.Session = fastapi.Depends(database.get_db)): cfg = config_core.get_config(db, "global") - assert isinstance(cfg, config_models.GlobalConfiguration) + assert isinstance(cfg, settings_config_models.GlobalConfiguration) return Metadata.model_validate( cfg.metadata.model_dump() | { "version": capellacollab.__version__, - "host": general_cfg.get("host"), - "port": str(general_cfg.get("port")), - "protocol": general_cfg.get("scheme"), + "host": general_cfg.host, + "port": str(general_cfg.port), + "protocol": general_cfg.scheme, } ) diff --git a/backend/capellacollab/projects/toolmodels/backups/runs/interface.py b/backend/capellacollab/projects/toolmodels/backups/runs/interface.py index d4fc43eae6..1f7da25d4d 100644 --- a/backend/capellacollab/projects/toolmodels/backups/runs/interface.py +++ b/backend/capellacollab/projects/toolmodels/backups/runs/interface.py @@ -23,7 +23,7 @@ log = logging.getLogger(__name__) -PIPELINES_TIMEOUT = config.get("pipelines", {}).get("timeout", 60) +PIPELINES_TIMEOUT = config.pipelines.timeout async def schedule_refresh_and_trigger_pipeline_jobs(interval=5): diff --git a/backend/capellacollab/projects/toolmodels/modelsources/git/github/handler.py b/backend/capellacollab/projects/toolmodels/modelsources/git/github/handler.py index 898940bd34..1a557137f0 100644 --- a/backend/capellacollab/projects/toolmodels/modelsources/git/github/handler.py +++ b/backend/capellacollab/projects/toolmodels/modelsources/git/github/handler.py @@ -66,7 +66,7 @@ def __get_file_from_repository( ) -> requests.Response: return requests.get( f"{self.git_instance.api_url}/repos/{project_id}/contents/{parse.quote(trusted_file_path)}?ref={parse.quote(revision, safe='')}", - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, headers=headers, ) @@ -112,7 +112,7 @@ def get_last_pipeline_runs( response = requests.get( f"{self.git_instance.api_url}/repos/{project_id}/actions/runs?branch={parse.quote(self.git_model.revision, safe='')}&per_page=20", headers=headers, - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, ) response.raise_for_status() return response.json()["workflow_runs"] @@ -128,7 +128,7 @@ def get_artifact_from_job( artifact_response = requests.get( f"{self.git_instance.api_url}/repos/{project_id}/actions/artifacts/{artifact_id}/zip", headers=self.__get_headers(self.git_model.password), - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, ) artifact_response.raise_for_status() @@ -146,7 +146,7 @@ def get_last_updated_for_file_path( if self.git_model.password else None ), - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, ) response.raise_for_status() if len(response.json()) == 0: @@ -186,7 +186,7 @@ def __get_latest_artifact_metadata(self, project_id: str, job_id: str): response = requests.get( f"{self.git_instance.api_url}/repos/{project_id}/actions/runs/{job_id}/artifacts", headers=self.__get_headers(self.git_model.password), - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, ) response.raise_for_status() artifact = response.json()["artifacts"][0] diff --git a/backend/capellacollab/projects/toolmodels/modelsources/git/gitlab/handler.py b/backend/capellacollab/projects/toolmodels/modelsources/git/gitlab/handler.py index a8c62b82d1..f36b586b5f 100644 --- a/backend/capellacollab/projects/toolmodels/modelsources/git/gitlab/handler.py +++ b/backend/capellacollab/projects/toolmodels/modelsources/git/gitlab/handler.py @@ -27,7 +27,7 @@ async def get_project_id_by_git_url(self) -> str: async with session.get( f"{self.git_instance.api_url}/projects/{project_name_encoded}", headers={"PRIVATE-TOKEN": self.git_model.password}, - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, ) as response: if response.status == 403: raise exceptions.GitlabAccessDeniedError @@ -60,7 +60,7 @@ def get_last_updated_for_file_path( response = requests.get( f"{self.git_instance.api_url}/projects/{project_id}/repository/commits?ref_name={revision or self.git_model.revision}&path={file_path}", headers={"PRIVATE-TOKEN": self.git_model.password}, - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, ) response.raise_for_status() if len(response.json()) == 0: @@ -77,7 +77,7 @@ async def __get_last_pipeline_run_ids( async with session.get( f"{self.git_instance.api_url}/projects/{project_id}/pipelines?ref={parse.quote(self.git_model.revision, safe='')}&per_page=20", headers={"PRIVATE-TOKEN": self.git_model.password}, - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, ) as response: response.raise_for_status() @@ -94,7 +94,7 @@ async def __get_job_id_for_job_name( async with session.get( f"{self.git_instance.api_url}/projects/{project_id}/pipelines/{pipeline_id}/jobs", headers={"PRIVATE-TOKEN": self.git_model.password}, - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, ) as response: response.raise_for_status() @@ -142,7 +142,7 @@ def get_artifact_from_job( response = requests.get( f"{self.git_instance.api_url}/projects/{project_id}/jobs/{job_id}/artifacts/{trusted_path_to_artifact}", headers={"PRIVATE-TOKEN": self.git_model.password}, - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, ) response.raise_for_status() return response @@ -157,7 +157,7 @@ async def get_file_from_repository( response = requests.get( f"{self.git_instance.api_url}/projects/{project_id}/repository/files/{parse.quote(trusted_file_path, safe='')}?ref={parse.quote(branch, safe='')}", headers={"PRIVATE-TOKEN": self.git_model.password}, - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, ) if response.status_code == 404: diff --git a/backend/capellacollab/sessions/guacamole.py b/backend/capellacollab/sessions/guacamole.py index 65f52be2c8..86da0cf8b2 100644 --- a/backend/capellacollab/sessions/guacamole.py +++ b/backend/capellacollab/sessions/guacamole.py @@ -11,8 +11,8 @@ from capellacollab.config import config from capellacollab.core import credentials -cfg = config["extensions"]["guacamole"] -GUACAMOLE_PREFIX = cfg["baseURI"] + "/api/session/data/postgresql" +cfg = config.extensions.guacamole +GUACAMOLE_PREFIX = cfg.baseURI + "/api/session/data/postgresql" headers = {"Content-Type": "application/x-www-form-urlencoded"} proxies = { "http": None, @@ -26,10 +26,10 @@ class GuacamoleError(Exception): def get_admin_token() -> str: r = requests.post( - cfg["baseURI"] + "/api/tokens", - auth=requests_auth.HTTPBasicAuth(cfg["username"], cfg["password"]), + cfg.baseURI + "/api/tokens", + auth=requests_auth.HTTPBasicAuth(cfg.username, cfg.password), headers=headers, - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, proxies=proxies, ) try: @@ -51,10 +51,10 @@ def get_admin_token() -> str: def get_token(username: str, password: str) -> str: r = requests.post( - cfg["baseURI"] + "/api/tokens", + cfg.baseURI + "/api/tokens", auth=requests_auth.HTTPBasicAuth(username, password), headers=headers, - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, proxies=proxies, ) r.raise_for_status() @@ -69,7 +69,7 @@ def create_user( r = requests.post( GUACAMOLE_PREFIX + "/users?token=" + token, json={"username": username, "password": password, "attributes": {}}, - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, proxies=proxies, ) r.raise_for_status() @@ -86,7 +86,7 @@ def assign_user_to_connection(token: str, username: str, connection_id: str): "value": "READ", } ], - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, proxies=proxies, ) r.raise_for_status() @@ -95,7 +95,7 @@ def assign_user_to_connection(token: str, username: str, connection_id: str): def delete_user(token: str, username: str): r = requests.delete( f"{GUACAMOLE_PREFIX}/users/{username}?token={token}", - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, proxies=proxies, ) r.raise_for_status() @@ -105,7 +105,7 @@ def delete_user(token: str, username: str): def delete_connection(token: str, connection_name: str): r = requests.delete( f"{GUACAMOLE_PREFIX}/connections/{connection_name}?token={token}", - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, proxies=proxies, ) r.raise_for_status() @@ -134,7 +134,7 @@ def create_connection( }, "attributes": {}, }, - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, proxies=proxies, ) diff --git a/backend/capellacollab/sessions/hooks/jupyter.py b/backend/capellacollab/sessions/hooks/jupyter.py index e74ffd3649..914f30bd0b 100644 --- a/backend/capellacollab/sessions/hooks/jupyter.py +++ b/backend/capellacollab/sessions/hooks/jupyter.py @@ -9,6 +9,7 @@ from sqlalchemy import orm from capellacollab.config import config +from capellacollab.config import models as config_models from capellacollab.core import credentials from capellacollab.core import models as core_models from capellacollab.core.authentication import injectables as auth_injectables @@ -37,16 +38,16 @@ class JupyterConfigEnvironment(t.TypedDict): class GeneralConfigEnvironment(t.TypedDict): scheme: str host: str - port: str + port: int wildcardHost: t.NotRequired[bool | None] class JupyterIntegration(interface.HookRegistration): def __init__(self): self._jupyter_public_uri: urllib_parse.ParseResult = ( - urllib_parse.urlparse(config["extensions"]["jupyter"]["publicURI"]) + urllib_parse.urlparse(config.extensions.jupyter.publicURI) ) - self._general_conf: GeneralConfigEnvironment = config["general"] + self._general_conf: config_models.GeneralConfig = config.general def configuration_hook( # type: ignore[override] self, @@ -65,8 +66,8 @@ def configuration_hook( # type: ignore[override] "JUPYTER_TOKEN": jupyter_token, "JUPYTER_BASE_URL": self._determine_base_url(user.name), "JUPYTER_PORT": "8888", - "JUPYTER_URI": f'{config["extensions"]["jupyter"]["publicURI"]}/{user.name}/lab?token={jupyter_token}', - "CSP_ORIGIN_HOST": f"{self._general_conf.get('scheme')}://{self._general_conf.get('host')}:{self._general_conf.get('port')}", + "JUPYTER_URI": f"{config.extensions.jupyter.publicURI}/{user.name}/lab?token={jupyter_token}", + "CSP_ORIGIN_HOST": f"{self._general_conf.scheme}://{self._general_conf.host}:{self._general_conf.port}", } volumes = self._get_project_share_volume_mounts(db, user.name, tool) @@ -86,7 +87,7 @@ def post_session_creation_hook( host=self._jupyter_public_uri.hostname or "", path=self._determine_base_url(user.name), port=8888, - wildcard_host=self._general_conf.get("wildcardHost", False), + wildcard_host=self._general_conf.wildcardHost or False, ) def pre_session_termination_hook( # type: ignore diff --git a/backend/capellacollab/sessions/idletimeout.py b/backend/capellacollab/sessions/idletimeout.py index 6978401224..164b0f9e1b 100644 --- a/backend/capellacollab/sessions/idletimeout.py +++ b/backend/capellacollab/sessions/idletimeout.py @@ -18,11 +18,11 @@ def terminate_idle_session(): - url = config["prometheus"]["url"] + url = config.prometheus.url url += "/".join(("api", "v1", 'query?query=ALERTS{alertstate="firing"}')) response = requests.get( url, - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, ) log.debug("Requested alerts %d", response.status_code) if response.status_code != 200: @@ -60,7 +60,7 @@ async def loop(): def run(): - logging.basicConfig(level=config["logging"]["level"]) + logging.basicConfig(level=config.logging.level) terminate_idle_session() diff --git a/backend/capellacollab/sessions/operators/k8s.py b/backend/capellacollab/sessions/operators/k8s.py index 20292005cc..fda2b9f392 100644 --- a/backend/capellacollab/sessions/operators/k8s.py +++ b/backend/capellacollab/sessions/operators/k8s.py @@ -26,6 +26,7 @@ from openshift.dynamic import exceptions as dynamic_exceptions from capellacollab.config import config +from capellacollab.config import models as config_models from . import helper, models @@ -38,35 +39,31 @@ "backend_sessions_killed", "Sessions killed, either by user or timeout" ) -external_registry: str = config["docker"]["externalRegistry"] +external_registry: str = config.docker.externalRegistry -cfg: dict[str, t.Any] = config["k8s"] +cfg: config_models.K8sConfig = config.k8s -namespace: str = cfg["namespace"] -storage_access_mode: str = cfg["storageAccessMode"] -storage_class_name: str = cfg["storageClassName"] +namespace: str = cfg.namespace +storage_access_mode: str = cfg.storageAccessMode +storage_class_name: str = cfg.storageClassName -loki_enabled: bool = cfg["promtail"]["lokiEnabled"] +loki_enabled: bool = cfg.promtail.lokiEnabled def deserialize_kubernetes_resource(content: t.Any, resource: str): # This is needed as "workaround" for the deserialize function class FakeKubeResponse: def __init__(self, obj): - self.data = json.dumps(obj) + self.data = json.dumps(obj.dict()) return client.ApiClient().deserialize(FakeKubeResponse(content), resource) # Resolve securityContext and pullPolicy -image_pull_policy: str = cfg.get("cluster", {}).get( - "imagePullPolicy", "Always" -) +image_pull_policy: str = cfg.cluster.imagePullPolicy pod_security_context = None -if _pod_security_context := cfg.get("cluster", {}).get( - "podSecurityContext", None -): +if _pod_security_context := cfg.cluster.podSecurityContext: pod_security_context = deserialize_kubernetes_resource( _pod_security_context, client.V1PodSecurityContext.__name__ ) @@ -115,9 +112,9 @@ def openshift(self): def load_config(self) -> None: self.kubectl_arguments = [] - if cfg.get("context", None): - self.kubectl_arguments += ["--context", cfg["context"]] - kubernetes.config.load_config(context=cfg["context"]) + if cfg.context: + self.kubectl_arguments += ["--context", cfg.context] + kubernetes.config.load_config(context=cfg.context) else: kubernetes.config.load_incluster_config() @@ -626,7 +623,7 @@ def create_secret( if overwrite: self.delete_secret(name) - return self.v1_core.create_namespaced_secret(cfg["namespace"], secret) + return self.v1_core.create_namespaced_secret(cfg.namespace, secret) def _create_cronjob( self, @@ -769,7 +766,7 @@ def _create_ingress( name=id, ), spec=client.V1IngressSpec( - ingress_class_name=cfg.get("ingressClassName"), + ingress_class_name=cfg.ingressClassName, rules=[ client.V1IngressRule( host=None if wildcard_host else host, @@ -920,18 +917,14 @@ def _create_promtail_configmap( "promtail.yaml": yaml.dump( { "server": { - "http_listen_port": cfg["promtail"]["serverPort"], + "http_listen_port": cfg.promtail.serverPort, }, "clients": [ { - "url": cfg["promtail"]["lokiUrl"] + "/push", + "url": cfg.promtail.lokiUrl + "/push", "basic_auth": { - "username": cfg["promtail"][ - "lokiUsername" - ], - "password": cfg["promtail"][ - "lokiPassword" - ], + "username": cfg.promtail.lokiUsername, + "password": cfg.promtail.lokiPassword, }, } ], @@ -1179,7 +1172,7 @@ def download_file(self, _id: str, filename: str) -> t.Iterable[bytes]: self.v1_core.connect_get_namespaced_pod_exec, pod_name, container=_id, - namespace=cfg["namespace"], + namespace=cfg.namespace, command=exec_command, stderr=True, stdin=False, diff --git a/backend/capellacollab/sessions/routes.py b/backend/capellacollab/sessions/routes.py index bf70a33227..83004cab25 100644 --- a/backend/capellacollab/sessions/routes.py +++ b/backend/capellacollab/sessions/routes.py @@ -602,7 +602,7 @@ def create_guacamole_token( ) return models.GuacamoleAuthentication( token=json.dumps(token), - url=config.config["extensions"]["guacamole"]["publicURI"] + "/#/", + url=config.config.extensions.guacamole.publicURI + "/#/", ) diff --git a/backend/capellacollab/sessions/sessions.py b/backend/capellacollab/sessions/sessions.py index 54be45839a..84a1219aca 100644 --- a/backend/capellacollab/sessions/sessions.py +++ b/backend/capellacollab/sessions/sessions.py @@ -41,12 +41,12 @@ def inject_attrs_in_sessions( def get_last_seen(sid: str) -> str: """Return project session last seen activity""" - url = config["prometheus"]["url"] + url = config.prometheus.url url += "/".join(("api", "v1", "query?query=idletime_minutes")) try: response = requests.get( url, - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, ) response.raise_for_status() for session in response.json()["data"]["result"]: diff --git a/backend/capellacollab/settings/modelsources/t4c/interface.py b/backend/capellacollab/settings/modelsources/t4c/interface.py index 055c7df17e..fc309fc983 100644 --- a/backend/capellacollab/settings/modelsources/t4c/interface.py +++ b/backend/capellacollab/settings/modelsources/t4c/interface.py @@ -22,7 +22,7 @@ def get_t4c_status( auth=requests_auth.HTTPBasicAuth( instance.username, instance.password ), - timeout=config.config["requests"]["timeout"], + timeout=config.config.requests.timeout, ) except requests.Timeout: raise fastapi.HTTPException( diff --git a/backend/capellacollab/settings/modelsources/t4c/repositories/interface.py b/backend/capellacollab/settings/modelsources/t4c/repositories/interface.py index 0cd5bdade2..e93074a62c 100644 --- a/backend/capellacollab/settings/modelsources/t4c/repositories/interface.py +++ b/backend/capellacollab/settings/modelsources/t4c/repositories/interface.py @@ -128,7 +128,7 @@ def make_request( method, url, auth=auth.HTTPBasicAuth(instance.username, instance.password), - timeout=config["requests"]["timeout"], + timeout=config.requests.timeout, **kwargs, )