From 6424741f332df337ae1002c93f8fa69b8e168ffe Mon Sep 17 00:00:00 2001 From: MoritzWeber Date: Fri, 1 Nov 2024 11:44:36 +0100 Subject: [PATCH] feat: Customize schedule and tz of nightly pipelines via global config The configuration only applies to newly created pipelines. --- .pre-commit-config.yaml | 1 + .../projects/toolmodels/backups/routes.py | 6 ++- .../capellacollab/sessions/operators/k8s.py | 4 +- .../settings/configuration/models.py | 38 +++++++++++++++++++ backend/pyproject.toml | 8 +++- .../settings/test_global_configuration.py | 18 +++++++++ .../src/app/openapi/.openapi-generator/FILES | 2 + .../model/global-configuration-input.ts | 2 + .../model/global-configuration-output.ts | 2 + frontend/src/app/openapi/model/models.ts | 2 + .../model/pipeline-configuration-input.ts | 24 ++++++++++++ .../model/pipeline-configuration-output.ts | 24 ++++++++++++ 12 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 frontend/src/app/openapi/model/pipeline-configuration-input.ts create mode 100644 frontend/src/app/openapi/model/pipeline-configuration-output.ts diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 92bfcb137e..226653fd9f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,6 +76,7 @@ repos: - typer - types-lxml - cryptography + - types-croniter - repo: local hooks: - id: pylint diff --git a/backend/capellacollab/projects/toolmodels/backups/routes.py b/backend/capellacollab/projects/toolmodels/backups/routes.py index 3cecdfa16d..8e6927bace 100644 --- a/backend/capellacollab/projects/toolmodels/backups/routes.py +++ b/backend/capellacollab/projects/toolmodels/backups/routes.py @@ -23,6 +23,7 @@ ) from capellacollab.projects.users import models as projects_users_models from capellacollab.sessions import operators +from capellacollab.settings.configuration import core as configuration_core from capellacollab.settings.modelsources.t4c.instance.repositories import ( interface as t4c_repository_interface, ) @@ -99,6 +100,8 @@ def create_backup( exceptions.PipelineOperation.CREATE ) + pipeline_config = configuration_core.get_global_configuration(db).pipelines + if body.run_nightly: if not toolmodel.version_id: raise toolmodels_exceptions.VersionIdNotSetError(toolmodel.id) @@ -117,7 +120,8 @@ def create_backup( labels=core.get_pipeline_labels(toolmodel), tool_resources=toolmodel.tool.config.resources, command="backup", - schedule="0 3 * * *", + schedule=pipeline_config.cron, + timezone=pipeline_config.timezone, ) else: reference = operators.get_operator()._generate_id() diff --git a/backend/capellacollab/sessions/operators/k8s.py b/backend/capellacollab/sessions/operators/k8s.py index 0732a48e9c..bf733fc392 100644 --- a/backend/capellacollab/sessions/operators/k8s.py +++ b/backend/capellacollab/sessions/operators/k8s.py @@ -275,6 +275,7 @@ def create_cronjob( tool_resources: tools_models.Resources, environment: dict[str, str | None], schedule="* * * * *", + timezone="UTC", timeout=18000, ) -> str: _id = self._generate_id() @@ -282,9 +283,10 @@ def create_cronjob( cronjob: client.V1CronJob = client.V1CronJob( kind="CronJob", api_version="batch/v1", - metadata=client.V1ObjectMeta(name=_id), + metadata=client.V1ObjectMeta(name=_id, labels=labels), spec=client.V1CronJobSpec( schedule=schedule, + time_zone=timezone, job_template=client.V1JobTemplateSpec( metadata=client.V1ObjectMeta(labels=labels), spec=self._create_job_spec( diff --git a/backend/capellacollab/settings/configuration/models.py b/backend/capellacollab/settings/configuration/models.py index a3b8d79f1e..ba8d0631cc 100644 --- a/backend/capellacollab/settings/configuration/models.py +++ b/backend/capellacollab/settings/configuration/models.py @@ -4,8 +4,10 @@ import abc import enum import typing as t +import zoneinfo import pydantic +from croniter import croniter from sqlalchemy import orm from capellacollab import core @@ -166,6 +168,38 @@ class ConfigurationBase(core_pydantic.BaseModelStrict, abc.ABC): _name: t.ClassVar[str] +class PipelineConfiguration(core_pydantic.BaseModelStrict): + cron: str = pydantic.Field( + default="0 3 * * *", + description=( + "Cron for nightly backup. Only applies to newly created pipelines." + ), + ) + timezone: str = pydantic.Field( + default="UTC", + description="Timezone for the cron expression.", + ) + + @pydantic.field_validator("cron") + @classmethod + def validate_cron(cls, v: str) -> str: + if croniter.is_valid(v): + return v + + raise ValueError("Cron doesn't have a valid syntax.") + + @pydantic.field_validator("timezone") + @classmethod + def validate_timezone(cls, v: str) -> str: + if v in zoneinfo.available_timezones(): + return v + + raise ValueError( + "Timezone is not valid. A list of timezones can be found at" + " https://en.wikipedia.org/wiki/List_of_tz_database_time_zones" + ) + + class GlobalConfiguration(ConfigurationBase): """Global application configuration.""" @@ -183,6 +217,10 @@ class GlobalConfiguration(ConfigurationBase): default_factory=FeedbackConfiguration ) + pipelines: PipelineConfiguration = pydantic.Field( + default_factory=PipelineConfiguration + ) + # All subclasses of ConfigurationBase are automatically registered using this dict. NAME_TO_MODEL_TYPE_MAPPING: dict[str, t.Type[ConfigurationBase]] = { diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 2bbcac0642..d5ab33f627 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -46,6 +46,7 @@ dependencies = [ "lxml", "valkey[libvalkey]", "cryptography", + "croniter", ] [project.urls] @@ -69,6 +70,7 @@ dev = [ "pytest-cov", "aioresponses", "types-lxml", + "types-croniter", ] [tool.black] @@ -223,7 +225,11 @@ extension-pkg-whitelist = "pydantic" # https://github.com/pydantic/pydantic/issu [tool.pylint.master] init-import = "yes" -load-plugins = ["pylint.extensions.bad_builtin", "pylint.extensions.mccabe", "pylint_pytest"] +load-plugins = [ + "pylint.extensions.bad_builtin", + "pylint.extensions.mccabe", + "pylint_pytest", +] extension-pkg-allow-list = ["lxml.etree"] [tool.pylint.similarities] diff --git a/backend/tests/settings/test_global_configuration.py b/backend/tests/settings/test_global_configuration.py index 8604f84af4..baa199d66f 100644 --- a/backend/tests/settings/test_global_configuration.py +++ b/backend/tests/settings/test_global_configuration.py @@ -161,3 +161,21 @@ def test_navbar_is_updated( "href": "https://example.com", "role": "user", } + + +@pytest.mark.usefixtures("admin") +def test_global_configuration_invalid_pipelines( + client: testclient.TestClient, +): + response = client.put( + "/api/v1/settings/configurations/global", + json={"pipelines": {"cron": "invalid", "timezone": "Berlin"}}, + ) + + assert response.status_code == 422 + + detail = response.json()["detail"] + assert detail[0]["type"] == "value_error" + assert detail[0]["loc"] == ["body", "pipelines", "cron"] + assert detail[1]["type"] == "value_error" + assert detail[1]["loc"] == ["body", "pipelines", "timezone"] diff --git a/frontend/src/app/openapi/.openapi-generator/FILES b/frontend/src/app/openapi/.openapi-generator/FILES index 3001204963..a55073549f 100644 --- a/frontend/src/app/openapi/.openapi-generator/FILES +++ b/frontend/src/app/openapi/.openapi-generator/FILES @@ -112,6 +112,8 @@ model/persistent-session-tool-configuration-input.ts model/persistent-session-tool-configuration-output.ts model/persistent-workspace-session-configuration-input.ts model/persistent-workspace-session-configuration-output.ts +model/pipeline-configuration-input.ts +model/pipeline-configuration-output.ts model/pipeline-run-status.ts model/pipeline-run.ts model/post-git-instance.ts diff --git a/frontend/src/app/openapi/model/global-configuration-input.ts b/frontend/src/app/openapi/model/global-configuration-input.ts index 6c568c5958..8306d95b04 100644 --- a/frontend/src/app/openapi/model/global-configuration-input.ts +++ b/frontend/src/app/openapi/model/global-configuration-input.ts @@ -11,6 +11,7 @@ import { NavbarConfigurationInput } from './navbar-configuration-input'; import { MetadataConfigurationInput } from './metadata-configuration-input'; +import { PipelineConfigurationInput } from './pipeline-configuration-input'; import { FeedbackConfigurationInput } from './feedback-configuration-input'; @@ -21,5 +22,6 @@ export interface GlobalConfigurationInput { metadata?: MetadataConfigurationInput; navbar?: NavbarConfigurationInput; feedback?: FeedbackConfigurationInput; + pipelines?: PipelineConfigurationInput; } diff --git a/frontend/src/app/openapi/model/global-configuration-output.ts b/frontend/src/app/openapi/model/global-configuration-output.ts index c52b7fa677..95c90653a8 100644 --- a/frontend/src/app/openapi/model/global-configuration-output.ts +++ b/frontend/src/app/openapi/model/global-configuration-output.ts @@ -11,6 +11,7 @@ import { NavbarConfigurationOutput } from './navbar-configuration-output'; import { MetadataConfigurationOutput } from './metadata-configuration-output'; +import { PipelineConfigurationOutput } from './pipeline-configuration-output'; import { FeedbackConfigurationOutput } from './feedback-configuration-output'; @@ -21,5 +22,6 @@ export interface GlobalConfigurationOutput { metadata: MetadataConfigurationOutput; navbar: NavbarConfigurationOutput; feedback: FeedbackConfigurationOutput; + pipelines: PipelineConfigurationOutput; } diff --git a/frontend/src/app/openapi/model/models.ts b/frontend/src/app/openapi/model/models.ts index 645353a7e9..402c61cc6b 100644 --- a/frontend/src/app/openapi/model/models.ts +++ b/frontend/src/app/openapi/model/models.ts @@ -90,6 +90,8 @@ export * from './persistent-session-tool-configuration-input'; export * from './persistent-session-tool-configuration-output'; export * from './persistent-workspace-session-configuration-input'; export * from './persistent-workspace-session-configuration-output'; +export * from './pipeline-configuration-input'; +export * from './pipeline-configuration-output'; export * from './pipeline-run'; export * from './pipeline-run-status'; export * from './post-git-instance'; diff --git a/frontend/src/app/openapi/model/pipeline-configuration-input.ts b/frontend/src/app/openapi/model/pipeline-configuration-input.ts new file mode 100644 index 0000000000..73861f8611 --- /dev/null +++ b/frontend/src/app/openapi/model/pipeline-configuration-input.ts @@ -0,0 +1,24 @@ +/* + * SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Capella Collaboration + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * Do not edit the class manually. + + To generate a new version, run `make openapi` in the root directory of this repository. + */ + + + +export interface PipelineConfigurationInput { + /** + * Cron for nightly backup. Only applies to newly created pipelines. + */ + cron?: string; + /** + * Timezone for the cron expression. + */ + timezone?: string; +} + diff --git a/frontend/src/app/openapi/model/pipeline-configuration-output.ts b/frontend/src/app/openapi/model/pipeline-configuration-output.ts new file mode 100644 index 0000000000..b8e27fcaf7 --- /dev/null +++ b/frontend/src/app/openapi/model/pipeline-configuration-output.ts @@ -0,0 +1,24 @@ +/* + * SPDX-FileCopyrightText: Copyright DB InfraGO AG and contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Capella Collaboration + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * Do not edit the class manually. + + To generate a new version, run `make openapi` in the root directory of this repository. + */ + + + +export interface PipelineConfigurationOutput { + /** + * Cron for nightly backup. Only applies to newly created pipelines. + */ + cron: string; + /** + * Timezone for the cron expression. + */ + timezone: string; +} +