From e284168af39a914ddf4b2883ccc6031cb322ada2 Mon Sep 17 00:00:00 2001 From: Jules Bertrand <33326907+julesbertrand@users.noreply.github.com> Date: Thu, 5 Oct 2023 16:22:15 +0200 Subject: [PATCH 1/3] docs: update installation guidelines (#61) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bea4293..b7d038f 100644 --- a/README.md +++ b/README.md @@ -86,8 +86,8 @@ cd example Install a specific version: ```bash -export VERSION=0.0.1 -wget https://storage.cloud.google.com/vertex-pipelines-deployer/vertex_deployer-$VERSION.tar.gz +export VERSION=0.1.0 +gsutil -m cp gs://vertex-pipelines-deployer/vertex_deployer-$VERSION.tar.gz . pip install ./vertex_deployer-$VERSION.tar.gz ``` From f005f4418a011ed1dbb70edc3579c0735394c97b Mon Sep 17 00:00:00 2001 From: Jules Bertrand <33326907+julesbertrand@users.noreply.github.com> Date: Thu, 5 Oct 2023 16:22:38 +0200 Subject: [PATCH 2/3] fix: checks temp directory removal (#62) --- deployer/pipeline_checks.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/deployer/pipeline_checks.py b/deployer/pipeline_checks.py index b90da97..b6a55b6 100644 --- a/deployer/pipeline_checks.py +++ b/deployer/pipeline_checks.py @@ -3,7 +3,8 @@ from typing import Any, Dict, Generic, List, TypeVar from loguru import logger -from pydantic import Field, computed_field, model_validator +from pydantic import Field, ValidationError, computed_field, model_validator +from pydantic.functional_validators import ModelWrapValidatorHandler from typing_extensions import Annotated from deployer.constants import ( @@ -100,15 +101,16 @@ class Pipelines(CustomBaseModel): pipelines: Dict[str, Pipeline] - @model_validator(mode="before") - @classmethod - def _init_temp_directory(cls, data: Any) -> Any: - """Create temporary directory""" + @model_validator(mode="wrap") + def _init_remove_temp_directory(self, handler: ModelWrapValidatorHandler) -> Any: + """Create and remove temporary directory""" Path(TEMP_LOCAL_PACKAGE_PATH).mkdir(exist_ok=True) - return data - @model_validator(mode="after") - def _remove_temp_directory(self) -> None: - """Remove temporary directory""" - shutil.rmtree(TEMP_LOCAL_PACKAGE_PATH) - return self + try: + validated_self = handler(self) + except ValidationError as e: + raise e + finally: + shutil.rmtree(TEMP_LOCAL_PACKAGE_PATH) + + return validated_self From 9c973f0fe51d6fd5a91fe7df190a363f57fc9944 Mon Sep 17 00:00:00 2001 From: Jules Bertrand <33326907+julesbertrand@users.noreply.github.com> Date: Thu, 5 Oct 2023 16:26:24 +0200 Subject: [PATCH 3/3] fix: make imports in cli commands to reduce overhead (#63) --- Makefile | 8 ++++++++ deployer/cli.py | 7 +++++-- pyproject.toml | 4 ++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4e0f4d8..b2d66a2 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,14 @@ run-tests: @poetry run pytest tests --cov=deployer --cov-report=term-missing -s -vv -W ignore:::pkg_resources +.PHONY: profile-cli +## Profile CLI using pyinstrument (https://pyinstrument.readthedocs.io/en/latest/index.html) +profile-cli: + @echo "Check that you have pyinstrument installed: poetry install -E profiling" + @poetry run pyinstrument -r html -o pyinstrument.html --from-path vertex-deployer --version + @open pyinstrument.html + + ################################################################################# # Self Documenting Commands # ################################################################################# diff --git a/deployer/cli.py b/deployer/cli.py index e37b265..6720957 100644 --- a/deployer/cli.py +++ b/deployer/cli.py @@ -15,8 +15,6 @@ PIPELINE_ROOT_PATH, PYTHON_CONFIG_TEMPLATE, ) -from deployer.pipeline_checks import Pipelines -from deployer.pipeline_deployer import VertexPipelineDeployer from deployer.utils.config import ( ConfigType, list_config_filepaths, @@ -190,6 +188,8 @@ def deploy( # noqa: C901 pipeline_func = import_pipeline_from_dir(PIPELINE_ROOT_PATH, pipeline_name.value) + from deployer.pipeline_deployer import VertexPipelineDeployer + deployer = VertexPipelineDeployer( project_id=vertex_settings.PROJECT_ID, region=vertex_settings.GCP_REGION, @@ -283,6 +283,8 @@ def check( **This command can be used to check pipelines in a Continuous Integration workflow.** """ + from deployer.pipeline_checks import Pipelines + if len(PipelineName.__members__) == 0: raise ValueError( "No pipeline found. Please check that the pipeline root path is correct" @@ -339,6 +341,7 @@ def list( "No pipeline found. Please check that the pipeline root path is" f" correct (current: '{PIPELINE_ROOT_PATH}')" ) + raise typer.Exit() if with_configs: pipelines_dict = { diff --git a/pyproject.toml b/pyproject.toml index 859a543..9d7c425 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ rich = "^13.5.3" loguru = "^0.7.2" pydantic-settings = "^2.0.3" pydantic = "^2.3.0" +pyinstrument = { version = "^4.5.3", optional = true } [tool.poetry.group.dev.dependencies] black = "^23.7.0" @@ -30,6 +31,9 @@ nbstripout = "^0.6.1" ruff = "^0.0.289" pytest-cov = "^4.1.0" +[tool.poetry.extras] +profiling = ["pyinstrument"] + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api"