diff --git a/README.md b/README.md index b7d038f..ddf8960 100644 --- a/README.md +++ b/README.md @@ -186,8 +186,23 @@ vertex #### Pipelines -You file `{pipeline_name}.py` must contain a function called `pipeline` decorated using `kfp.dsl.pipeline`. - +You file `{pipeline_name}.py` must contain a function called `{pipeline_name}` decorated using `kfp.dsl.pipeline`. +In previous versions, the functions / object used to be called `pipeline` but it was changed to `{pipeline_name}` to avoid confusion with the `kfp.dsl.pipeline` decorator. + +```python +# vertex/pipelines/dummy_pipeline.py +import kfp.dsl + +# New name to avoid confusion with the kfp.dsl.pipeline decorator +@kfp.dsl.pipeline() +def dummy_pipeline(): + ... + +# Old name +@kfp.dsl.pipeline() +def pipeline(): + ... +``` #### Configs diff --git a/deployer/cli.py b/deployer/cli.py index 6720957..7def058 100644 --- a/deployer/cli.py +++ b/deployer/cli.py @@ -272,7 +272,7 @@ def check( Checking that a pipeline is valid includes: * Checking that the pipeline can be imported. It must be a valid python module with a - `pipeline` function decorated with `@kfp.dsl.pipeline`. + `{pipeline_name}` function decorated with `@kfp.dsl.pipeline`. * Checking that the pipeline can be compiled using `kfp.compiler.Compiler`. diff --git a/deployer/pipeline_checks.py b/deployer/pipeline_checks.py index bcd0f89..9352d21 100644 --- a/deployer/pipeline_checks.py +++ b/deployer/pipeline_checks.py @@ -70,8 +70,12 @@ def populate_config_names(cls, data: Any) -> Any: @computed_field def pipeline(self) -> Any: """Import pipeline""" - with disable_logger("deployer.utils.utils"): - return import_pipeline_from_dir(PIPELINE_ROOT_PATH, self.pipeline_name.value) + if getattr(self, "_pipeline", None) is None: + with disable_logger("deployer.utils.utils"): + self._pipeline = import_pipeline_from_dir( + PIPELINE_ROOT_PATH, self.pipeline_name.value + ) + return self._pipeline @model_validator(mode="after") def import_pipeline(self): diff --git a/deployer/utils/utils.py b/deployer/utils/utils.py index 511a867..1879307 100644 --- a/deployer/utils/utils.py +++ b/deployer/utils/utils.py @@ -1,4 +1,5 @@ import importlib +import warnings from enum import Enum from pathlib import Path from typing import Dict, Optional @@ -40,11 +41,22 @@ def import_pipeline_from_dir(dirpath: Path, pipeline_name: str) -> graph_compone ) from e try: - pipeline: Optional[graph_component.GraphComponent] = pipeline_module.pipeline + pipeline: Optional[graph_component.GraphComponent] + pipeline = getattr(pipeline_module, pipeline_name, None) + if pipeline is None: + pipeline = pipeline_module.pipeline + warnings.warn( + f"Pipeline in `{module_path}` is named `pipeline` instead of `{pipeline_name}`. " + "This is deprecated and will be removed in a future version. " + f"Please rename your pipeline to `{pipeline_name}`.", + FutureWarning, + stacklevel=1, + ) except AttributeError as e: raise ImportError( - f"Pipeline {module_path}:pipeline not found. " + f"Pipeline object not found in `{module_path}`. " "Please check that the pipeline is correctly defined and named." + f"It should be named `{pipeline_name}` or `pipeline` (deprecated)." ) from e logger.debug(f"Pipeline {module_path} imported successfully.") diff --git a/example/vertex/pipelines/broken_pipeline.py b/example/vertex/pipelines/broken_pipeline.py index 28d7e8d..5ccba5b 100644 --- a/example/vertex/pipelines/broken_pipeline.py +++ b/example/vertex/pipelines/broken_pipeline.py @@ -4,6 +4,6 @@ @kfp.dsl.pipeline(name="broken-pipeline") -def pipeline(name: str): +def broken_pipeline(name: str): """This pipeline is broken!""" broken_component(name=name) diff --git a/example/vertex/pipelines/dummy_pipeline.py b/example/vertex/pipelines/dummy_pipeline.py index c150c05..802d7ea 100644 --- a/example/vertex/pipelines/dummy_pipeline.py +++ b/example/vertex/pipelines/dummy_pipeline.py @@ -4,6 +4,6 @@ @kfp.dsl.pipeline(name="dummy-pipeline") -def pipeline(name: str): +def dummy_pipeline(name: str): """This pipeline prints hello {name}""" dummy_component(name=name)