chore(release): 0.4.4 (#175)

.github/workflows/release.yaml

@@ -35,6 +35,9 @@ jobs:
     permissions:
       id-token: write
       contents: write
+    environment:
+      name: pypi
+      url: https://pypi.org/p/vertex-deployer
 
     steps:
     - uses: actions/checkout@v4
@@ -48,20 +51,26 @@ jobs:
       with:
         github_token: ${{ secrets.RELEASE_ACCESS_TOKEN }}
 
+    # Publish to Pypi
+    - name: Publish package distributions to PyPI
+      if: steps.python-semantic-release.outputs.released == 'true'
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+    # Publish to Google Artifact Registry
     - name: 'Authenticate to Google Cloud'
-      if: ${{ steps.python-semantic-release.outputs.released == 'true' }}
+      if: steps.python-semantic-release.outputs.released == 'true'
       uses: 'google-github-actions/auth@v2'
       with:
         token_format: 'access_token'
         workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' # e.g. - projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider
         service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' # e.g. - [email protected]
 
     - name: Install twine
-      if: ${{ steps.python-semantic-release.outputs.released == 'true' }}
+      if: steps.python-semantic-release.outputs.released == 'true'
       run: pip install twine keyrings.google-artifactregistry-auth
 
     - name: Upload dist to Google Artifact Registry
-      if: ${{ steps.python-semantic-release.outputs.released == 'true' }}
+      if: steps.python-semantic-release.outputs.released == 'true'
       run: |
         python3 -m twine upload \
           --repository-url https://europe-west1-python.pkg.dev/data-sandbox-fr/artefactory \

.pre-commit-config.yaml

@@ -45,3 +45,9 @@ repos:
       types: [markdown]
       language: system
       stages: [commit, push]
+    - id: check-cli-documentation-updated
+      name: check cli documentation updated
+      entry: poetry run typer deployer/cli.py utils docs --output docs/CLI_REFERENCE.md --name vertex-deployer
+      pass_filenames: false
+      language: system
+      stages: [push]

.skaff/skaff.yaml

@@ -0,0 +1,28 @@
+spec_version: 1
+
+# Everyting in this file will be used to populate the catalog on skaff.artefact.com
+# It is also vectorized as is for the search, so make sure to provide relevant keywords
+
+name: Vertex Pipelines Deployer
+owner: [email protected]
+description: >
+  Check, compile, upload, run and schedule Vertex Pipelines in a standardized manner.
+documentation_url: https://supreme-funicular-mzylwj5.pages.github.io/
+
+type: deployable   # deployable, knowldege pack
+lifecycle: production  # prototype, production
+
+users:                 # DS, DA, DE
+  - DS
+  - DE
+
+clouds:                # aws, gcp, azure, databricks
+  - gcp
+
+technologies:          # python, dbt, terraform, airbyte, ...
+  - vertex
+  - python
+
+expertises:            # MMM, forecasting, ELT, release engineering, ...
+  - MLEng
+  - MLOps

Makefile

@@ -46,7 +46,7 @@ update-requirements:
 .PHONY: format-code
 ## Format/lint all-files using pre-commit hooks (black, flake8, isort, ...)
 format-code:
-	@poetry run pre-commit run -a
+	@poetry run pre-commit run -a --hook-stage pre-push
 
 
 .PHONY: run-unit-tests

README.md

@@ -1,6 +1,11 @@
 <br />
 <div align="center">
     <h1 align="center">Vertex Pipelines Deployer</h1>
+    <p align="center">
+        <a href="https://www.artefact.com/">
+        <img src="docs/assets/Artefact-AI-is-about-People-logo-1200x424.png" style="max-width:50%;height:auto;background-color:#111146;" alt="Artefact Logo"/>
+        </a>
+    </p>
     <h3 align="center">Deploy Vertex Pipelines within minutes</h3>
         <p align="center">
         This tool is a wrapper around <a href="https://www.kubeflow.org/docs/components/pipelines/v2/hello-world/">kfp</a> and <a href="https://cloud.google.com/python/docs/reference/aiplatform/latest">google-cloud-aiplatform</a> that allows you to check, compile, upload, run, and schedule Vertex Pipelines in a standardized manner.
@@ -57,9 +62,9 @@
 
 [Full CLI documentation](docs/CLI_REFERENCE.md)
 
-<!-- --8<-- [start:why] -->
-## ❓ Why this tool?
 
+## ❓ Why this tool?
+<!-- --8<-- [start:why] -->
 
 Three use cases:
 
@@ -86,6 +91,11 @@ Two main commands:
 
 ## 📦 Installation
 <!-- --8<-- [start:installation] -->
+### From PyPI
+
+```bash
+pip install vertex-deployer
+```
 
 ### From git repo
 
@@ -105,32 +115,6 @@ git clone [email protected]:artefactory/vertex-pipelines-deployer.git
 poetry install
 cd example
 ```
-
-### From Artifact Registry (not available in PyPI yet)
-
-The package is available on a public Google Artifact Registry repo. You need to specify a
-[pip extra index url](https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-extra-index-url) to install it.
-
-Install latest version:
-```bash
-pip install --extra-index-url https://europe-west1-python.pkg.dev/data-sandbox-fr/artefactory/simple vertex-deployer
-```
-
-!!! tip "Add to requirements"
-    You can add the extra index URL to your `requirements.in` or `requirements.txt` file.
-    ```txt title="requirements.txt"
-    --extra-index-url https://europe-west1-python.pkg.dev/data-sandbox-fr/artefactory/simple
-
-    vertex-deployer==0.3.3
-    ```
-
-??? info "About extra index URL"
-    You can add the extra index URL to your `pip.conf` file to avoid having to specify it every time.
-
-List available versions:
-```bash
-pip index versions --extra-index-url https://europe-west1-python.pkg.dev/data-sandbox-fr/artefactory/simple vertex-deployer
-```
 <!-- --8<-- [end:installation] -->
 
 ## 🚀 Usage

deployer/cli.py

@@ -83,7 +83,7 @@ def main(
 def pipeline_name_callback(ctx: typer.Context, value: Union[str, bool]) -> Union[str, bool]:
     """Callback to check that the pipeline name is valid."""
     if value is None:  # None is allowed for optional arguments
-        return value
+        return []
 
     pipeline_names: enum.Enum = ctx.obj["pipeline_names"]
 
@@ -203,11 +203,17 @@ def deploy(  # noqa: C901
         ),
     ] = None,
     enable_caching: Annotated[
-        bool,
+        Optional[bool],
         typer.Option(
-            "--enable-caching", "-ec", help="Whether to enable caching when running the pipeline."
+            "--enable-caching / --no-cache",
+            "-ec / -nec",
+            help="Whether to turn on caching for the run."
+            "If this is not set, defaults to the compile time settings, which are True for all"
+            "tasks by default, while users may specify different caching options for individual"
+            "tasks. If this is set, the setting applies to all tasks in the pipeline."
+            "Overrides the compile time settings. Defaults to None.",
         ),
-    ] = False,
+    ] = None,
     experiment_name: Annotated[
         Optional[str],
         typer.Option(
@@ -327,7 +333,7 @@ def check(
     pipeline_names: Annotated[
         Optional[List[str]],
         typer.Argument(
-            ..., help="The names of the pipeline to run.", callback=pipeline_name_callback
+            ..., help="The names of the pipeline to check.", callback=pipeline_name_callback
         ),
     ] = None,
     all: Annotated[

deployer/pipeline_deployer.py

@@ -110,10 +110,34 @@ def _check_experiment_name(self, experiment_name: Optional[str] = None) -> str:
     def _create_pipeline_job(
         self,
         template_path: str,
-        enable_caching: bool = False,
+        enable_caching: Optional[bool] = None,
         parameter_values: Optional[dict] = None,
         input_artifacts: Optional[dict] = None,
     ) -> aiplatform.PipelineJob:
+        """Create a pipeline job object
+
+        Args:
+            template_path (str): The path of PipelineJob or PipelineSpec JSON or YAML file. If the
+                Artifact Registry host is provided, this is the path to the pipeline template in
+                the Artifact Registry. Otherwise, this is the path to the pipeline template in
+                the local package.
+            enable_caching (Optional[bool], optional): Whether to turn on caching for the run.
+                If this is not set, defaults to the compile time settings, which are True for all
+                tasks by default, while users may specify different caching options for individual
+                tasks.
+                If this is set, the setting applies to all tasks in the pipeline.
+                Overrides the compile time settings. Defaults to None.
+            parameter_values (Optional[dict], optional): The mapping from runtime parameter names
+                to its values that control the pipeline run. Defaults to None.
+            input_artifacts (Optional[dict], optional): The mapping from the runtime parameter
+                name for this artifact to its resource id.
+                For example: "vertex_model":"456".
+                Note: full resource name ("projects/123/locations/us-central1/metadataStores/default/artifacts/456")
+                    cannot be used. Defaults to None.
+
+        Returns:
+            aiplatform.PipelineJob: The pipeline job object
+        """  # noqa: E501
         job = aiplatform.PipelineJob(
             display_name=self.pipeline_name,
             template_path=template_path,
@@ -156,7 +180,7 @@ def upload_to_registry(
 
     def run(
         self,
-        enable_caching: bool = False,
+        enable_caching: Optional[bool] = None,
         parameter_values: Optional[dict] = None,
         input_artifacts: Optional[dict] = None,
         experiment_name: Optional[str] = None,
@@ -170,16 +194,33 @@ def run(
         provided. Otherwise, use the pipeline file in the local package.
 
         Args:
-            enable_caching (bool, optional): Whether to enable caching. Defaults to False.
-            parameter_values (dict, optional): Pipeline parameter values. Defaults to None.
-            input_artifacts (dict, optional): Pipeline input artifacts. Defaults to None.
+            enable_caching (Optional[bool], optional): Whether to turn on caching for the run.
+                If this is not set, defaults to the compile time settings, which are True for all
+                tasks by default, while users may specify different caching options for individual
+                tasks.
+                If this is set, the setting applies to all tasks in the pipeline.
+                Overrides the compile time settings. Defaults to None.
+            parameter_values (Optional[dict], optional): The mapping from runtime parameter names
+                to its values that control the pipeline run. Defaults to None.
+            input_artifacts (Optional[dict], optional): The mapping from the runtime parameter
+                name for this artifact to its resource id.
+                For example: "vertex_model":"456".
+                Note: full resource name ("projects/123/locations/us-central1/metadataStores/default/artifacts/456")
+                    cannot be used. Defaults to None.
             experiment_name (str, optional): Experiment name. Defaults to None.
             tag (str, optional): Tag of the pipeline template. Defaults to None.
-        """
+        """  # noqa: E501
         experiment_name = self._check_experiment_name(experiment_name)
 
         template_path = self._get_template_path(tag)
 
+        logger.debug(
+            f"Running pipeline '{self.pipeline_name}' with settings:"
+            f"\n {'template_path':<20} {template_path:<30}"
+            f"\n {'enable_caching':<20} {enable_caching!s:<30}"
+            f"\n {'experiment_name':<20} {experiment_name:<30}"
+        )
+
         job = self._create_pipeline_job(
             template_path=template_path,
             enable_caching=enable_caching,
@@ -207,7 +248,7 @@ def run(
 
     def compile_upload_run(
         self,
-        enable_caching: bool = False,
+        enable_caching: Optional[bool] = None,
         parameter_values: Optional[dict] = None,
         experiment_name: Optional[str] = None,
         tags: Optional[List[str]] = None,
@@ -229,7 +270,7 @@ def compile_upload_run(
     def schedule(
         self,
         cron: str,
-        enable_caching: bool = False,
+        enable_caching: Optional[bool] = None,
         parameter_values: Optional[dict] = None,
         tag: Optional[str] = None,
         delete_last_schedule: bool = False,

deployer/settings.py

@@ -8,7 +8,7 @@
 from pydantic import ValidationError
 from tomlkit.toml_file import TOMLFile
 
-from deployer import constants
+from deployer import __version__, constants
 from deployer.utils.config import ConfigType
 from deployer.utils.exceptions import InvalidPyProjectTOMLError
 from deployer.utils.models import CustomBaseModel
@@ -28,7 +28,7 @@ class _DeployerDeploySettings(CustomBaseModel):
     tags: Optional[List[str]] = constants.DEFAULT_TAGS
     config_filepath: Optional[Path] = None
     config_name: Optional[str] = None
-    enable_caching: bool = False
+    enable_caching: Optional[bool] = None
     experiment_name: Optional[str] = None
     local_package_path: Path = constants.DEFAULT_LOCAL_PACKAGE_PATH
     skip_validation: bool = True
@@ -136,8 +136,9 @@ def load_deployer_settings() -> DeployerSettings:
     try:
         settings = DeployerSettings(**settings)
     except ValidationError as e:
-        msg = f"In {path_pyproject_toml}:\n{e}\n"
-        msg += "Please check your configuration file."
+        msg = f"Invalid section tools.vertex_deployer:\n{e}\n"
+        msg += f"\nPlease check your configuration file: {path_pyproject_toml}"
+        msg += f" and check settings are compatible with deployer version (current: {__version__})"
 
         raise InvalidPyProjectTOMLError(msg) from e
 

deployer/utils/utils.py

@@ -37,8 +37,7 @@ class GraphComponentType(Protocol):
 
     def __call__(  # noqa: D102
         self, component_spec: Any, pipeline_func: Callable, display_name: Optional[str] = None
-    ):
-        ...
+    ): ...
 
 
 def filter_lines_from(tb: TracebackType, target_file: Union[Path, str]) -> str: