chore(release): 0.1.0 (#55)

.github/workflows/ci.yaml

@@ -18,9 +18,9 @@ jobs:
         python-version: ['3.8', '3.9', '3.10']
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -82,6 +82,11 @@ jobs:
           poetry run pip install --upgrade pip
           poetry install
 
+      - name: Test root command
+        run: |
+          cd example
+          poetry run vertex-deployer --version
+
       - name: Test list command
         run: |
           cd example

.github/workflows/release.yaml

@@ -1,3 +1,15 @@
+# This workflow triggers the CI, updates the version, and uploads the release to GitHub and Google Cloud Storage when a push is made to either the 'main' or 'develop' branch.
+#
+# Workflow Steps:
+#
+# 1. Ci is triggered using the CI workflow defined in .github/workflows/ci.yaml
+# 2. If it succeeds, the version is updated using Python Semantic Release
+# 3. The release is uploaded to GitHub (same step and GitHub action)
+# 5. Authentication to Google Cloud is achieved using Workload Identity Federation
+# 6. The release is uploaded to Google Cloud Storage
+#
+# For more details on setting up Workload Identity Federation for GitHub, visit https://github.com/google-github-actions/auth#setting-up-workload-identity-federation
+
 name: CI and Release on main
 
 on:
@@ -26,3 +38,20 @@ jobs:
       uses: python-semantic-release/python-semantic-release@master
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: 'Authenticate to Google Cloud'
+      uses: 'google-github-actions/auth@v1'
+      with:
+        token_format: 'access_token'
+        workload_identity_provider: '${{ secrets.WIF_PROVIDER }}' # e.g. - projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider
+        service_account: '${{ secrets.WIF_SERVICE_ACCOUNT }}' # e.g. - [email protected]
+
+    - name: Copy release to root
+      run: cp -r dist/*.tar.gz .
+
+    - name: 'Upload Release to Google Cloud Storage'
+      uses: 'google-github-actions/upload-cloud-storage@v1'
+      with:
+        path: '.'
+        destination: vertex-pipelines-deployer
+        glob: '*.tar.gz'

README.md

@@ -23,14 +23,12 @@
 </div>
 
 
-> **Warning**
-> This is a work in progress and is not ready for production use.
-
-
 ## Table of Contents
 - [Why this tool?](#why-this-tool)
 - [Prerequisites](#prerequisites)
 - [Installation](#installation)
+    - [From git repo](#from-git-repo)
+    - [From GCS (not available in PyPI yet)](#from-gcs-not-available-in-pypi-yet)
 - [Usage](#usage)
   - [Setup](#setup)
   - [Folder Structure](#folder-structure)
@@ -52,6 +50,8 @@ Two uses cases:
 Commands:
 - `check`: check your pipelines (imports, compile, check configs validity against pipeline definition).
 - `deploy`: compile, upload to Artifact Registry, run and schedule your pipelines.
+- `create`: create a new pipeline and config files.
+- `list`: list all pipelines in the `vertex/pipelines` folder.
 
 ## Prerequisites
 
@@ -62,6 +62,15 @@ Commands:
 
 ## Installation
 
+
+### From git repo
+
+Stable version:
+```bash
+pip install git+https://github.com/artefactory/vertex-pipelines-deployer.git@main
+```
+
+Develop version:
 ```bash
 pip install git+https://github.com/artefactory/vertex-pipelines-deployer.git@develop
 ```
@@ -73,6 +82,20 @@ poetry install
 cd example
 ```
 
+### From GCS (not available in PyPI yet)
+
+Install a specific version:
+```bash
+export VERSION=0.0.1
+wget https://storage.cloud.google.com/vertex-pipelines-deployer/vertex_deployer-$VERSION.tar.gz
+pip install ./vertex_deployer-$VERSION.tar.gz
+```
+
+List available versions:
+```bash
+gsutil ls gs://vertex-pipelines-deployer
+```
+
 ## Usage
 
 ### Setup
@@ -150,13 +173,17 @@ You must respect the following folder structure. If you already follow the
 
 ```
 vertex
-├─ config/
+├─ configs/
 │  └─ {pipeline_name}
 │     └─ {config_name}.json
 └─ pipelines/
    └─ {pipeline_name}.py
 ```
 
+> [!NOTE]
+> You must have at lease these files. If you need to share some config elements between pipelines,
+> you can have a `shared` folder in `configs` and import them in your pipeline configs.
+
 #### Pipelines
 
 You file `{pipeline_name}.py` must contain a function called `pipeline` decorated using `kfp.dsl.pipeline`.
@@ -201,7 +228,7 @@ VERTEX_SERVICE_ACCOUNT=YOUR_VERTEX_SERVICE_ACCOUNT  # Vertex Pipelines Service A
 > **Note**
 > We're using env files and dotenv to load the environment variables.
 > No default value for `--env-file` argument is provided to ensure that you don't accidentally deploy to the wrong project.
-> An [`example.env`](example/example.env) file is provided in this repo.
+> An [`example.env`](./example/example.env) file is provided in this repo.
 > This also allows you to work with multiple environments thanks to env files (`test.env`, `dev.env`, `prod.env`, etc)
 
 ### CLI: Deploying a Pipeline