diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e33a21c..9c5b5e1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -28,7 +28,7 @@ jobs: run: make download-poetry - name: Set up pip cache - uses: actions/cache@v4.0.0 + uses: actions/cache@v4.1.1 with: path: ~/.cache/pypoetry/virtualenvs key: venv-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }} @@ -67,7 +67,7 @@ jobs: run: make download-poetry - name: Set up pip cache - uses: actions/cache@v4.0.0 + uses: actions/cache@v4.1.1 with: path: ~/.cache/pypoetry/virtualenvs key: venv-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }} diff --git a/.github/workflows/deploy_docs.yaml b/.github/workflows/deploy_docs.yaml index 1924aaf..8424064 100644 --- a/.github/workflows/deploy_docs.yaml +++ b/.github/workflows/deploy_docs.yaml @@ -25,7 +25,7 @@ jobs: run: make download-poetry - name: Set up pip cache - uses: actions/cache@v4.0.0 + uses: actions/cache@v4.1.1 with: path: ~/.cache/pypoetry/virtualenvs key: venv-${{ env.python-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('poetry.lock') }} diff --git a/.github/workflows/pr_agent.yaml b/.github/workflows/pr_agent.yaml index 25aa042..6f8ea83 100644 --- a/.github/workflows/pr_agent.yaml +++ b/.github/workflows/pr_agent.yaml @@ -15,7 +15,7 @@ jobs: steps: - name: PR Agent action step id: pragent - uses: Codium-ai/pr-agent@main + uses: docker://codiumai/pr-agent:0.24-github_action env: OPENAI_KEY: ${{ secrets.AZURE_API_KEY }} OPENAI.API_TYPE: "azure" @@ -23,3 +23,6 @@ jobs: OPENAI.API_BASE: ${{ secrets.AZURE_API_BASE }} OPENAI.API_VERSION: ${{ secrets.AZURE_API_VERSION }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_ACTION_CONFIG.AUTO_DESCRIBE: false + GITHUB_ACTION_CONFIG.AUTO_REVIEW: true + GITHUB_ACTION_CONFIG.AUTO_IMPROVE: true diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 435c33d..29528db 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,18 +1,3 @@ -# This workflow triggers the CI, updates the version, and uploads the release to GitHub and -# Google Artifact Registry when a push is made to either the 'main' or 'develop' branch. -# -# Workflow Steps: -# -# 1. Ci is triggered using the CI workflow defined in .github/workflows/ci.yaml -# 2. If it succeeds, the version is updated using Python Semantic Release -# 3. The release is uploaded to GitHub (same step and GitHub action) -# 5. Authentication to Google Cloud is achieved using Workload Identity Federation -# 6. The release is uploaded to Google Artifact Registry -# -# For more details on setting up Workload Identity Federation for GitHub, -# visit https://github.com/google-github-actions/auth#setting-up-workload-identity-federation -# WIF service account must have roles/artifactregistry.write access to the Artifact Registry repository - name: CI and Release on main on: @@ -47,7 +32,7 @@ jobs: - name: Python Semantic Release id: python-semantic-release - uses: python-semantic-release/python-semantic-release@v9.6.0 # need python >=3.12 to use latest versions + uses: python-semantic-release/python-semantic-release@v9.11.0 with: github_token: ${{ secrets.RELEASE_ACCESS_TOKEN }} diff --git a/README.md b/README.md index 725a3f1..7763d6b 100644 --- a/README.md +++ b/README.md @@ -415,11 +415,11 @@ You can configure the deployer using the `pyproject.toml` file to better fit you This will overwrite default values. It can be useful if you always use the same options, e.g. always the same `--scheduler-timezone` ```toml -[tool.vertex-deployer] +[tool.vertex_deployer] vertex_folder_path = "my/path/to/vertex" log_level = "INFO" -[tool.vertex-deployer.deploy] +[tool.vertex_deployer.deploy] scheduler_timezone = "Europe/Paris" ``` diff --git a/deployer/pipeline_checks.py b/deployer/pipeline_checks.py index 60d693d..9d14692 100644 --- a/deployer/pipeline_checks.py +++ b/deployer/pipeline_checks.py @@ -146,7 +146,7 @@ def _convert_artifact_type_to_str(annotation: type) -> type: """Convert a kfp.dsl.Artifact type to a string. This is mandatory for type checking, as kfp.dsl.Artifact types should be passed as strings - to VertexAI. See https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform.PipelineJob + to VertexAI. See for details. """ if isinstance(annotation, _AnnotatedAlias): diff --git a/docs/advanced_user_guide.md b/docs/advanced/advanced_user_guide.md similarity index 100% rename from docs/advanced_user_guide.md rename to docs/advanced/advanced_user_guide.md diff --git a/docs/configuration.md b/docs/advanced/configuration.md similarity index 100% rename from docs/configuration.md rename to docs/advanced/configuration.md diff --git a/docs/api/pipeline_checks.md b/docs/api/pipeline_checks.md new file mode 100644 index 0000000..4b6d160 --- /dev/null +++ b/docs/api/pipeline_checks.md @@ -0,0 +1,11 @@ +::: deployer.pipeline_checks + options: + allow_inspection: false + merge_init_into_class: false + group_by_category: false + members: + - Pipelines + - Pipeline + - ConfigsDynamicModel + - ConfigDynamicModel + - _convert_artifact_type_to_str diff --git a/docs/api/utils.md b/docs/api/utils.md new file mode 100644 index 0000000..0116872 --- /dev/null +++ b/docs/api/utils.md @@ -0,0 +1,26 @@ +::: deployer.utils.models + options: + show_root_heading: true + members: + - create_model_from_func + +::: deployer.utils.utils + options: + show_root_heading: true + members: + - make_enum_from_python_package_dir + - import_pipeline_from_dir + +::: deployer.utils.config + options: + show_root_heading: true + merge_init_into_class: false + allow_inspection: true + members: + - VertexPipelinesSettings + - load_vertex_settings + - list_config_filepaths + - load_config + - _load_config_python + - _load_config_yaml + - _load_config_toml diff --git a/docs/api/vertex_deployer.md b/docs/api/vertex_deployer.md new file mode 100644 index 0000000..1c9af78 --- /dev/null +++ b/docs/api/vertex_deployer.md @@ -0,0 +1,6 @@ +::: deployer.pipeline_deployer.VertexPipelineDeployer + options: + show_root_heading: true + allow_inspection: true + merge_init_into_class: false + group_by_category: false diff --git a/mkdocs.yml b/mkdocs.yml index 9dd8bb4..38b60c0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -40,6 +40,20 @@ theme: plugins: - search + - mkdocstrings: + handlers: + python: + paths: [.] + options: + members_order: source + separate_signature: true + filters: ["!^_"] + docstring_options: + ignore_init_summary: true + merge_init_into_class: true + show_signature_annotations: true + signature_crossrefs: true + show_symbol_type_heading: true markdown_extensions: - admonition @@ -81,8 +95,12 @@ nav: - Folder Structure: folder_structure.md - Basic Usage: usage.md - Advanced User Guide: - - Vertex DevOps: advanced_user_guide.md - - Understand settings and configurations: configuration.md + - Vertex DevOps: advanced/advanced_user_guide.md + - Understand settings and configurations: advanced/configuration.md - CLI Reference: CLI_REFERENCE.md + - API Documentation: + - VertexPipelineDeployer: api/vertex_deployer.md + - pipeline_checks: api/pipeline_checks.md + - utils: api/utils.md - Contributing: contributing.md - Changelog: changelog.md diff --git a/pyproject.toml b/pyproject.toml index f139f33..142c967 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,9 +27,9 @@ rich = "^13.5" loguru = "^0.7" pydantic-settings = "^2.0, <2.9.0" pydantic = "^2.3" -pyinstrument = { version = "^4.5", optional = true } +pyinstrument = { version = ">=4.5,<6.0", optional = true } toml = "^0.10" -tomlkit = "^0.12" +tomlkit = ">=0.12,<0.14" jinja2 = "^3.1.3" numpy = [ # numpy is not a direct dependency but is required by some of the generated code { version = ">1.24", markers = "python_version >= '3.9'" }, @@ -49,6 +49,7 @@ codespell = "^2.2" [tool.poetry.group.docs.dependencies] mkdocs = "^1.5" mkdocs-material = {extras = ["pygments"], version = "^9.5.17"} +mkdocstrings = "^0.26" [tool.poetry.extras] profiling = ["pyinstrument"] @@ -109,7 +110,7 @@ version_variables = ["deployer/__init__.py:__version__"] version_toml = ["pyproject.toml:tool.poetry.version"] upload_to_pypi = false upload_to_release = true -build_command = "pip install poetry && poetry env use 3.10 && poetry build" +build_command = "pip install poetry && poetry build" commit_message = "chore(release): {version}\n\n[skip ci]\nAutomatically generated by semantic-release" tag_format = "{version}"