From 042551bd7b7a8bf71ce62671b9096ac589a1d0c8 Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Wed, 18 Dec 2024 12:51:20 +0530 Subject: [PATCH 1/2] Mention in docs callback support for ExecutionMode.VIRTUALENV --- dev/dags/example_virtualenv.py | 10 ++++++++++ docs/configuration/callbacks.rst | 9 +++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/dev/dags/example_virtualenv.py b/dev/dags/example_virtualenv.py index 24f4a8250..02f70e07c 100644 --- a/dev/dags/example_virtualenv.py +++ b/dev/dags/example_virtualenv.py @@ -56,6 +56,16 @@ def example_virtualenv() -> None: "py_requirements": ["dbt-postgres"], "install_deps": True, "emit_datasets": False, # Example of how to not set inlets and outlets + # -------------------------------------------------------------------------- + # Callback function to upload files using Airflow Object storage and Cosmos remote_target_path setting on + # Airflow 2.8 and above + # "callback": upload_to_cloud_storage, + # -------------------------------------------------------------------------- + # Callback function if you'd like to upload files from the target directory to remote store e.g. AWS S3 that + # works with Airflow < 2.8 too + # "callback": upload_to_aws_s3, + # "callback_args": {"aws_conn_id": "aws_s3_conn", "bucket_name": "cosmos-artifacts-upload"} + # -------------------------------------------------------------------------- }, ) diff --git a/docs/configuration/callbacks.rst b/docs/configuration/callbacks.rst index 9ffbc246f..5acf81cfe 100644 --- a/docs/configuration/callbacks.rst +++ b/docs/configuration/callbacks.rst @@ -3,7 +3,8 @@ Callbacks ========= -Cosmos supports callback functions that execute at the end of a task's execution when using ``ExecutionMode.LOCAL``. +Cosmos supports callback functions that execute at the end of a task's execution when using ``ExecutionMode.LOCAL`` and +``ExecutionMode.VIRTUALENV``. These callbacks can be used for various purposes, such as uploading files from the target directory to remote storage. While this feature has been available for some time, users may not be fully aware of its capabilities. @@ -46,6 +47,6 @@ metadata. Limitations and Contributions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Currently, callback support is available only when using ``ExecutionMode.LOCAL``. Contributions to extend this -functionality to other execution modes are welcome and encouraged. You can reference the implementation for -``ExecutionMode.LOCAL`` to add support for other modes. +Currently, callback support is available only when using ``ExecutionMode.LOCAL`` and ``ExecutionMode.VIRTUALENV``. +Contributions to extend this functionality to other execution modes are welcome and encouraged. You can reference the +implementation for ``ExecutionMode.LOCAL`` to add support for other modes. From 012b191a7e72b873ac30e299166130d67bf28e88 Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Wed, 18 Dec 2024 16:20:00 +0530 Subject: [PATCH 2/2] Add comment on why callback functions are commented in the DAG --- dev/dags/example_virtualenv.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/dags/example_virtualenv.py b/dev/dags/example_virtualenv.py index 02f70e07c..104f15980 100644 --- a/dev/dags/example_virtualenv.py +++ b/dev/dags/example_virtualenv.py @@ -57,6 +57,9 @@ def example_virtualenv() -> None: "install_deps": True, "emit_datasets": False, # Example of how to not set inlets and outlets # -------------------------------------------------------------------------- + # For the sake of avoiding additional latency observed while uploading files for each of the tasks, the + # below callback functions to be executed are commented, but you can uncomment them if you'd like to + # enable callback execution. # Callback function to upload files using Airflow Object storage and Cosmos remote_target_path setting on # Airflow 2.8 and above # "callback": upload_to_cloud_storage,