From b4991d48e15b345ad0a73b71406107a6ba8026aa Mon Sep 17 00:00:00 2001 From: TJaniF Date: Wed, 11 Oct 2023 10:23:30 +0200 Subject: [PATCH 1/9] add retries to code samples --- dev/dags/basic_cosmos_task_group.py | 1 + dev/dags/cosmos_manifest_example.py | 1 + dev/dags/cosmos_profile_mapping.py | 1 + dev/dags/cosmos_seed_dag.py | 2 +- dev/dags/dbt_docs.py | 1 + dev/dags/example_cosmos_python_models.py | 1 + dev/dags/example_model_version.py | 1 + dev/dags/example_virtualenv.py | 1 + dev/dags/user_defined_profile.py | 1 + docs/getting_started/astro.rst | 1 + docs/getting_started/gcc.rst | 1 + docs/getting_started/kubernetes.rst | 2 +- docs/getting_started/mwaa.rst | 1 + docs/getting_started/open-source.rst | 1 + docs/index.rst | 1 + 15 files changed, 15 insertions(+), 2 deletions(-) diff --git a/dev/dags/basic_cosmos_task_group.py b/dev/dags/basic_cosmos_task_group.py index 4685d6275..2f875b8c3 100644 --- a/dev/dags/basic_cosmos_task_group.py +++ b/dev/dags/basic_cosmos_task_group.py @@ -42,6 +42,7 @@ def basic_cosmos_task_group() -> None: ), operator_args={"install_deps": True}, profile_config=profile_config, + default_args={"retries": 2}, ) post_dbt = EmptyOperator(task_id="post_dbt") diff --git a/dev/dags/cosmos_manifest_example.py b/dev/dags/cosmos_manifest_example.py index ddf5f1b10..14ce42606 100644 --- a/dev/dags/cosmos_manifest_example.py +++ b/dev/dags/cosmos_manifest_example.py @@ -36,5 +36,6 @@ start_date=datetime(2023, 1, 1), catchup=False, dag_id="cosmos_manifest_example", + default_args={"retries": 2}, ) # [END local_example] diff --git a/dev/dags/cosmos_profile_mapping.py b/dev/dags/cosmos_profile_mapping.py index b3f16c9e6..33619a39d 100644 --- a/dev/dags/cosmos_profile_mapping.py +++ b/dev/dags/cosmos_profile_mapping.py @@ -41,6 +41,7 @@ def cosmos_profile_mapping() -> None: ), ), operator_args={"install_deps": True}, + default_args={"retries": 2}, ) post_dbt = EmptyOperator(task_id="post_dbt") diff --git a/dev/dags/cosmos_seed_dag.py b/dev/dags/cosmos_seed_dag.py index afdf56c72..0f3353bc1 100644 --- a/dev/dags/cosmos_seed_dag.py +++ b/dev/dags/cosmos_seed_dag.py @@ -40,7 +40,7 @@ doc_md=__doc__, catchup=False, max_active_runs=1, - default_args={"owner": "01-EXTRACT"}, + default_args={"owner": "01-EXTRACT", "retries": 2}, ) as dag: with TaskGroup(group_id="drop_seeds_if_exist") as drop_seeds: for seed in ["raw_customers", "raw_payments", "raw_orders"]: diff --git a/dev/dags/dbt_docs.py b/dev/dags/dbt_docs.py index 1bb5fb1c5..1fcd1c341 100644 --- a/dev/dags/dbt_docs.py +++ b/dev/dags/dbt_docs.py @@ -66,6 +66,7 @@ def which_upload(): schedule_interval="@daily", doc_md=__doc__, catchup=False, + default_args={"retries": 2}, ) as dag: generate_dbt_docs_aws = DbtDocsS3Operator( task_id="generate_dbt_docs_aws", diff --git a/dev/dags/example_cosmos_python_models.py b/dev/dags/example_cosmos_python_models.py index 92012fe62..7d9a61465 100644 --- a/dev/dags/example_cosmos_python_models.py +++ b/dev/dags/example_cosmos_python_models.py @@ -48,5 +48,6 @@ start_date=datetime(2023, 1, 1), catchup=False, dag_id="example_cosmos_python_models", + default_args={"retries": 2}, ) # [END example_cosmos_python_models] diff --git a/dev/dags/example_model_version.py b/dev/dags/example_model_version.py index a4226a611..78f38647d 100644 --- a/dev/dags/example_model_version.py +++ b/dev/dags/example_model_version.py @@ -34,5 +34,6 @@ start_date=datetime(2023, 1, 1), catchup=False, dag_id="example_model_version", + default_args={"retries": 2}, ) # [END local_example] diff --git a/dev/dags/example_virtualenv.py b/dev/dags/example_virtualenv.py index e91c881cf..7b1368f8c 100644 --- a/dev/dags/example_virtualenv.py +++ b/dev/dags/example_virtualenv.py @@ -41,5 +41,6 @@ start_date=datetime(2023, 1, 1), catchup=False, dag_id="example_virtualenv", + default_args={"retries": 2}, ) # [END virtualenv_example] diff --git a/dev/dags/user_defined_profile.py b/dev/dags/user_defined_profile.py index 2624ebb4f..ab30cdb2f 100644 --- a/dev/dags/user_defined_profile.py +++ b/dev/dags/user_defined_profile.py @@ -36,6 +36,7 @@ def user_defined_profile() -> None: profiles_yml_filepath=PROFILES_FILE_PATH, ), operator_args={"append_env": True, "install_deps": True}, + default_args={"retries": 2}, ) post_dbt = EmptyOperator(task_id="post_dbt") diff --git a/docs/getting_started/astro.rst b/docs/getting_started/astro.rst index a3fa14577..fbb51742a 100644 --- a/docs/getting_started/astro.rst +++ b/docs/getting_started/astro.rst @@ -106,6 +106,7 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c start_date=datetime(2023, 1, 1), catchup=False, dag_id="my_cosmos_dag", + default_args={"retries": 2}, ) diff --git a/docs/getting_started/gcc.rst b/docs/getting_started/gcc.rst index 00fa503a0..1ecce3ed7 100644 --- a/docs/getting_started/gcc.rst +++ b/docs/getting_started/gcc.rst @@ -75,4 +75,5 @@ Make sure to rename the ```` value below to your adapter's Python start_date=datetime(2023, 1, 1), catchup=False, dag_id="my_cosmos_dag", + default_args={"retries": 2}, ) diff --git a/docs/getting_started/kubernetes.rst b/docs/getting_started/kubernetes.rst index 6d2368997..1ae918a9a 100644 --- a/docs/getting_started/kubernetes.rst +++ b/docs/getting_started/kubernetes.rst @@ -28,7 +28,7 @@ Additional KubernetesPodOperator parameters can be added on the operator_args pa For instance, -.. code-block:: text +.. code-block:: python run_models = DbtTaskGroup( profile_config=ProfileConfig( diff --git a/docs/getting_started/mwaa.rst b/docs/getting_started/mwaa.rst index 0a3fa400e..9c2258e8b 100644 --- a/docs/getting_started/mwaa.rst +++ b/docs/getting_started/mwaa.rst @@ -109,4 +109,5 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c start_date=datetime(2023, 1, 1), catchup=False, dag_id="my_cosmos_dag", + default_args={"retries": 2}, ) diff --git a/docs/getting_started/open-source.rst b/docs/getting_started/open-source.rst index 2c44be4a6..8ed5ce66d 100644 --- a/docs/getting_started/open-source.rst +++ b/docs/getting_started/open-source.rst @@ -61,4 +61,5 @@ For example, if you wanted to put your dbt project in the ``/usr/local/airflow/d start_date=datetime(2023, 1, 1), catchup=False, dag_id="my_cosmos_dag", + default_args={"retries": 2}, ) diff --git a/docs/index.rst b/docs/index.rst index b9768f621..f627c3fb6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -70,6 +70,7 @@ You can render an Airflow Task Group using the ``DbtTaskGroup`` class. Here's an dbt_tg = DbtTaskGroup( project_config=ProjectConfig("jaffle_shop"), profile_config=profile_config, + default_args={"retries": 2}, ) e2 = EmptyOperator(task_id="post_dbt") From 4256c0a7e0116fb023bfa6b63ec30732400e6dc7 Mon Sep 17 00:00:00 2001 From: TJaniF Date: Wed, 11 Oct 2023 10:23:51 +0200 Subject: [PATCH 2/9] forgot one --- dev/dags/basic_cosmos_dag.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/dags/basic_cosmos_dag.py b/dev/dags/basic_cosmos_dag.py index abe5ecfeb..99f6b44e3 100644 --- a/dev/dags/basic_cosmos_dag.py +++ b/dev/dags/basic_cosmos_dag.py @@ -34,5 +34,6 @@ start_date=datetime(2023, 1, 1), catchup=False, dag_id="basic_cosmos_dag", + default_args={"retries": 2}, ) # [END local_example] From 6f28c84c504453cae409048c0e955917a874e401 Mon Sep 17 00:00:00 2001 From: TJaniF Date: Wed, 11 Oct 2023 10:25:20 +0200 Subject: [PATCH 3/9] change schedule_interval to schedule --- dev/dags/basic_cosmos_task_group.py | 2 +- dev/dags/cosmos_manifest_example.py | 2 +- dev/dags/cosmos_profile_mapping.py | 2 +- dev/dags/dbt_docs.py | 2 +- dev/dags/example_cosmos_python_models.py | 2 +- dev/dags/example_model_version.py | 2 +- dev/dags/example_virtualenv.py | 2 +- dev/dags/user_defined_profile.py | 2 +- docs/configuration/scheduling.rst | 4 ++-- docs/getting_started/astro.rst | 2 +- docs/getting_started/gcc.rst | 2 +- docs/getting_started/mwaa.rst | 2 +- docs/getting_started/open-source.rst | 2 +- 13 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dev/dags/basic_cosmos_task_group.py b/dev/dags/basic_cosmos_task_group.py index 2f875b8c3..fe7528ce9 100644 --- a/dev/dags/basic_cosmos_task_group.py +++ b/dev/dags/basic_cosmos_task_group.py @@ -25,7 +25,7 @@ @dag( - schedule_interval="@daily", + schedule="@daily", start_date=datetime(2023, 1, 1), catchup=False, ) diff --git a/dev/dags/cosmos_manifest_example.py b/dev/dags/cosmos_manifest_example.py index 14ce42606..ffcc2a03d 100644 --- a/dev/dags/cosmos_manifest_example.py +++ b/dev/dags/cosmos_manifest_example.py @@ -32,7 +32,7 @@ render_config=RenderConfig(load_method=LoadMode.DBT_MANIFEST, select=["path:models/customers.sql"]), operator_args={"install_deps": True}, # normal dag parameters - schedule_interval="@daily", + schedule="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="cosmos_manifest_example", diff --git a/dev/dags/cosmos_profile_mapping.py b/dev/dags/cosmos_profile_mapping.py index 33619a39d..1b3ea80aa 100644 --- a/dev/dags/cosmos_profile_mapping.py +++ b/dev/dags/cosmos_profile_mapping.py @@ -18,7 +18,7 @@ @dag( - schedule_interval="@daily", + schedule="@daily", start_date=datetime(2023, 1, 1), catchup=False, ) diff --git a/dev/dags/dbt_docs.py b/dev/dags/dbt_docs.py index 1fcd1c341..7bf6c71d5 100644 --- a/dev/dags/dbt_docs.py +++ b/dev/dags/dbt_docs.py @@ -63,7 +63,7 @@ def which_upload(): with DAG( dag_id="docs_dag", start_date=datetime(2023, 1, 1), - schedule_interval="@daily", + schedule="@daily", doc_md=__doc__, catchup=False, default_args={"retries": 2}, diff --git a/dev/dags/example_cosmos_python_models.py b/dev/dags/example_cosmos_python_models.py index 7d9a61465..9a0b1847f 100644 --- a/dev/dags/example_cosmos_python_models.py +++ b/dev/dags/example_cosmos_python_models.py @@ -44,7 +44,7 @@ "append_env": True, }, # normal dag parameters - schedule_interval="@daily", + schedule="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="example_cosmos_python_models", diff --git a/dev/dags/example_model_version.py b/dev/dags/example_model_version.py index 78f38647d..0fef1c75a 100644 --- a/dev/dags/example_model_version.py +++ b/dev/dags/example_model_version.py @@ -30,7 +30,7 @@ profile_config=profile_config, operator_args={"install_deps": True}, # normal dag parameters - schedule_interval="@daily", + schedule="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="example_model_version", diff --git a/dev/dags/example_virtualenv.py b/dev/dags/example_virtualenv.py index 7b1368f8c..096baf8e3 100644 --- a/dev/dags/example_virtualenv.py +++ b/dev/dags/example_virtualenv.py @@ -37,7 +37,7 @@ "install_deps": True, }, # normal dag parameters - schedule_interval="@daily", + schedule="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="example_virtualenv", diff --git a/dev/dags/user_defined_profile.py b/dev/dags/user_defined_profile.py index ab30cdb2f..569a2f1dd 100644 --- a/dev/dags/user_defined_profile.py +++ b/dev/dags/user_defined_profile.py @@ -16,7 +16,7 @@ @dag( - schedule_interval="@daily", + schedule="@daily", start_date=datetime(2023, 1, 1), catchup=False, ) diff --git a/docs/configuration/scheduling.rst b/docs/configuration/scheduling.rst index 738031848..bd7b7c890 100644 --- a/docs/configuration/scheduling.rst +++ b/docs/configuration/scheduling.rst @@ -17,7 +17,7 @@ To schedule a dbt project on a time-based schedule, you can use Airflow's schedu jaffle_shop = DbtDag( # ... start_date=datetime(2023, 1, 1), - schedule_interval="@daily", + schedule="@daily", ) @@ -45,7 +45,7 @@ Then, you can use Airflow's data-aware scheduling capabilities to schedule ``my_ project_one = DbtDag( # ... start_date=datetime(2023, 1, 1), - schedule_interval="@daily", + schedule="@daily", ) project_two = DbtDag( diff --git a/docs/getting_started/astro.rst b/docs/getting_started/astro.rst index fbb51742a..69d44c555 100644 --- a/docs/getting_started/astro.rst +++ b/docs/getting_started/astro.rst @@ -102,7 +102,7 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c dbt_executable_path=f"{os.environ['AIRFLOW_HOME']}/dbt_venv/bin/dbt", ), # normal dag parameters - schedule_interval="@daily", + schedule="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="my_cosmos_dag", diff --git a/docs/getting_started/gcc.rst b/docs/getting_started/gcc.rst index 1ecce3ed7..1900acd4d 100644 --- a/docs/getting_started/gcc.rst +++ b/docs/getting_started/gcc.rst @@ -71,7 +71,7 @@ Make sure to rename the ```` value below to your adapter's Python "py_requirements": [""], }, # normal dag parameters - schedule_interval="@daily", + schedule="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="my_cosmos_dag", diff --git a/docs/getting_started/mwaa.rst b/docs/getting_started/mwaa.rst index 9c2258e8b..79e769068 100644 --- a/docs/getting_started/mwaa.rst +++ b/docs/getting_started/mwaa.rst @@ -105,7 +105,7 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c ), profile_config=profile_config, # normal dag parameters - schedule_interval="@daily", + schedule="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="my_cosmos_dag", diff --git a/docs/getting_started/open-source.rst b/docs/getting_started/open-source.rst index 8ed5ce66d..1deb93688 100644 --- a/docs/getting_started/open-source.rst +++ b/docs/getting_started/open-source.rst @@ -57,7 +57,7 @@ For example, if you wanted to put your dbt project in the ``/usr/local/airflow/d dbt_executable_path=f"{os.environ['AIRFLOW_HOME']}/dbt_venv/bin/dbt", ), # normal dag parameters - schedule_interval="@daily", + schedule="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="my_cosmos_dag", From 3187df6601b9b3c12b37325c2fd76c5719edcf5c Mon Sep 17 00:00:00 2001 From: TJaniF Date: Wed, 11 Oct 2023 10:35:58 +0200 Subject: [PATCH 4/9] add note about dagbad timeout error --- docs/getting_started/astro.rst | 3 +++ docs/getting_started/gcc.rst | 5 +++++ docs/getting_started/mwaa.rst | 5 +++++ docs/getting_started/open-source.rst | 5 +++++ 4 files changed, 18 insertions(+) diff --git a/docs/getting_started/astro.rst b/docs/getting_started/astro.rst index 69d44c555..2234349ea 100644 --- a/docs/getting_started/astro.rst +++ b/docs/getting_started/astro.rst @@ -109,6 +109,9 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c default_args={"retries": 2}, ) +.. note:: + In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. + This error can be resolved by increasing the value of the Airflow configuration `AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT `_. Start your project ~~~~~~~~~~~~~~~~~~ diff --git a/docs/getting_started/gcc.rst b/docs/getting_started/gcc.rst index 1900acd4d..26d3edfa2 100644 --- a/docs/getting_started/gcc.rst +++ b/docs/getting_started/gcc.rst @@ -77,3 +77,8 @@ Make sure to rename the ```` value below to your adapter's Python dag_id="my_cosmos_dag", default_args={"retries": 2}, ) + + +.. note:: + In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. + This error can be resolved by increasing the value of the Airflow configuration `AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT `_. \ No newline at end of file diff --git a/docs/getting_started/mwaa.rst b/docs/getting_started/mwaa.rst index 79e769068..3663c149c 100644 --- a/docs/getting_started/mwaa.rst +++ b/docs/getting_started/mwaa.rst @@ -111,3 +111,8 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c dag_id="my_cosmos_dag", default_args={"retries": 2}, ) + + +.. note:: + In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. + This error can be resolved by increasing the value of the Airflow configuration `AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT `_. \ No newline at end of file diff --git a/docs/getting_started/open-source.rst b/docs/getting_started/open-source.rst index 1deb93688..c8411e6a1 100644 --- a/docs/getting_started/open-source.rst +++ b/docs/getting_started/open-source.rst @@ -63,3 +63,8 @@ For example, if you wanted to put your dbt project in the ``/usr/local/airflow/d dag_id="my_cosmos_dag", default_args={"retries": 2}, ) + + +.. note:: + In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. + This error can be resolved by increasing the value of the Airflow configuration `AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT `_. \ No newline at end of file From 774ea9fcbc3c61a883d9a94f447736eb96054595 Mon Sep 17 00:00:00 2001 From: TJaniF Date: Wed, 11 Oct 2023 10:41:35 +0200 Subject: [PATCH 5/9] update note --- docs/getting_started/astro.rst | 3 ++- docs/getting_started/gcc.rst | 3 ++- docs/getting_started/mwaa.rst | 3 ++- docs/getting_started/open-source.rst | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/getting_started/astro.rst b/docs/getting_started/astro.rst index 2234349ea..5aa96b73e 100644 --- a/docs/getting_started/astro.rst +++ b/docs/getting_started/astro.rst @@ -111,7 +111,8 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c .. note:: In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. - This error can be resolved by increasing the value of the Airflow configuration `AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT `_. + The recommended solution is to use the Cosmos manifest parsing method `dbt_manifest `_. + Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. Start your project ~~~~~~~~~~~~~~~~~~ diff --git a/docs/getting_started/gcc.rst b/docs/getting_started/gcc.rst index 26d3edfa2..2d7ce8bfd 100644 --- a/docs/getting_started/gcc.rst +++ b/docs/getting_started/gcc.rst @@ -81,4 +81,5 @@ Make sure to rename the ```` value below to your adapter's Python .. note:: In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. - This error can be resolved by increasing the value of the Airflow configuration `AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT `_. \ No newline at end of file + The recommended solution is to use the Cosmos manifest parsing method `dbt_manifest `_. + Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. \ No newline at end of file diff --git a/docs/getting_started/mwaa.rst b/docs/getting_started/mwaa.rst index 3663c149c..54227e565 100644 --- a/docs/getting_started/mwaa.rst +++ b/docs/getting_started/mwaa.rst @@ -115,4 +115,5 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c .. note:: In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. - This error can be resolved by increasing the value of the Airflow configuration `AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT `_. \ No newline at end of file + The recommended solution is to use the Cosmos manifest parsing method `dbt_manifest `_. + Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. \ No newline at end of file diff --git a/docs/getting_started/open-source.rst b/docs/getting_started/open-source.rst index c8411e6a1..ac3060ea4 100644 --- a/docs/getting_started/open-source.rst +++ b/docs/getting_started/open-source.rst @@ -67,4 +67,5 @@ For example, if you wanted to put your dbt project in the ``/usr/local/airflow/d .. note:: In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. - This error can be resolved by increasing the value of the Airflow configuration `AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT `_. \ No newline at end of file + The recommended solution is to use the Cosmos manifest parsing method `dbt_manifest `_. + Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. \ No newline at end of file From 23771cd4f2ed543f576e3b1803da4b76fc86f095 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 08:46:16 +0000 Subject: [PATCH 6/9] =?UTF-8?q?=F0=9F=8E=A8=20[pre-commit.ci]=20Auto=20for?= =?UTF-8?q?mat=20from=20pre-commit.com=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/getting_started/astro.rst | 2 +- docs/getting_started/gcc.rst | 4 ++-- docs/getting_started/mwaa.rst | 4 ++-- docs/getting_started/open-source.rst | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/getting_started/astro.rst b/docs/getting_started/astro.rst index 5aa96b73e..418f2a0c8 100644 --- a/docs/getting_started/astro.rst +++ b/docs/getting_started/astro.rst @@ -110,7 +110,7 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c ) .. note:: - In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. + In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. The recommended solution is to use the Cosmos manifest parsing method `dbt_manifest `_. Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. diff --git a/docs/getting_started/gcc.rst b/docs/getting_started/gcc.rst index 2d7ce8bfd..8830b530a 100644 --- a/docs/getting_started/gcc.rst +++ b/docs/getting_started/gcc.rst @@ -80,6 +80,6 @@ Make sure to rename the ```` value below to your adapter's Python .. note:: - In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. + In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. The recommended solution is to use the Cosmos manifest parsing method `dbt_manifest `_. - Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. \ No newline at end of file + Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. diff --git a/docs/getting_started/mwaa.rst b/docs/getting_started/mwaa.rst index 54227e565..75f9a9688 100644 --- a/docs/getting_started/mwaa.rst +++ b/docs/getting_started/mwaa.rst @@ -114,6 +114,6 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c .. note:: - In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. + In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. The recommended solution is to use the Cosmos manifest parsing method `dbt_manifest `_. - Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. \ No newline at end of file + Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. diff --git a/docs/getting_started/open-source.rst b/docs/getting_started/open-source.rst index ac3060ea4..7a84b35a5 100644 --- a/docs/getting_started/open-source.rst +++ b/docs/getting_started/open-source.rst @@ -66,6 +66,6 @@ For example, if you wanted to put your dbt project in the ``/usr/local/airflow/d .. note:: - In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. + In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. The recommended solution is to use the Cosmos manifest parsing method `dbt_manifest `_. - Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. \ No newline at end of file + Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. From 6753ce70a654fd781342e58c9bb9d8f4c5f9b9cd Mon Sep 17 00:00:00 2001 From: TJaniF Date: Wed, 11 Oct 2023 11:53:45 +0200 Subject: [PATCH 7/9] change back to schedule_interval --- README.rst | 2 +- dev/dags/basic_cosmos_task_group.py | 2 +- dev/dags/cosmos_manifest_example.py | 2 +- dev/dags/cosmos_profile_mapping.py | 2 +- dev/dags/cosmos_seed_dag.py | 2 +- dev/dags/dbt_docs.py | 2 +- dev/dags/example_cosmos_python_models.py | 2 +- dev/dags/example_model_version.py | 2 +- dev/dags/example_virtualenv.py | 2 +- dev/dags/user_defined_profile.py | 2 +- docs/configuration/scheduling.rst | 6 +++--- docs/getting_started/astro.rst | 2 +- docs/getting_started/gcc.rst | 2 +- docs/getting_started/mwaa.rst | 2 +- docs/getting_started/open-source.rst | 2 +- docs/index.rst | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.rst b/README.rst index d75c4eec9..e4f69af63 100644 --- a/README.rst +++ b/README.rst @@ -61,7 +61,7 @@ You can render an Airflow Task Group using the ``DbtTaskGroup`` class. Here's an with DAG( dag_id="extract_dag", start_date=datetime(2022, 11, 27), - schedule="@daily", + schedule_interval="@daily", ): e1 = EmptyOperator(task_id="pre_dbt") diff --git a/dev/dags/basic_cosmos_task_group.py b/dev/dags/basic_cosmos_task_group.py index fe7528ce9..2f875b8c3 100644 --- a/dev/dags/basic_cosmos_task_group.py +++ b/dev/dags/basic_cosmos_task_group.py @@ -25,7 +25,7 @@ @dag( - schedule="@daily", + schedule_interval="@daily", start_date=datetime(2023, 1, 1), catchup=False, ) diff --git a/dev/dags/cosmos_manifest_example.py b/dev/dags/cosmos_manifest_example.py index ffcc2a03d..14ce42606 100644 --- a/dev/dags/cosmos_manifest_example.py +++ b/dev/dags/cosmos_manifest_example.py @@ -32,7 +32,7 @@ render_config=RenderConfig(load_method=LoadMode.DBT_MANIFEST, select=["path:models/customers.sql"]), operator_args={"install_deps": True}, # normal dag parameters - schedule="@daily", + schedule_interval="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="cosmos_manifest_example", diff --git a/dev/dags/cosmos_profile_mapping.py b/dev/dags/cosmos_profile_mapping.py index 1b3ea80aa..33619a39d 100644 --- a/dev/dags/cosmos_profile_mapping.py +++ b/dev/dags/cosmos_profile_mapping.py @@ -18,7 +18,7 @@ @dag( - schedule="@daily", + schedule_interval="@daily", start_date=datetime(2023, 1, 1), catchup=False, ) diff --git a/dev/dags/cosmos_seed_dag.py b/dev/dags/cosmos_seed_dag.py index 0f3353bc1..cef84dd66 100644 --- a/dev/dags/cosmos_seed_dag.py +++ b/dev/dags/cosmos_seed_dag.py @@ -36,7 +36,7 @@ with DAG( dag_id="extract_dag", start_date=datetime(2022, 11, 27), - schedule="@daily", + schedule_interval="@daily", doc_md=__doc__, catchup=False, max_active_runs=1, diff --git a/dev/dags/dbt_docs.py b/dev/dags/dbt_docs.py index 7bf6c71d5..1fcd1c341 100644 --- a/dev/dags/dbt_docs.py +++ b/dev/dags/dbt_docs.py @@ -63,7 +63,7 @@ def which_upload(): with DAG( dag_id="docs_dag", start_date=datetime(2023, 1, 1), - schedule="@daily", + schedule_interval="@daily", doc_md=__doc__, catchup=False, default_args={"retries": 2}, diff --git a/dev/dags/example_cosmos_python_models.py b/dev/dags/example_cosmos_python_models.py index 9a0b1847f..7d9a61465 100644 --- a/dev/dags/example_cosmos_python_models.py +++ b/dev/dags/example_cosmos_python_models.py @@ -44,7 +44,7 @@ "append_env": True, }, # normal dag parameters - schedule="@daily", + schedule_interval="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="example_cosmos_python_models", diff --git a/dev/dags/example_model_version.py b/dev/dags/example_model_version.py index 0fef1c75a..78f38647d 100644 --- a/dev/dags/example_model_version.py +++ b/dev/dags/example_model_version.py @@ -30,7 +30,7 @@ profile_config=profile_config, operator_args={"install_deps": True}, # normal dag parameters - schedule="@daily", + schedule_interval="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="example_model_version", diff --git a/dev/dags/example_virtualenv.py b/dev/dags/example_virtualenv.py index 096baf8e3..7b1368f8c 100644 --- a/dev/dags/example_virtualenv.py +++ b/dev/dags/example_virtualenv.py @@ -37,7 +37,7 @@ "install_deps": True, }, # normal dag parameters - schedule="@daily", + schedule_interval="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="example_virtualenv", diff --git a/dev/dags/user_defined_profile.py b/dev/dags/user_defined_profile.py index 569a2f1dd..ab30cdb2f 100644 --- a/dev/dags/user_defined_profile.py +++ b/dev/dags/user_defined_profile.py @@ -16,7 +16,7 @@ @dag( - schedule="@daily", + schedule_interval="@daily", start_date=datetime(2023, 1, 1), catchup=False, ) diff --git a/docs/configuration/scheduling.rst b/docs/configuration/scheduling.rst index bd7b7c890..de21f8495 100644 --- a/docs/configuration/scheduling.rst +++ b/docs/configuration/scheduling.rst @@ -17,7 +17,7 @@ To schedule a dbt project on a time-based schedule, you can use Airflow's schedu jaffle_shop = DbtDag( # ... start_date=datetime(2023, 1, 1), - schedule="@daily", + schedule_interval="@daily", ) @@ -45,12 +45,12 @@ Then, you can use Airflow's data-aware scheduling capabilities to schedule ``my_ project_one = DbtDag( # ... start_date=datetime(2023, 1, 1), - schedule="@daily", + schedule_interval="@daily", ) project_two = DbtDag( # ... - schedule=[get_dbt_dataset("my_conn", "project_one", "my_model")], + schedule_interval=[get_dbt_dataset("my_conn", "project_one", "my_model")], dbt_project_name="project_two", ) diff --git a/docs/getting_started/astro.rst b/docs/getting_started/astro.rst index 418f2a0c8..6188ae676 100644 --- a/docs/getting_started/astro.rst +++ b/docs/getting_started/astro.rst @@ -102,7 +102,7 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c dbt_executable_path=f"{os.environ['AIRFLOW_HOME']}/dbt_venv/bin/dbt", ), # normal dag parameters - schedule="@daily", + schedule_interval="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="my_cosmos_dag", diff --git a/docs/getting_started/gcc.rst b/docs/getting_started/gcc.rst index 8830b530a..cc0b8c318 100644 --- a/docs/getting_started/gcc.rst +++ b/docs/getting_started/gcc.rst @@ -71,7 +71,7 @@ Make sure to rename the ```` value below to your adapter's Python "py_requirements": [""], }, # normal dag parameters - schedule="@daily", + schedule_interval="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="my_cosmos_dag", diff --git a/docs/getting_started/mwaa.rst b/docs/getting_started/mwaa.rst index 75f9a9688..cb742495a 100644 --- a/docs/getting_started/mwaa.rst +++ b/docs/getting_started/mwaa.rst @@ -105,7 +105,7 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c ), profile_config=profile_config, # normal dag parameters - schedule="@daily", + schedule_interval="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="my_cosmos_dag", diff --git a/docs/getting_started/open-source.rst b/docs/getting_started/open-source.rst index 7a84b35a5..a560929e1 100644 --- a/docs/getting_started/open-source.rst +++ b/docs/getting_started/open-source.rst @@ -57,7 +57,7 @@ For example, if you wanted to put your dbt project in the ``/usr/local/airflow/d dbt_executable_path=f"{os.environ['AIRFLOW_HOME']}/dbt_venv/bin/dbt", ), # normal dag parameters - schedule="@daily", + schedule_interval="@daily", start_date=datetime(2023, 1, 1), catchup=False, dag_id="my_cosmos_dag", diff --git a/docs/index.rst b/docs/index.rst index f627c3fb6..3c61b645d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -63,7 +63,7 @@ You can render an Airflow Task Group using the ``DbtTaskGroup`` class. Here's an with DAG( dag_id="extract_dag", start_date=datetime(2022, 11, 27), - schedule="@daily", + schedule_interval="@daily", ): e1 = EmptyOperator(task_id="pre_dbt") From 6d3789e0425206fde0d5ac57d96bd3786e86cd48 Mon Sep 17 00:00:00 2001 From: TJaniF Date: Wed, 11 Oct 2023 13:10:19 +0200 Subject: [PATCH 8/9] change note wording --- docs/getting_started/astro.rst | 5 ++--- docs/getting_started/gcc.rst | 5 ++--- docs/getting_started/mwaa.rst | 5 ++--- docs/getting_started/open-source.rst | 5 ++--- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/docs/getting_started/astro.rst b/docs/getting_started/astro.rst index 6188ae676..a3e2ea077 100644 --- a/docs/getting_started/astro.rst +++ b/docs/getting_started/astro.rst @@ -110,9 +110,8 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c ) .. note:: - In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. - The recommended solution is to use the Cosmos manifest parsing method `dbt_manifest `_. - Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. + In some cases, especially in larger dbt projects, you might run into a ``DagBag import timeout`` error. + This error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. Start your project ~~~~~~~~~~~~~~~~~~ diff --git a/docs/getting_started/gcc.rst b/docs/getting_started/gcc.rst index cc0b8c318..53c7584f1 100644 --- a/docs/getting_started/gcc.rst +++ b/docs/getting_started/gcc.rst @@ -80,6 +80,5 @@ Make sure to rename the ```` value below to your adapter's Python .. note:: - In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. - The recommended solution is to use the Cosmos manifest parsing method `dbt_manifest `_. - Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. + In some cases, especially in larger dbt projects, you might run into a ``DagBag import timeout`` error. + This error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. diff --git a/docs/getting_started/mwaa.rst b/docs/getting_started/mwaa.rst index cb742495a..49570ecc2 100644 --- a/docs/getting_started/mwaa.rst +++ b/docs/getting_started/mwaa.rst @@ -114,6 +114,5 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c .. note:: - In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. - The recommended solution is to use the Cosmos manifest parsing method `dbt_manifest `_. - Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. + In some cases, especially in larger dbt projects, you might run into a ``DagBag import timeout`` error. + This error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. diff --git a/docs/getting_started/open-source.rst b/docs/getting_started/open-source.rst index a560929e1..043a454e9 100644 --- a/docs/getting_started/open-source.rst +++ b/docs/getting_started/open-source.rst @@ -66,6 +66,5 @@ For example, if you wanted to put your dbt project in the ``/usr/local/airflow/d .. note:: - In some cases, especially in projects containing a large number of DAGs, you might run into an error saying ``<> is not a valid ResourceType``. - The recommended solution is to use the Cosmos manifest parsing method `dbt_manifest `_. - Alternatively, the error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. + In some cases, especially in larger dbt projects, you might run into a ``DagBag import timeout`` error. + This error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. From b5cf9e0d7b78e2ac6514b6f1c56400be4b64eccf Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 11:10:32 +0000 Subject: [PATCH 9/9] =?UTF-8?q?=F0=9F=8E=A8=20[pre-commit.ci]=20Auto=20for?= =?UTF-8?q?mat=20from=20pre-commit.com=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/getting_started/astro.rst | 2 +- docs/getting_started/gcc.rst | 2 +- docs/getting_started/mwaa.rst | 2 +- docs/getting_started/open-source.rst | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/getting_started/astro.rst b/docs/getting_started/astro.rst index a3e2ea077..c0bedc7e6 100644 --- a/docs/getting_started/astro.rst +++ b/docs/getting_started/astro.rst @@ -110,7 +110,7 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c ) .. note:: - In some cases, especially in larger dbt projects, you might run into a ``DagBag import timeout`` error. + In some cases, especially in larger dbt projects, you might run into a ``DagBag import timeout`` error. This error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. Start your project diff --git a/docs/getting_started/gcc.rst b/docs/getting_started/gcc.rst index 53c7584f1..1ec056e84 100644 --- a/docs/getting_started/gcc.rst +++ b/docs/getting_started/gcc.rst @@ -80,5 +80,5 @@ Make sure to rename the ```` value below to your adapter's Python .. note:: - In some cases, especially in larger dbt projects, you might run into a ``DagBag import timeout`` error. + In some cases, especially in larger dbt projects, you might run into a ``DagBag import timeout`` error. This error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. diff --git a/docs/getting_started/mwaa.rst b/docs/getting_started/mwaa.rst index 49570ecc2..f7a569302 100644 --- a/docs/getting_started/mwaa.rst +++ b/docs/getting_started/mwaa.rst @@ -114,5 +114,5 @@ In your ``my_cosmos_dag.py`` file, import the ``DbtDag`` class from Cosmos and c .. note:: - In some cases, especially in larger dbt projects, you might run into a ``DagBag import timeout`` error. + In some cases, especially in larger dbt projects, you might run into a ``DagBag import timeout`` error. This error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_. diff --git a/docs/getting_started/open-source.rst b/docs/getting_started/open-source.rst index 043a454e9..ba9bbdb15 100644 --- a/docs/getting_started/open-source.rst +++ b/docs/getting_started/open-source.rst @@ -66,5 +66,5 @@ For example, if you wanted to put your dbt project in the ``/usr/local/airflow/d .. note:: - In some cases, especially in larger dbt projects, you might run into a ``DagBag import timeout`` error. + In some cases, especially in larger dbt projects, you might run into a ``DagBag import timeout`` error. This error can be resolved by increasing the value of the Airflow configuration `core.dagbag_import_timeout `_.