From 358f4b0f18288457f903b65ea5e5de477f0f6b44 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Thu, 2 Nov 2023 11:23:56 +0000 Subject: [PATCH] Fix CI issue running integration tests (#640) At the moment, all PRs are failing due to issues with integration tests not being able to run. An example of error message: ``` FAILED tests/test_example_dags.py::test_example_dag[extract_dag] - assert not {'/home/runner/work/astronomer-cosmos/astronomer-cosmos/dev/dags/example_cosmos_python_models.py': 'Traceback (most recent call last):\n File "/opt/hostedtoolcache/Python/3.9.18/x64/lib/python3.9/selectors.py", line 416, in select\n fd_event_list = self._selector.poll(timeout)\n File "/home/runner/.local/share/hatch/env/virtual/astronomer-cosmos/Za_bFbg4/tests.py3.9-2.4/lib/python3.9/site-packages/airflow/utils/timeout.py", line 69, in handle_timeout\n raise AirflowTaskTimeout(self.error_message)\nairflow.exceptions.AirflowTaskTimeout: DagBag import timeout for /home/runner/work/astronomer-cosmos/astronomer-cosmos/dev/dags/example_cosmos_python_models.py after 30.0s.\nPlease take a look at these docs to improve your DAG import time:\n* https://airflow.apache.org/docs/apache-airflow/2.4.0/best-practices.html#top-level-python-code\n* https://airflow.apache.org/docs/apache-airflow/2.4.0/best-practices.html#reducing-dag-complexity, PID: 2558\n'} ``` This issue relates to #520 - something we'll be soon working on. In the meantime, we're increasing the DAG Parsing time in the CI - so our tests can run again. We should also review how we run the integration tests - we probably do not need to run all DAGs against the Python x Airflow matrix, and we could parallelise some of those. --- .github/workflows/test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6889fb851..af71efc1c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -121,8 +121,9 @@ jobs: env: AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres - PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} + AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 + PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} @@ -192,6 +193,7 @@ jobs: AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} + AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} DATABRICKS_WAREHOUSE_ID: ${{ secrets.DATABRICKS_WAREHOUSE_ID }} @@ -256,6 +258,7 @@ jobs: env: AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ AIRFLOW_CONN_AIRFLOW_DB: postgres://postgres:postgres@0.0.0.0:5432/postgres + AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}