diff --git a/cosmos/dbt/graph.py b/cosmos/dbt/graph.py index 3b61ef57d..242bca6f9 100644 --- a/cosmos/dbt/graph.py +++ b/cosmos/dbt/graph.py @@ -21,6 +21,7 @@ LoadMode, ) from cosmos.dbt.parser.project import LegacyDbtProject +from cosmos.dbt.project import create_symlinks from cosmos.dbt.selector import select_nodes from cosmos.log import get_logger @@ -51,14 +52,6 @@ class DbtNode: has_test: bool = False -def create_symlinks(project_path: Path, tmp_dir: Path) -> None: - """Helper function to create symlinks to the dbt project files.""" - ignore_paths = (DBT_LOG_DIR_NAME, DBT_TARGET_DIR_NAME, "dbt_packages", "profiles.yml") - for child_name in os.listdir(project_path): - if child_name not in ignore_paths: - os.symlink(project_path / child_name, tmp_dir / child_name) - - def run_command(command: list[str], tmp_dir: Path, env_vars: dict[str, str]) -> str: """Run a command in a subprocess, returning the stdout.""" logger.info("Running command: `%s`", " ".join(command)) diff --git a/cosmos/dbt/project.py b/cosmos/dbt/project.py new file mode 100644 index 000000000..63f4fc007 --- /dev/null +++ b/cosmos/dbt/project.py @@ -0,0 +1,14 @@ +from pathlib import Path +import os +from cosmos.constants import ( + DBT_LOG_DIR_NAME, + DBT_TARGET_DIR_NAME, +) + + +def create_symlinks(project_path: Path, tmp_dir: Path) -> None: + """Helper function to create symlinks to the dbt project files.""" + ignore_paths = (DBT_LOG_DIR_NAME, DBT_TARGET_DIR_NAME, "dbt_packages", "profiles.yml") + for child_name in os.listdir(project_path): + if child_name not in ignore_paths: + os.symlink(project_path / child_name, tmp_dir / child_name) diff --git a/cosmos/operators/local.py b/cosmos/operators/local.py index 121ed66d0..57379c7ba 100644 --- a/cosmos/operators/local.py +++ b/cosmos/operators/local.py @@ -1,7 +1,6 @@ from __future__ import annotations import os -import shutil import signal import tempfile from attr import define @@ -45,6 +44,7 @@ FullOutputSubprocessResult, ) from cosmos.dbt.parser.output import extract_log_issues, parse_output +from cosmos.dbt.project import create_symlinks DBT_NO_TESTS_MSG = "Nothing to do" DBT_WARN_MSG = "WARN" @@ -192,19 +192,14 @@ def run_command( """ Copies the dbt project to a temporary directory and runs the command. """ - with tempfile.TemporaryDirectory() as tmp_dir: + with tempfile.TemporaryDirectory() as tmp_project_dir: logger.info( "Cloning project to writable temp directory %s from %s", - tmp_dir, + tmp_project_dir, self.project_dir, ) - # need a subfolder because shutil.copytree will fail if the destination dir already exists - tmp_project_dir = os.path.join(tmp_dir, "dbt_project") - shutil.copytree( - self.project_dir, - tmp_project_dir, - ) + create_symlinks(Path(self.project_dir), Path(tmp_project_dir)) with self.profile_config.ensure_profile() as profile_values: (profile_path, env_vars) = profile_values diff --git a/dev/dags/dbt/simple/data/imdb.db b/dev/dags/dbt/data/imdb.db similarity index 71% rename from dev/dags/dbt/simple/data/imdb.db rename to dev/dags/dbt/data/imdb.db index 605f6526e..be0c1fe77 100644 Binary files a/dev/dags/dbt/simple/data/imdb.db and b/dev/dags/dbt/data/imdb.db differ diff --git a/dev/dags/dbt/simple/models/movies_ratings_simplified.sql b/dev/dags/dbt/simple/models/movies_ratings_simplified.sql index 342782e34..671a5764f 100644 --- a/dev/dags/dbt/simple/models/movies_ratings_simplified.sql +++ b/dev/dags/dbt/simple/models/movies_ratings_simplified.sql @@ -11,4 +11,4 @@ select "Domestic", "Foreign", "Worldwide" -from {{ source('imdb', 'movies_ratings') }} +from {{ source('main', 'movies_ratings') }} diff --git a/dev/dags/dbt/simple/models/source.yml b/dev/dags/dbt/simple/models/source.yml index efb9b1560..b240ba542 100644 --- a/dev/dags/dbt/simple/models/source.yml +++ b/dev/dags/dbt/simple/models/source.yml @@ -1,7 +1,7 @@ version: 2 sources: - - name: imdb + - name: main description: Example of IMDB SQlite database tables: - name: movies_ratings diff --git a/dev/dags/dbt/simple/profiles.yml b/dev/dags/dbt/simple/profiles.yml index 5de7e781a..24841363e 100644 --- a/dev/dags/dbt/simple/profiles.yml +++ b/dev/dags/dbt/simple/profiles.yml @@ -7,5 +7,5 @@ simple: database: 'database' schema: 'main' schemas_and_paths: - main: "{{ env_var('DBT_SQLITE_PATH', '.') }}/data/imdb.db" - schema_directory: 'data' + main: "{{ env_var('DBT_SQLITE_PATH') }}/imdb.db" + schema_directory: "{{ env_var('DBT_SQLITE_PATH') }}" diff --git a/dev/dags/example_cosmos_sources.py b/dev/dags/example_cosmos_sources.py index ddeb9ab15..29c70db5a 100644 --- a/dev/dags/example_cosmos_sources.py +++ b/dev/dags/example_cosmos_sources.py @@ -26,7 +26,7 @@ DEFAULT_DBT_ROOT_PATH = Path(__file__).parent / "dbt" DBT_ROOT_PATH = Path(os.getenv("DBT_ROOT_PATH", DEFAULT_DBT_ROOT_PATH)) -os.environ["DBT_SQLITE_PATH"] = str(DEFAULT_DBT_ROOT_PATH / "simple") +os.environ["DBT_SQLITE_PATH"] = str(DEFAULT_DBT_ROOT_PATH / "data") profile_config = ProfileConfig( diff --git a/tests/dbt/test_graph.py b/tests/dbt/test_graph.py index 6ae6ab200..607e56641 100644 --- a/tests/dbt/test_graph.py +++ b/tests/dbt/test_graph.py @@ -12,7 +12,6 @@ DbtGraph, DbtNode, LoadMode, - create_symlinks, run_command, parse_dbt_ls_output, ) @@ -388,7 +387,7 @@ def test_load_via_dbt_ls_with_sources(load_method): ) getattr(dbt_graph, load_method)() assert len(dbt_graph.nodes) == 4 - assert "source.simple.imdb.movies_ratings" in dbt_graph.nodes + assert "source.simple.main.movies_ratings" in dbt_graph.nodes assert "exposure.simple.weekly_metrics" in dbt_graph.nodes @@ -661,17 +660,6 @@ def test_load_dbt_ls_and_manifest_with_model_version(load_method): } == set(dbt_graph.nodes["model.jaffle_shop.orders"].depends_on) -def test_create_symlinks(tmp_path): - """Tests that symlinks are created for expected files in the dbt project directory.""" - tmp_dir = tmp_path / "dbt-project" - tmp_dir.mkdir() - - create_symlinks(DBT_PROJECTS_ROOT_DIR / "jaffle_shop", tmp_dir) - for child in tmp_dir.iterdir(): - assert child.is_symlink() - assert child.name not in ("logs", "target", "profiles.yml", "dbt_packages") - - @pytest.mark.parametrize( "stdout,returncode", [ diff --git a/tests/dbt/test_project.py b/tests/dbt/test_project.py new file mode 100644 index 000000000..bd2555c98 --- /dev/null +++ b/tests/dbt/test_project.py @@ -0,0 +1,15 @@ +from pathlib import Path +from cosmos.dbt.project import create_symlinks + +DBT_PROJECTS_ROOT_DIR = Path(__file__).parent.parent.parent / "dev/dags/dbt" + + +def test_create_symlinks(tmp_path): + """Tests that symlinks are created for expected files in the dbt project directory.""" + tmp_dir = tmp_path / "dbt-project" + tmp_dir.mkdir() + + create_symlinks(DBT_PROJECTS_ROOT_DIR / "jaffle_shop", tmp_dir) + for child in tmp_dir.iterdir(): + assert child.is_symlink() + assert child.name not in ("logs", "target", "profiles.yml", "dbt_packages") diff --git a/tests/sample/manifest_source.json b/tests/sample/manifest_source.json index 5326e2ab2..9a84180f5 100644 --- a/tests/sample/manifest_source.json +++ b/tests/sample/manifest_source.json @@ -7,7 +7,7 @@ "model.simple.top_animations": [ "exposure.simple.weekly_metrics" ], - "source.simple.imdb.movies_ratings": [ + "source.simple.main.movies_ratings": [ "model.simple.movies_ratings_simplified" ] }, @@ -6992,7 +6992,7 @@ "depends_on": { "macros": [], "nodes": [ - "source.simple.imdb.movies_ratings" + "source.simple.main.movies_ratings" ] }, "description": "", @@ -7110,16 +7110,16 @@ "model.simple.top_animations" ], "model.simple.movies_ratings_simplified": [ - "source.simple.imdb.movies_ratings" + "source.simple.main.movies_ratings" ], "model.simple.top_animations": [ "model.simple.movies_ratings_simplified" ], - "source.simple.imdb.movies_ratings": [] + "source.simple.main.movies_ratings": [] }, "selectors": {}, "sources": { - "source.simple.imdb.movies_ratings": { + "source.simple.main.movies_ratings": { "columns": {}, "config": { "enabled": true @@ -7166,7 +7166,7 @@ "source_meta": {}, "source_name": "imdb", "tags": [], - "unique_id": "source.simple.imdb.movies_ratings", + "unique_id": "source.simple.main.movies_ratings", "unrendered_config": {} } }