Skip to content

Commit

Permalink
Fix tests
Browse files Browse the repository at this point in the history
  • Loading branch information
tatiana committed Oct 9, 2023
1 parent c4e5cdf commit ea61ef7
Show file tree
Hide file tree
Showing 4 changed files with 363 additions and 310 deletions.
2 changes: 1 addition & 1 deletion cosmos/dbt/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ def load_from_dbt_manifest(self) -> None:
with open(self.project.manifest_path) as fp: # type: ignore[arg-type]
manifest = json.load(fp)

resources = {**manifest.get("nodes", {}), **manifest.get("sources", {})}
resources = {**manifest.get("nodes", {}), **manifest.get("sources", {}), **manifest.get("exposures", {})}
for unique_id, node_dict in resources.items():
node = DbtNode(
name=node_dict["name"],
Expand Down
13 changes: 7 additions & 6 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -161,18 +161,18 @@ test = 'pytest -vv --durations=0 . -m "not integration" --ignore=tests/test_exam
test-cov = """pytest -vv --cov=cosmos --cov-report=term-missing --cov-report=xml --durations=0 -m "not integration" --ignore=tests/test_example_dags.py --ignore=tests/test_example_dags_no_connections.py"""
# we install using the following workaround to overcome installation conflicts, such as:
# apache-airflow 2.3.0 and dbt-core [0.13.0 - 1.5.2] and jinja2>=3.0.0 because these package versions have conflicting dependencies
test-integration-setup = """pip uninstall dbt-databricks dbt-sqlite dbt-postgres dbt-sqlite; \
test-integration-setup = """pip uninstall -y dbt-core dbt-databricks dbt-sqlite dbt-postgres dbt-sqlite; \
rm -rf airflow.*; \
airflow db init; \
pip install 'dbt-core' 'dbt-databricks' 'dbt-postgres' 'openlineage-airflow' 'dbt-sqlite'"""
pip install 'dbt-core==1.5.4' 'dbt-databricks<=1.5.4' 'dbt-postgres<=1.5.4' 'openlineage-airflow'"""
test-integration = """rm -rf dbt/jaffle_shop/dbt_packages;
pytest -vv \
pytest -x -vv \
--cov=cosmos \
--cov-report=term-missing \
--cov-report=xml \
--durations=0 \
-m integration \
-k 'not (example_cosmos_sources or example_cosmos_python_models or example_virtualenv)'
-k 'not (sqlite or example_cosmos_sources or example_cosmos_python_models or example_virtualenv)'
"""
test-integration-expensive = """echo "example_cosmos_sources" >> dev/dags/.airflowignore
rm -rf dbt/jaffle_shop/dbt_packages;
Expand All @@ -183,7 +183,7 @@ pytest -vv \
--durations=0 \
-m integration \
-k 'example_cosmos_python_models or example_virtualenv'"""
test-integration-sqlite-setup = """pip uninstall dbt-sqlite; \
test-integration-sqlite-setup = """pip uninstall -y dbt-core dbt-sqlite openlineage-airflow openlineage-integration-common; \
rm -rf airflow.*; \
airflow db init; \
pip install 'dbt-core==1.4' 'dbt-sqlite<=1.4' 'dbt-databricks<=1.4' 'dbt-postgres<=1.4' """
Expand All @@ -194,7 +194,7 @@ pytest -vv \
--cov-report=xml \
--durations=0 \
-m integration \
-k 'example_cosmos_sources'"""
-k 'example_cosmos_sources or sqlite'"""

[tool.pytest.ini_options]
filterwarnings = [
Expand All @@ -203,6 +203,7 @@ filterwarnings = [
minversion = "6.0"
markers = [
"integration",
"sqlite"
]

######################################
Expand Down
4 changes: 3 additions & 1 deletion tests/dbt/test_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,7 @@ def test_load_via_dbt_ls_with_invalid_dbt_path():
assert err_info.value.args[0] == expected


@pytest.mark.sqlite
@pytest.mark.parametrize("load_method", ["load_via_dbt_ls", "load_from_dbt_manifest"])
@pytest.mark.integration
def test_load_via_dbt_ls_with_sources(load_method):
Expand All @@ -268,8 +269,9 @@ def test_load_via_dbt_ls_with_sources(load_method):
),
)
getattr(dbt_graph, load_method)()
assert len(dbt_graph.nodes) == 3
assert len(dbt_graph.nodes) == 4
assert "source.simple.imdb.movies_ratings" in dbt_graph.nodes
assert "exposure.simple.weekly_metrics" in dbt_graph.nodes


@pytest.mark.integration
Expand Down
Loading

0 comments on commit ea61ef7

Please sign in to comment.