Skip to content

Commit

Permalink
Improve dbt ls parsing resilience to missing tags/config (#859)
Browse files Browse the repository at this point in the history
An Astronomer customer reported an issue when using `LoadMode.DBT_LS`
and Cosmos raising the error:

```
E               KeyError: 'tags'

cosmos/dbt/graph.py:113: KeyError
```

While parsing the DAG in Airflow. It is unclear which of the thousand
nodes in their project had this behaviour, but it feels like a good idea
to make Cosmos more resilient to this use case regardless.
  • Loading branch information
tatiana authored Feb 26, 2024
1 parent 31edff4 commit 71fd4ec
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 2 deletions.
4 changes: 2 additions & 2 deletions cosmos/dbt/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,8 @@ def parse_dbt_ls_output(project_path: Path, ls_stdout: str) -> dict[str, DbtNode
resource_type=DbtResourceType(node_dict["resource_type"]),
depends_on=node_dict.get("depends_on", {}).get("nodes", []),
file_path=project_path / node_dict["original_file_path"],
tags=node_dict["tags"],
config=node_dict["config"],
tags=node_dict.get("tags", []),
config=node_dict.get("config", {}),
)
nodes[node.unique_id] = node
logger.debug("Parsed dbt resource `%s` of type `%s`", node.unique_id, node.resource_type)
Expand Down
18 changes: 18 additions & 0 deletions tests/dbt/test_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -811,6 +811,24 @@ def test_parse_dbt_ls_output():
assert expected_nodes == nodes


def test_parse_dbt_ls_output_with_json_without_tags_or_config():
some_ls_stdout = '{"resource_type": "model", "name": "some-name", "original_file_path": "some-file-path.sql", "unique_id": "some-unique-id", "config": {}}'

expected_nodes = {
"some-unique-id": DbtNode(
unique_id="some-unique-id",
resource_type=DbtResourceType.MODEL,
file_path=Path("some-project/some-file-path.sql"),
tags=[],
config={},
depends_on=[],
),
}
nodes = parse_dbt_ls_output(Path("some-project"), some_ls_stdout)

assert expected_nodes == nodes


@patch("cosmos.dbt.graph.Popen")
@patch("cosmos.dbt.graph.DbtGraph.update_node_dependency")
@patch("cosmos.config.RenderConfig.validate_dbt_command")
Expand Down

0 comments on commit 71fd4ec

Please sign in to comment.