diff --git a/.changes/unreleased/Features-20230811-223109.yaml b/.changes/unreleased/Features-20230811-223109.yaml new file mode 100644 index 00000000000..7bd87fa57a1 --- /dev/null +++ b/.changes/unreleased/Features-20230811-223109.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Allows for specifying a childs parents max depth +time: 2023-08-11T22:31:09.729033-07:00 +custom: + Author: kentkr + Issue: CT-2599 diff --git a/core/dbt/graph/graph.py b/core/dbt/graph/graph.py index cf569f3547d..36f2da994dc 100644 --- a/core/dbt/graph/graph.py +++ b/core/dbt/graph/graph.py @@ -52,9 +52,11 @@ def exclude_edge_type(self, edge_type_to_exclude): def filter_edges_by_type(self, first_node, second_node, edge_type): return self.graph.get_edge_data(first_node, second_node).get("edge_type") != edge_type - def select_childrens_parents(self, selected: Set[UniqueId]) -> Set[UniqueId]: + def select_childrens_parents( + self, selected: Set[UniqueId], max_depth: Optional[int] = None + ) -> Set[UniqueId]: ancestors_for = self.select_children(selected) | selected - return self.select_parents(ancestors_for) | ancestors_for + return self.select_parents(ancestors_for, max_depth) | ancestors_for def select_children( self, selected: Set[UniqueId], max_depth: Optional[int] = None diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index cc0b4ebe9fc..047056e3806 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -109,7 +109,8 @@ def collect_specified_neighbors( """ additional: Set[UniqueId] = set() if spec.childrens_parents: - additional.update(self.graph.select_childrens_parents(selected)) + depth = spec.childrens_parents_depth + additional.update(self.graph.select_childrens_parents(selected, depth)) if spec.parents: depth = spec.parents_depth diff --git a/core/dbt/graph/selector_spec.py b/core/dbt/graph/selector_spec.py index e801aef7396..639c24301e5 100644 --- a/core/dbt/graph/selector_spec.py +++ b/core/dbt/graph/selector_spec.py @@ -14,7 +14,7 @@ RAW_SELECTOR_PATTERN = re.compile( r"\A" - r"(?P(\@))?" + r"(?P((?P(\d*))\@))?" r"(?P((?P(\d*))\+))?" r"((?P([\w.]+)):)?(?P(.*?))" r"(?P(\+(?P(\d*))))?" @@ -68,6 +68,7 @@ class SelectionCriteria: method_arguments: List[str] value: Any childrens_parents: bool + childrens_parents_depth: Optional[int] parents: bool parents_depth: Optional[int] children: bool @@ -118,6 +119,7 @@ def selection_criteria_from_dict( parents_depth = _match_to_int(dct, "parents_depth") children_depth = _match_to_int(dct, "children_depth") + childrens_parents_depth = _match_to_int(dct, "childrens_parents_depth") # If defined field in selector, override CLI flag indirect_selection = IndirectSelection( @@ -130,6 +132,7 @@ def selection_criteria_from_dict( method_arguments=method_arguments, value=dct["value"], childrens_parents=bool(dct.get("childrens_parents")), + childrens_parents_depth=childrens_parents_depth, parents=bool(dct.get("parents")), parents_depth=parents_depth, children=bool(dct.get("children")), diff --git a/tests/unit/graph/test_selector_spec.py b/tests/unit/graph/test_selector_spec.py index 451b107d85c..ddb61134449 100644 --- a/tests/unit/graph/test_selector_spec.py +++ b/tests/unit/graph/test_selector_spec.py @@ -70,6 +70,7 @@ def test_raw_parse_simple(): assert not result.parents assert result.parents_depth is None assert result.children_depth is None + assert result.childrens_parents_depth is None def test_raw_parse_simple_infer_path(): @@ -84,6 +85,7 @@ def test_raw_parse_simple_infer_path(): assert not result.parents assert result.parents_depth is None assert result.children_depth is None + assert result.childrens_parents_depth is None def test_raw_parse_simple_infer_path_modified(): @@ -98,6 +100,22 @@ def test_raw_parse_simple_infer_path_modified(): assert not result.parents assert result.parents_depth is None assert result.children_depth is None + assert result.childrens_parents_depth is None + + +def test_raw_parse_simple_infer_fqn_childrens_parents(): + raw = "1@asdf" + result = SelectionCriteria.from_single_spec(raw) + assert result.raw == raw + assert result.method == MethodName.FQN + assert result.method_arguments == [] + assert result.value == "asdf" + assert result.childrens_parents + assert not result.children + assert not result.parents + assert result.parents_depth is None + assert result.children_depth is None + assert result.childrens_parents_depth == 1 def test_raw_parse_simple_infer_fqn_parents(): @@ -112,6 +130,7 @@ def test_raw_parse_simple_infer_fqn_parents(): assert result.parents assert result.parents_depth is None assert result.children_depth is None + assert result.childrens_parents_depth is None def test_raw_parse_simple_infer_fqn_children(): @@ -126,6 +145,7 @@ def test_raw_parse_simple_infer_fqn_children(): assert not result.parents assert result.parents_depth is None assert result.children_depth is None + assert result.childrens_parents_depth is None def test_raw_parse_complex(): @@ -140,6 +160,7 @@ def test_raw_parse_complex(): assert result.parents assert result.parents_depth == 2 assert result.children_depth == 4 + assert result.childrens_parents_depth is None def test_raw_parse_weird(): @@ -155,6 +176,7 @@ def test_raw_parse_weird(): assert not result.parents assert result.parents_depth is None assert result.children_depth is None + assert result.childrens_parents_depth is None def test_raw_parse_invalid(): diff --git a/tests/unit/test_graph_selection.py b/tests/unit/test_graph_selection.py index 5d5cbf7469d..a5e2cad7cff 100644 --- a/tests/unit/test_graph_selection.py +++ b/tests/unit/test_graph_selection.py @@ -89,6 +89,7 @@ def id_macro(arg): (["1+Y.f"], [], {"m.X.c", "m.Y.f"}), # childrens parents (["@X.c"], [], {"m.X.a", "m.X.c", "m.Y.f", "m.X.g"}), + (["1@X.c"], [], {"m.X.a", "m.X.c", "m.Y.f", "m.X.g"}), # multiple selection/exclusion (["tag:abc", "tag:bcef"], [], {"m.X.a", "m.Y.b", "m.X.c", "m.X.e", "m.Y.f"}), (["tag:abc", "tag:bcef"], ["tag:efg"], {"m.X.a", "m.Y.b", "m.X.c"}), @@ -123,48 +124,51 @@ def test_run_specs(include, exclude, expected, graph, manifest): param_specs = [ - ("a", False, None, False, None, "fqn", "a", False), - ("+a", True, None, False, None, "fqn", "a", False), - ("256+a", True, 256, False, None, "fqn", "a", False), - ("a+", False, None, True, None, "fqn", "a", False), - ("a+256", False, None, True, 256, "fqn", "a", False), - ("+a+", True, None, True, None, "fqn", "a", False), - ("16+a+32", True, 16, True, 32, "fqn", "a", False), - ("@a", False, None, False, None, "fqn", "a", True), - ("a.b", False, None, False, None, "fqn", "a.b", False), - ("+a.b", True, None, False, None, "fqn", "a.b", False), - ("256+a.b", True, 256, False, None, "fqn", "a.b", False), - ("a.b+", False, None, True, None, "fqn", "a.b", False), - ("a.b+256", False, None, True, 256, "fqn", "a.b", False), - ("+a.b+", True, None, True, None, "fqn", "a.b", False), - ("16+a.b+32", True, 16, True, 32, "fqn", "a.b", False), - ("@a.b", False, None, False, None, "fqn", "a.b", True), - ("a.b.*", False, None, False, None, "fqn", "a.b.*", False), - ("+a.b.*", True, None, False, None, "fqn", "a.b.*", False), - ("256+a.b.*", True, 256, False, None, "fqn", "a.b.*", False), - ("a.b.*+", False, None, True, None, "fqn", "a.b.*", False), - ("a.b.*+256", False, None, True, 256, "fqn", "a.b.*", False), - ("+a.b.*+", True, None, True, None, "fqn", "a.b.*", False), - ("16+a.b.*+32", True, 16, True, 32, "fqn", "a.b.*", False), - ("@a.b.*", False, None, False, None, "fqn", "a.b.*", True), - ("tag:a", False, None, False, None, "tag", "a", False), - ("+tag:a", True, None, False, None, "tag", "a", False), - ("256+tag:a", True, 256, False, None, "tag", "a", False), - ("tag:a+", False, None, True, None, "tag", "a", False), - ("tag:a+256", False, None, True, 256, "tag", "a", False), - ("+tag:a+", True, None, True, None, "tag", "a", False), - ("16+tag:a+32", True, 16, True, 32, "tag", "a", False), - ("@tag:a", False, None, False, None, "tag", "a", True), - ("source:a", False, None, False, None, "source", "a", False), - ("source:a+", False, None, True, None, "source", "a", False), - ("source:a+1", False, None, True, 1, "source", "a", False), - ("source:a+32", False, None, True, 32, "source", "a", False), - ("@source:a", False, None, False, None, "source", "a", True), + ("a", False, None, False, None, "fqn", "a", False, None), + ("+a", True, None, False, None, "fqn", "a", False, None), + ("256+a", True, 256, False, None, "fqn", "a", False, None), + ("a+", False, None, True, None, "fqn", "a", False, None), + ("a+256", False, None, True, 256, "fqn", "a", False, None), + ("+a+", True, None, True, None, "fqn", "a", False, None), + ("16+a+32", True, 16, True, 32, "fqn", "a", False, None), + ("@a", False, None, False, None, "fqn", "a", True, None), + ("20@a", False, None, False, None, "fqn", "a", True, 20), + ("a.b", False, None, False, None, "fqn", "a.b", False, None), + ("+a.b", True, None, False, None, "fqn", "a.b", False, None), + ("256+a.b", True, 256, False, None, "fqn", "a.b", False, None), + ("a.b+", False, None, True, None, "fqn", "a.b", False, None), + ("a.b+256", False, None, True, 256, "fqn", "a.b", False, None), + ("+a.b+", True, None, True, None, "fqn", "a.b", False, None), + ("16+a.b+32", True, 16, True, 32, "fqn", "a.b", False, None), + ("@a.b", False, None, False, None, "fqn", "a.b", True, None), + ("5@a.b", False, None, False, None, "fqn", "a.b", True, 5), + ("a.b.*", False, None, False, None, "fqn", "a.b.*", False, None), + ("+a.b.*", True, None, False, None, "fqn", "a.b.*", False, None), + ("256+a.b.*", True, 256, False, None, "fqn", "a.b.*", False, None), + ("a.b.*+", False, None, True, None, "fqn", "a.b.*", False, None), + ("a.b.*+256", False, None, True, 256, "fqn", "a.b.*", False, None), + ("+a.b.*+", True, None, True, None, "fqn", "a.b.*", False, None), + ("16+a.b.*+32", True, 16, True, 32, "fqn", "a.b.*", False, None), + ("@a.b.*", False, None, False, None, "fqn", "a.b.*", True, None), + ("12@a.b.*", False, None, False, None, "fqn", "a.b.*", True, 12), + ("tag:a", False, None, False, None, "tag", "a", False, None), + ("+tag:a", True, None, False, None, "tag", "a", False, None), + ("256+tag:a", True, 256, False, None, "tag", "a", False, None), + ("tag:a+", False, None, True, None, "tag", "a", False, None), + ("tag:a+256", False, None, True, 256, "tag", "a", False, None), + ("+tag:a+", True, None, True, None, "tag", "a", False, None), + ("16+tag:a+32", True, 16, True, 32, "tag", "a", False, None), + ("3@tag:a", False, None, False, None, "tag", "a", True, 3), + ("source:a", False, None, False, None, "source", "a", False, None), + ("source:a+", False, None, True, None, "source", "a", False, None), + ("source:a+1", False, None, True, 1, "source", "a", False, None), + ("source:a+32", False, None, True, 32, "source", "a", False, None), + ("1@source:a", False, None, False, None, "source", "a", True, 1), ] @pytest.mark.parametrize( - "spec,parents,parents_depth,children,children_depth,filter_type,filter_value,childrens_parents", + "spec,parents,parents_depth,children,children_depth,filter_type,filter_value,childrens_parents,childrens_parents_depth", param_specs, ids=id_macro, ) @@ -177,6 +181,7 @@ def test_parse_specs( filter_type, filter_value, childrens_parents, + childrens_parents_depth, ): parsed = graph_selector.SelectionCriteria.from_single_spec(spec) assert parsed.parents == parents @@ -186,6 +191,7 @@ def test_parse_specs( assert parsed.method == filter_type assert parsed.value == filter_value assert parsed.childrens_parents == childrens_parents + assert parsed.childrens_parents_depth == childrens_parents_depth invalid_specs = [