From d65bae5f050288aa37dca979afaaba09f6b0c97b Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 13 Mar 2024 16:17:47 -0400 Subject: [PATCH] Provide ability to exclude resource_types, instead of listing everything not excluded (#9756) --- .../unreleased/Features-20240312-140407.yaml | 6 ++++ core/dbt/cli/flags.py | 5 +++- core/dbt/cli/main.py | 3 ++ core/dbt/cli/params.py | 30 ++++++++++++++++++- core/dbt/task/base.py | 30 ++++++++++++++++++- core/dbt/task/build.py | 13 +++----- core/dbt/task/clone.py | 21 +++++-------- core/dbt/task/list.py | 17 ++++------- .../unit_testing/test_unit_testing.py | 12 +++++++- 9 files changed, 100 insertions(+), 37 deletions(-) create mode 100644 .changes/unreleased/Features-20240312-140407.yaml diff --git a/.changes/unreleased/Features-20240312-140407.yaml b/.changes/unreleased/Features-20240312-140407.yaml new file mode 100644 index 00000000000..5486d560aec --- /dev/null +++ b/.changes/unreleased/Features-20240312-140407.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Allow excluding resource types for build, list, and clone commands +time: 2024-03-12T14:04:07.086017-04:00 +custom: + Author: gshank + Issue: "9237" diff --git a/core/dbt/cli/flags.py b/core/dbt/cli/flags.py index 3cfafc9296f..d688d3e4fa4 100644 --- a/core/dbt/cli/flags.py +++ b/core/dbt/cli/flags.py @@ -399,7 +399,10 @@ def add_fn(x): # MultiOption flags come back as lists, but we want to pass them as space separated strings if isinstance(v, list): - v = " ".join(v) + if len(v) > 0: + v = " ".join(v) + else: + continue if k == "macro" and command == CliCommand.RUN_OPERATION: add_fn(v) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 8b28224eac5..5dbeae13697 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -194,6 +194,7 @@ def cli(ctx, **kwargs): @p.profiles_dir @p.project_dir @p.resource_type +@p.exclude_resource_type @p.select @p.selector @p.show @@ -499,6 +500,7 @@ def init(ctx, **kwargs): @p.profiles_dir @p.project_dir @p.resource_type +@p.exclude_resource_type @p.raw_select @p.selector @p.target @@ -627,6 +629,7 @@ def retry(ctx, **kwargs): @p.profiles_dir @p.project_dir @p.resource_type +@p.exclude_resource_type @p.select @p.selector @p.target diff --git a/core/dbt/cli/params.py b/core/dbt/cli/params.py index 291ac7b44c4..1ffeb53c94f 100644 --- a/core/dbt/cli/params.py +++ b/core/dbt/cli/params.py @@ -391,7 +391,7 @@ resource_type = click.option( "--resource-types", "--resource-type", - envvar=None, + envvar="DBT_RESOURCE_TYPES", help="Restricts the types of resources that dbt will include", type=ChoiceTuple( [ @@ -402,6 +402,7 @@ "analysis", "model", "test", + "unit_test", "exposure", "snapshot", "seed", @@ -415,6 +416,33 @@ default=(), ) +exclude_resource_type = click.option( + "--exclude-resource-types", + "--exclude-resource-type", + envvar="DBT_EXCLUDE_RESOURCE_TYPES", + help="Specify the types of resources that dbt will exclude", + type=ChoiceTuple( + [ + "metric", + "semantic_model", + "saved_query", + "source", + "analysis", + "model", + "test", + "unit_test", + "exposure", + "snapshot", + "seed", + "default", + ], + case_sensitive=False, + ), + cls=MultiOption, + multiple=True, + default=(), +) + # Renamed to --export-saved-queries deprecated_include_saved_query = click.option( "--include-saved-query/--no-include-saved-query", diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 690ae36a71b..fcc03adb170 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -6,7 +6,7 @@ from contextlib import nullcontext from datetime import datetime from pathlib import Path -from typing import Any, Dict, List, Optional, Type, Union +from typing import Any, Dict, List, Optional, Type, Union, Set from dbt.compilation import Compiler import dbt_common.exceptions.base @@ -16,6 +16,7 @@ from dbt.config.profile import read_profile from dbt.constants import DBT_PROJECT_FILE_NAME from dbt.contracts.graph.manifest import Manifest +from dbt.artifacts.resources.types import NodeType from dbt.artifacts.schemas.results import TimingInfo, collect_timing_info from dbt.artifacts.schemas.results import NodeStatus, RunningStatus, RunStatus from dbt.artifacts.schemas.run import RunResult @@ -480,3 +481,30 @@ def on_skip(self): def do_skip(self, cause=None): self.skip = True self.skip_cause = cause + + +def resource_types_from_args( + args, all_resource_values: Set[NodeType], default_resource_values: Set[NodeType] +) -> Set[NodeType]: + + if not args.resource_types: + resource_types = default_resource_values + else: + # This is a list of strings, not NodeTypes + arg_resource_types = set(args.resource_types) + + if "all" in arg_resource_types: + arg_resource_types.remove("all") + arg_resource_types.update(all_resource_values) + if "default" in arg_resource_types: + arg_resource_types.remove("default") + arg_resource_types.update(default_resource_values) + # Convert to a set of NodeTypes now that the non-NodeType strings are gone + resource_types = set([NodeType(rt) for rt in arg_resource_types]) + + if args.exclude_resource_types: + # Convert from a list of strings to a set of NodeTypes + exclude_resource_types = set([NodeType(rt) for rt in args.exclude_resource_types]) + resource_types = resource_types - exclude_resource_types + + return resource_types diff --git a/core/dbt/task/build.py b/core/dbt/task/build.py index 7c11839942a..5d3a42b3b9f 100644 --- a/core/dbt/task/build.py +++ b/core/dbt/task/build.py @@ -11,7 +11,7 @@ from dbt.graph import ResourceTypeSelector, GraphQueue, Graph from dbt.node_types import NodeType from dbt.task.test import TestSelector -from dbt.task.base import BaseRunner +from dbt.task.base import BaseRunner, resource_types_from_args from dbt_common.events.functions import fire_event from dbt.events.types import LogNodeNoOpResult from dbt.exceptions import DbtInternalError @@ -80,14 +80,9 @@ def __init__(self, args, config, manifest) -> None: self.model_to_unit_test_map: Dict[str, List] = {} def resource_types(self, no_unit_tests=False): - if not self.args.resource_types: - resource_types = list(self.ALL_RESOURCE_VALUES) - else: - resource_types = set(self.args.resource_types) - - if "all" in resource_types: - resource_types.remove("all") - resource_types.update(self.ALL_RESOURCE_VALUES) + resource_types = resource_types_from_args( + self.args, set(self.ALL_RESOURCE_VALUES), set(self.ALL_RESOURCE_VALUES) + ) # First we get selected_nodes including unit tests, then without, # and do a set difference. diff --git a/core/dbt/task/clone.py b/core/dbt/task/clone.py index 53c322211cb..44f6a30841d 100644 --- a/core/dbt/task/clone.py +++ b/core/dbt/task/clone.py @@ -9,8 +9,8 @@ from dbt_common.dataclass_schema import dbtClassMixin from dbt_common.exceptions import DbtInternalError, CompilationError from dbt.graph import ResourceTypeSelector -from dbt.node_types import NodeType, REFABLE_NODE_TYPES -from dbt.task.base import BaseRunner +from dbt.node_types import REFABLE_NODE_TYPES +from dbt.task.base import BaseRunner, resource_types_from_args from dbt.task.run import _validate_materialization_relations_dict from dbt.task.runnable import GraphRunnableTask @@ -132,18 +132,13 @@ def before_run(self, adapter, selected_uids: AbstractSet[str]): @property def resource_types(self): - if not self.args.resource_types: - return REFABLE_NODE_TYPES - - values = set(self.args.resource_types) - - if "all" in values: - values.remove("all") - values.update(REFABLE_NODE_TYPES) - - values = [NodeType(val) for val in values if val in REFABLE_NODE_TYPES] + resource_types = resource_types_from_args( + self.args, set(REFABLE_NODE_TYPES), set(REFABLE_NODE_TYPES) + ) - return list(values) + # filter out any non-refable node types + resource_types = [rt for rt in resource_types if rt in REFABLE_NODE_TYPES] + return list(resource_types) def get_node_selector(self) -> ResourceTypeSelector: resource_types = self.resource_types diff --git a/core/dbt/task/list.py b/core/dbt/task/list.py index d71ff3faf18..ff6fea3447e 100644 --- a/core/dbt/task/list.py +++ b/core/dbt/task/list.py @@ -10,6 +10,7 @@ ) from dbt.flags import get_flags from dbt.graph import ResourceTypeSelector +from dbt.task.base import resource_types_from_args from dbt.task.runnable import GraphRunnableTask from dbt.task.test import TestSelector from dbt.node_types import NodeType @@ -183,17 +184,11 @@ def resource_types(self): if self.args.models: return [NodeType.Model] - if not self.args.resource_types: - return list(self.DEFAULT_RESOURCE_VALUES) - - values = set(self.args.resource_types) - if "default" in values: - values.remove("default") - values.update(self.DEFAULT_RESOURCE_VALUES) - if "all" in values: - values.remove("all") - values.update(self.ALL_RESOURCE_VALUES) - return list(values) + resource_types = resource_types_from_args( + self.args, set(self.ALL_RESOURCE_VALUES), set(self.DEFAULT_RESOURCE_VALUES) + ) + + return list(resource_types) @property def selection_arg(self): diff --git a/tests/functional/unit_testing/test_unit_testing.py b/tests/functional/unit_testing/test_unit_testing.py index ffa3d0e34b2..c1cb048dbfa 100644 --- a/tests/functional/unit_testing/test_unit_testing.py +++ b/tests/functional/unit_testing/test_unit_testing.py @@ -50,12 +50,22 @@ def test_basic(self, project): results = run_dbt(["test", "--select", "my_model"], expect_pass=False) assert len(results) == 5 - results = run_dbt(["build", "--select", "my_model"], expect_pass=False) + results = run_dbt( + ["build", "--select", "my_model", "--resource-types", "model unit_test"], + expect_pass=False, + ) assert len(results) == 6 for result in results: if result.node.unique_id == "model.test.my_model": result.status == NodeStatus.Skipped + # Run build command but specify no unit tests + results = run_dbt( + ["build", "--select", "my_model", "--exclude-resource-types", "unit_test"], + expect_pass=True, + ) + assert len(results) == 1 + # Test select by test name results = run_dbt(["test", "--select", "test_name:test_my_model_string_concat"]) assert len(results) == 1