Skip to content

Commit

Permalink
Clearer no-op logging in stubbed SavedQueryRunner (#9605)
Browse files Browse the repository at this point in the history
* Clearer no-op logging in stubbed SavedQueryRunner

* Add changelog entry

* Fix unit test

* More logging touchups

* Fix failing test

* Rename flag + refactor per #9629

* Fix failing test

* regenerate core_proto_types with libprotoc 25.3

---------

Co-authored-by: Michelle Ark <[email protected]>
  • Loading branch information
jtcohen6 and MichelleArk authored Feb 27, 2024
1 parent 7329143 commit 865b09b
Show file tree
Hide file tree
Showing 12 changed files with 322 additions and 283 deletions.
6 changes: 6 additions & 0 deletions .changes/unreleased/Fixes-20240220-165453.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: Clearer no-op logging in stubbed SavedQueryRunner
time: 2024-02-20T16:54:53.623096-05:00
custom:
Author: jtcohen6
Issue: "9533"
3 changes: 2 additions & 1 deletion core/dbt/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,8 +187,9 @@ def cli(ctx, **kwargs):
@click.pass_context
@global_flags
@p.exclude
@p.export_saved_queries
@p.full_refresh
@p.include_saved_query
@p.deprecated_include_saved_query
@p.profile
@p.profiles_dir
@p.project_dir
Expand Down
11 changes: 10 additions & 1 deletion core/dbt/cli/params.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,14 @@
help="Specify the nodes to exclude.",
)

export_saved_queries = click.option(
"--export-saved-queries/--no-export-saved-queries",
envvar="DBT_EXPORT_SAVED_QUERIES",
help="Export saved queries within the 'build' command, otherwise no-op",
is_flag=True,
hidden=True,
)

fail_fast = click.option(
"--fail-fast/--no-fail-fast",
"-x/ ",
Expand Down Expand Up @@ -406,7 +414,8 @@
default=(),
)

include_saved_query = click.option(
# Renamed to --export-saved-queries
deprecated_include_saved_query = click.option(
"--include-saved-query/--no-include-saved-query",
envvar="DBT_INCLUDE_SAVED_QUERY",
help="Include saved queries in the list of resources to be selected for build command",
Expand Down
5 changes: 4 additions & 1 deletion core/dbt/compilation.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@ def print_compile_stats(stats: Dict[NodeType, int]):
dbt.tracking.track_resource_counts(resource_counts)

# do not include resource types that are not actually defined in the project
stat_line = ", ".join([pluralize(ct, t) for t, ct in stats.items() if ct != 0])
stat_line = ", ".join(
[pluralize(ct, t).replace("_", " ") for t, ct in stats.items() if ct != 0]
)
fire_event(FoundStats(stat_line=stat_line))


Expand All @@ -77,6 +79,7 @@ def _generate_stats(manifest: Manifest) -> Dict[NodeType, int]:
stats[NodeType.Macro] += len(manifest.macros)
stats[NodeType.Group] += len(manifest.groups)
stats[NodeType.SemanticModel] += len(manifest.semantic_models)
stats[NodeType.SavedQuery] += len(manifest.saved_queries)
stats[NodeType.Unit] += len(manifest.unit_tests)

# TODO: should we be counting dimensions + entities?
Expand Down
17 changes: 16 additions & 1 deletion core/dbt/events/core_types.proto
Original file line number Diff line number Diff line change
Expand Up @@ -1331,8 +1331,23 @@ message LogFreshnessResultMsg {
LogFreshnessResult data = 2;
}

// Q018
message LogNodeNoOpResult {
NodeInfo node_info = 1;
string description = 2;
string status = 3;
int32 index = 4;
int32 total = 5;
float execution_time = 6;
}

message LogNodeNoOpResultMsg {
CoreEventInfo info = 1;
LogNodeNoOpResult data = 2;
}


// Skipped Q019, Q020, Q021
// Skipped Q020, Q021


// Q022
Expand Down
490 changes: 247 additions & 243 deletions core/dbt/events/core_types_pb2.py

Large diffs are not rendered by default.

17 changes: 16 additions & 1 deletion core/dbt/events/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -1348,7 +1348,22 @@ def status_to_level(cls, status):
return EventLevel.INFO


# Skipped Q019, Q020, Q021
class LogNodeNoOpResult(InfoLevel):
def code(self) -> str:
return "Q019"

def message(self) -> str:
msg = f"NO-OP {self.description}"
return format_fancy_output_line(
msg=msg,
status=yellow("NO-OP"),
index=self.index,
total=self.total,
execution_time=self.execution_time,
)


# Skipped Q020, Q021


class LogCancelLine(ErrorLevel):
Expand Down
36 changes: 9 additions & 27 deletions core/dbt/task/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,45 +13,30 @@
from dbt.task.test import TestSelector
from dbt.task.base import BaseRunner
from dbt_common.events.functions import fire_event
from dbt.events.types import LogStartLine, LogModelResult
from dbt_common.events.base_types import EventLevel
from dbt.events.types import LogNodeNoOpResult
from dbt.exceptions import DbtInternalError


class SavedQueryRunner(BaseRunner):
# A no-op Runner for Saved Queries
# Stub. No-op Runner for Saved Queries, which require MetricFlow for execution.
@property
def description(self):
return "Saved Query {}".format(self.node.unique_id)
return f"saved query {self.node.name}"

def before_execute(self):
fire_event(
LogStartLine(
description=self.description,
index=self.node_index,
total=self.num_nodes,
node_info=self.node.node_info,
)
)
pass

def compile(self, manifest):
return self.node

def after_execute(self, result):
if result.status == NodeStatus.Error:
level = EventLevel.ERROR
else:
level = EventLevel.INFO
fire_event(
LogModelResult(
LogNodeNoOpResult(
description=self.description,
status=result.status,
index=self.node_index,
total=self.num_nodes,
execution_time=result.execution_time,
node_info=self.node.node_info,
),
level=level,
)
)

def execute(self, compiled_node, manifest):
Expand All @@ -61,8 +46,8 @@ def execute(self, compiled_node, manifest):
status=RunStatus.Success,
timing=[],
thread_id=threading.current_thread().name,
execution_time=0.1,
message="done",
execution_time=0,
message="NO-OP",
adapter_response={},
failures=0,
agate_table=None,
Expand All @@ -85,6 +70,7 @@ class BuildTask(RunTask):
NodeType.Seed: seed_runner,
NodeType.Test: test_runner,
NodeType.Unit: test_runner,
NodeType.SavedQuery: SavedQueryRunner,
}
ALL_RESOURCE_VALUES = frozenset({x for x in RUNNER_MAP.keys()})

Expand All @@ -94,10 +80,6 @@ def __init__(self, args, config, manifest) -> None:
self.model_to_unit_test_map: Dict[str, List] = {}

def resource_types(self, no_unit_tests=False):
if self.args.include_saved_query:
self.RUNNER_MAP[NodeType.SavedQuery] = SavedQueryRunner
self.ALL_RESOURCE_VALUES = self.ALL_RESOURCE_VALUES.union({NodeType.SavedQuery})

if not self.args.resource_types:
resource_types = list(self.ALL_RESOURCE_VALUES)
else:
Expand Down
4 changes: 2 additions & 2 deletions core/dbt/task/printer.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,11 @@ def get_counts(flat_nodes) -> str:
if node.resource_type == NodeType.Model:
t = "{} {}".format(node.get_materialization(), t)
elif node.resource_type == NodeType.Operation:
t = "hook"
t = "project hook"

counts[t] = counts.get(t, 0) + 1

stat_line = ", ".join([pluralize(v, k) for k, v in counts.items()])
stat_line = ", ".join([pluralize(v, k).replace("_", " ") for k, v in counts.items()])

return stat_line

Expand Down
2 changes: 1 addition & 1 deletion tests/functional/minimal_cli/test_minimal_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def test_ls(self, runner, project):
ls_result = runner.invoke(cli, ["ls"])
assert "1 seed" in ls_result.output
assert "1 model" in ls_result.output
assert "5 data_tests" in ls_result.output
assert "5 data tests" in ls_result.output
assert "1 snapshot" in ls_result.output


Expand Down
7 changes: 2 additions & 5 deletions tests/functional/saved_queries/test_saved_query_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,8 @@ def packages(self):
version: 1.1.1
"""

def test_semantic_model_parsing(self, project):
def test_build_saved_queries(self, project):
run_dbt(["deps"])
result = run_dbt(["build"])
assert len(result.results) == 2
assert "test_saved_query" not in [r.node.name for r in result.results]
result = run_dbt(["build", "--include-saved-query"])
assert len(result.results) == 3
assert "test_saved_query" in [r.node.name for r in result.results]
assert "NO-OP" in [r.message for r in result.results]
7 changes: 7 additions & 0 deletions tests/unit/test_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,13 @@ def test_event_codes(self):
total=0,
execution_time=0,
),
core_types.LogNodeNoOpResult(
description="",
status="",
index=0,
total=0,
execution_time=0,
),
core_types.LogCancelLine(conn_name=""),
core_types.DefaultSelector(name=""),
core_types.NodeStart(),
Expand Down

0 comments on commit 865b09b

Please sign in to comment.