Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix deprecation firing for microbatch model w custom strategy #10989

Merged
merged 7 commits into from
Nov 13, 2024
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/Fixes-20241112-210839.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: Correct when custom microbatch macro deprecation warning is fired
time: 2024-11-12T21:08:39.866837-06:00
custom:
Author: QMalcolm MichelleArk
Issue: "10994"
2 changes: 1 addition & 1 deletion core/dbt/parser/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -662,7 +662,7 @@
has_microbatch_model = True
break

if has_microbatch_model and self.manifest._microbatch_macro_is_core(
if has_microbatch_model and not self.manifest._microbatch_macro_is_core(

Check warning on line 665 in core/dbt/parser/manifest.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/parser/manifest.py#L665

Added line #L665 was not covered by tests
self.root_project.project_name
):
dbt.deprecations.warn("microbatch-macro-outside-of-batches-deprecation")
Expand Down
2 changes: 1 addition & 1 deletion dev-requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
git+https://github.com/dbt-labs/dbt-adapters.git@main
git+https://github.com/dbt-labs/dbt-adapters.git@fix-microbatch-behavior-flag-conditional
git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter
MichelleArk marked this conversation as resolved.
Show resolved Hide resolved
git+https://github.com/dbt-labs/dbt-common.git@main
git+https://github.com/dbt-labs/dbt-postgres.git@main
Expand Down
42 changes: 23 additions & 19 deletions tests/functional/microbatch/test_microbatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,19 +182,16 @@ def test_use_custom_microbatch_strategy_by_default(
project,
deprecation_catcher: EventCatcher,
):
with mock.patch.object(
type(project.adapter), "valid_incremental_strategies", lambda _: []
):
# Initial run
run_dbt(["run"])
# Initial run fires deprecation
run_dbt(["run"], callbacks=[deprecation_catcher.catch])
# Deprecation warning about custom microbatch macro fired
assert len(deprecation_catcher.caught_events) == 1

# Incremental run uses custom strategy
_, logs = run_dbt_and_capture(["run"])
assert "custom microbatch strategy" in logs
# The custom strategy wasn't used with batch functionality
assert "START batch" not in logs
# Deprecation warning about custom microbatch macro fired
assert len(deprecation_catcher.caught_events) == 0
# Incremental run uses custom strategy
_, logs = run_dbt_and_capture(["run"])
assert "custom microbatch strategy" in logs
# The custom strategy wasn't used with batch functionality
assert "START batch" not in logs


class TestMicrobatchCustomUserStrategyProjectFlagTrueValid(BaseMicrobatchCustomUserStrategy):
Expand All @@ -208,30 +205,37 @@ def test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_st
):
# Initial run
with patch_microbatch_end_time("2020-01-03 13:57:00"):
run_dbt(["run"])
run_dbt(["run"], callbacks=[deprecation_catcher.catch])
# Deprecation warning about custom microbatch macro not fired
assert len(deprecation_catcher.caught_events) == 0

# Incremental run uses custom strategy
with patch_microbatch_end_time("2020-01-03 13:57:00"):
_, logs = run_dbt_and_capture(["run"])
assert "custom microbatch strategy" in logs
# The custom strategy was used with batch functionality
assert "START batch" in logs
# Deprecation warning about custom microbatch macro not fired
assert len(deprecation_catcher.caught_events) == 0


class TestMicrobatchCustomUserStrategyProjectFlagTrueInvalid(BaseMicrobatchCustomUserStrategy):
class TestMicrobatchCustomUserStrategyProjectFlagTrueNoValidBuiltin(
BaseMicrobatchCustomUserStrategy
):
def test_use_custom_microbatch_strategy_project_flag_true_invalid_incremental_strategy(
self, project
):
with mock.patch.object(
type(project.adapter), "valid_incremental_strategies", lambda _: []
):
# Run of microbatch model while adapter doesn't have a "valid"
# microbatch strategy causes an error to be raised
# microbatch strategy causes no error when behaviour flag set to true
# and there is a custom microbatch macro
with patch_microbatch_end_time("2020-01-03 13:57:00"):
_, logs = run_dbt_and_capture(["run"], expect_pass=False)
assert "'microbatch' is not valid" in logs
_, logs = run_dbt_and_capture(["run"])
assert "'microbatch' is not valid" not in logs
assert (
"The use of a custom microbatch macro outside of batched execution is deprecated"
not in logs
)


class BaseMicrobatchTest:
Expand Down
Loading