Skip to content

Commit

Permalink
Change microbatch lookback default from 0 to 1 (#10876)
Browse files Browse the repository at this point in the history 
* Change `lookback` default from `0` to `1`

* Regen jsonschema manifest v12 to include `lookback` default change

* Regen saved state of v12 manifest for functional artifact testing

* Add changie doc for lookback default change
  • Loading branch information
@QMalcolm
QMalcolm authored Oct 24, 2024
1 parent 8ae689c commit d07bfda
Show file tree
Hide file tree
Showing 9 changed files with 53 additions and 47 deletions.
6 changes: 6 additions & 0 deletions .changes/unreleased/Fixes-20241017-134857.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: Change `lookback` default from `0` to `1` to ensure better data completeness
time: 2024-10-17T13:48:57.805205-07:00
custom:
Author: QMalcolm
Issue: "10867"
2 changes: 1 addition & 1 deletion core/dbt/artifacts/resources/v1/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ class NodeConfig(NodeAndTestConfig):
materialized: str = "view"
incremental_strategy: Optional[str] = None
batch_size: Any = None
lookback: Any = 0
lookback: Any = 1
begin: Any = None
persist_docs: Dict[str, Any] = field(default_factory=dict)
post_hook: List[Hook] = field(
Expand Down
36 changes: 18 additions & 18 deletions schemas/dbt/manifest/v12.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -931,7 +931,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -1317,7 +1317,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -2762,7 +2762,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -3564,7 +3564,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -4493,7 +4493,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -4933,7 +4933,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -6439,7 +6439,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -7360,7 +7360,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -10154,7 +10154,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -10822,7 +10822,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -11208,7 +11208,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -12653,7 +12653,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -13455,7 +13455,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -14384,7 +14384,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -14824,7 +14824,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -16330,7 +16330,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down Expand Up @@ -17251,7 +17251,7 @@
"default": null
},
"lookback": {
"default": 0
"default": 1
},
"begin": {
"default": null
Expand Down
2 changes: 1 addition & 1 deletion tests/functional/artifacts/data/state/v12/manifest.json
View file

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions tests/functional/artifacts/expected_manifest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def get_rendered_model_config(**updates):
"contract": {"enforced": False, "alias_types": True},
"access": "protected",
"event_time": None,
"lookback": 0,
"lookback": 1,
"batch_size": None,
"begin": None,
}
Expand Down Expand Up @@ -79,7 +79,7 @@ def get_rendered_seed_config(**updates):
"docs": {"node_color": None, "show": True},
"contract": {"enforced": False, "alias_types": True},
"event_time": None,
"lookback": 0,
"lookback": 1,
"batch_size": None,
"begin": None,
}
Expand Down Expand Up @@ -130,7 +130,7 @@ def get_rendered_snapshot_config(**updates):
"docs": {"node_color": None, "show": True},
"contract": {"enforced": False, "alias_types": True},
"event_time": None,
"lookback": 0,
"lookback": 1,
"batch_size": None,
"begin": None,
}
Expand Down
18 changes: 9 additions & 9 deletions tests/functional/list/test_list.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ def expect_snapshot_output(self, happy_path_project): # noqa: F811
"docs": {"node_color": None, "show": True},
"contract": {"enforced": False, "alias_types": True},
"event_time": None,
"lookback": 0,
"lookback": 1,
"batch_size": None,
"begin": None,
},
Expand Down Expand Up @@ -133,7 +133,7 @@ def expect_analyses_output(self):
"docs": {"node_color": None, "show": True},
"contract": {"enforced": False, "alias_types": True},
"event_time": None,
"lookback": 0,
"lookback": 1,
"batch_size": None,
"begin": None,
},
Expand Down Expand Up @@ -198,7 +198,7 @@ def expect_model_output(self):
"contract": {"enforced": False, "alias_types": True},
"access": "protected",
"event_time": None,
"lookback": 0,
"lookback": 1,
"batch_size": None,
"begin": None,
},
Expand Down Expand Up @@ -240,7 +240,7 @@ def expect_model_output(self):
"contract": {"enforced": False, "alias_types": True},
"access": "protected",
"event_time": None,
"lookback": 0,
"lookback": 1,
"batch_size": None,
"begin": None,
},
Expand Down Expand Up @@ -282,7 +282,7 @@ def expect_model_output(self):
"contract": {"enforced": False, "alias_types": True},
"access": "protected",
"event_time": None,
"lookback": 0,
"lookback": 1,
"batch_size": None,
"begin": None,
},
Expand Down Expand Up @@ -324,7 +324,7 @@ def expect_model_output(self):
"contract": {"enforced": False, "alias_types": True},
"access": "protected",
"event_time": None,
"lookback": 0,
"lookback": 1,
"batch_size": None,
"begin": None,
},
Expand Down Expand Up @@ -366,7 +366,7 @@ def expect_model_output(self):
"contract": {"enforced": False, "alias_types": True},
"access": "protected",
"event_time": None,
"lookback": 0,
"lookback": 1,
"batch_size": None,
"begin": None,
},
Expand Down Expand Up @@ -408,7 +408,7 @@ def expect_model_output(self):
"contract": {"enforced": False, "alias_types": True},
"access": "protected",
"event_time": None,
"lookback": 0,
"lookback": 1,
"batch_size": None,
"begin": None,
},
Expand Down Expand Up @@ -531,7 +531,7 @@ def expect_seed_output(self):
"docs": {"node_color": None, "show": True},
"contract": {"enforced": False, "alias_types": True},
"event_time": None,
"lookback": 0,
"lookback": 1,
"batch_size": None,
"begin": None,
},
Expand Down
2 changes: 1 addition & 1 deletion tests/unit/context/test_providers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def test_resolve_event_time_filter(
resolver.model.config.materialized = materialized
resolver.model.config.incremental_strategy = incremental_strategy
resolver.model.config.batch_size = BatchSize.day
resolver.model.config.lookback = 0
resolver.model.config.lookback = 1

# Try to get an EventTimeFilter
event_time_filter = resolver.resolve_event_time_filter(target=target)
Expand Down
2 changes: 1 addition & 1 deletion tests/unit/contracts/graph/test_nodes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ def basic_compiled_dict():
"contract": {"enforced": False, "alias_types": True},
"docs": {"show": True},
"access": "protected",
"lookback": 0,
"lookback": 1,
},
"docs": {"show": True},
"columns": {},
Expand Down
Loading

0 comments on commit d07bfda

Please sign in to comment.