Merge branch 'main' into qmalcolm--10862-ensure-keyboard-interrupt-in…

…terrupts-microbatch-model-execution

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.9.0b2
+current_version = 1.9.0b3
 parse = (?P<major>[\d]+) # major version number
 	\.(?P<minor>[\d]+) # minor version number
 	\.(?P<patch>[\d]+) # patch version number

.changes/1.9.0-b3.md

@@ -0,0 +1,38 @@
+## dbt-core 1.9.0-b3 - October 30, 2024
+
+### Features
+
+- Add `order_by` and `limit` fields to saved queries. ([#10531](https://github.com/dbt-labs/dbt-core/issues/10531))
+- Enable specification of dbt_valid_to for current records ([#10187](https://github.com/dbt-labs/dbt-core/issues/10187))
+- Enable use of multi-column unique key in snapshots ([#9992](https://github.com/dbt-labs/dbt-core/issues/9992))
+- Ensure `--event-time-start` is before `--event-time-end` ([#10786](https://github.com/dbt-labs/dbt-core/issues/10786))
+- Ensure microbatch models use same `current_time` value ([#10819](https://github.com/dbt-labs/dbt-core/issues/10819))
+- Emit warning when microbatch model has no input with `event_time` config ([#10926](https://github.com/dbt-labs/dbt-core/issues/10926))
+
+### Fixes
+
+- Pass test user config to adapter pre_hook by explicitly adding test builder config to node ([#10484](https://github.com/dbt-labs/dbt-core/issues/10484))
+- Handle edge cases when a specified `--event-time-end` is equivalent to the batch size truncated batch start time ([#10824](https://github.com/dbt-labs/dbt-core/issues/10824))
+- Begin tracking execution time of microbatch model batches ([#10825](https://github.com/dbt-labs/dbt-core/issues/10825))
+- Allow instances of generic data tests to be documented ([#2578](https://github.com/dbt-labs/dbt-core/issues/2578))
+- Fix warnings for models referring to a deprecated model ([#10833](https://github.com/dbt-labs/dbt-core/issues/10833))
+- Change `lookback` default from `0` to `1` to ensure better data completeness ([#10867](https://github.com/dbt-labs/dbt-core/issues/10867))
+- Make `--event-time-start` and `--event-time-end` mutually required ([#10874](https://github.com/dbt-labs/dbt-core/issues/10874))
+- Exclude hook result from results in on-run-end context ([#7387](https://github.com/dbt-labs/dbt-core/issues/7387))
+- Implement partial parsing for all-yaml snapshots ([#10903](https://github.com/dbt-labs/dbt-core/issues/10903))
+- Restore source quoting behaviour when quoting config provided in dbt_project.yml ([#10892](https://github.com/dbt-labs/dbt-core/issues/10892))
+- Fix bug when referencing deprecated models ([#10915](https://github.com/dbt-labs/dbt-core/issues/10915))
+- Fix 'model' jinja context variable type to dict  ([#10927](https://github.com/dbt-labs/dbt-core/issues/10927))
+- Take `end_time` for batches to the ceiling to handle edge case where `event_time` column is a date ([#10868](https://github.com/dbt-labs/dbt-core/issues/10868))
+
+### Under the Hood
+
+- Remove support and testing for Python 3.8, which is now EOL. ([#10861](https://github.com/dbt-labs/dbt-core/issues/10861))
+
+### Dependencies
+
+- Bump minimnum allowed dbt-adapters version to 1.8.0 ([#N/A](https://github.com/dbt-labs/dbt-core/issues/N/A))
+
+### Contributors
+- [@danlsn](https://github.com/danlsn) ([#10915](https://github.com/dbt-labs/dbt-core/issues/10915))
+- [@plypaul](https://github.com/plypaul) ([#10531](https://github.com/dbt-labs/dbt-core/issues/10531))

.changes/1.9.0/Dependencies-20241030-134526.yaml

@@ -0,0 +1,6 @@
+kind: Dependencies
+body: Bump minimnum allowed dbt-adapters version to 1.8.0
+time: 2024-10-30T13:45:26.144328-05:00
+custom:
+  Author: QMalcolm
+  Issue: N/A

.changes/1.9.0/Features-20241001-134051.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Enable use of multi-column unique key in snapshots
+time: 2024-10-01T13:40:51.297529-04:00
+custom:
+  Author: gshank
+  Issue: "9992"

.changes/1.9.0/Features-20241028-173419.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Emit warning when microbatch model has no input with `event_time` config
+time: 2024-10-28T17:34:19.195209-04:00
+custom:
+  Author: michelleark
+  Issue: "10926"

.changes/1.9.0/Fixes-20241017-134857.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Change `lookback` default from `0` to `1` to ensure better data completeness
+time: 2024-10-17T13:48:57.805205-07:00
+custom:
+  Author: QMalcolm
+  Issue: "10867"

.changes/1.9.0/Fixes-20241017-145357.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Make `--event-time-start` and `--event-time-end` mutually required
+time: 2024-10-17T14:53:57.149238-07:00
+custom:
+  Author: QMalcolm
+  Issue: "10874"

.changes/1.9.0/Fixes-20241018-135810.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Exclude hook result from results in on-run-end context
+time: 2024-10-18T13:58:10.396884-07:00
+custom:
+  Author: ChenyuLInx
+  Issue: "7387"

.changes/1.9.0/Fixes-20241022-222927.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Implement partial parsing for all-yaml snapshots
+time: 2024-10-22T22:29:27.396378-04:00
+custom:
+  Author: gshank
+  Issue: "10903"

.changes/1.9.0/Fixes-20241023-152054.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Restore source quoting behaviour when quoting config provided in dbt_project.yml
+time: 2024-10-23T15:20:54.766893-04:00
+custom:
+  Author: michelleark
+  Issue: "10892"

.changes/1.9.0/Fixes-20241024-104938.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix bug when referencing deprecated models
+time: 2024-10-24T10:49:38.352328-06:00
+custom:
+  Author: dbeatty10 danlsn
+  Issue: "10915"

.changes/1.9.0/Fixes-20241028-132751.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: 'Fix ''model'' jinja context variable type to dict '
+time: 2024-10-28T13:27:51.604093-04:00
+custom:
+  Author: michelleark
+  Issue: "10927"

.changes/1.9.0/Fixes-20241029-161615.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Take `end_time` for batches to the ceiling to handle edge case where `event_time`
+  column is a date
+time: 2024-10-29T16:16:15.714993-05:00
+custom:
+  Author: QMalcolm MichelleArk
+  Issue: "10868"

.changes/unreleased/Features-20241029-181728.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Emit debug logging event whenever artifacts are written
+time: 2024-10-29T18:17:28.321188-05:00
+custom:
+  Author: QMalcolm
+  Issue: 10937

.changes/unreleased/Fixes-20241029-182034.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Handle exceptions in `get_execution_status` more broadly to better ensure `run_results.json`
+  gets written
+time: 2024-10-29T18:20:34.782845-05:00
+custom:
+  Author: QMalcolm
+  Issue: "10934"

CHANGELOG.md

@@ -5,9 +5,47 @@
 - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
 - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry)
 
-## dbt-core 1.9.0-b2 - October 07, 2024
+## dbt-core 1.9.0-b3 - October 30, 2024
+
+### Features
+
+- Add `order_by` and `limit` fields to saved queries. ([#10531](https://github.com/dbt-labs/dbt-core/issues/10531))
+- Enable specification of dbt_valid_to for current records ([#10187](https://github.com/dbt-labs/dbt-core/issues/10187))
+- Enable use of multi-column unique key in snapshots ([#9992](https://github.com/dbt-labs/dbt-core/issues/9992))
+- Ensure `--event-time-start` is before `--event-time-end` ([#10786](https://github.com/dbt-labs/dbt-core/issues/10786))
+- Ensure microbatch models use same `current_time` value ([#10819](https://github.com/dbt-labs/dbt-core/issues/10819))
+- Emit warning when microbatch model has no input with `event_time` config ([#10926](https://github.com/dbt-labs/dbt-core/issues/10926))
+
+### Fixes
+
+- Pass test user config to adapter pre_hook by explicitly adding test builder config to node ([#10484](https://github.com/dbt-labs/dbt-core/issues/10484))
+- Handle edge cases when a specified `--event-time-end` is equivalent to the batch size truncated batch start time ([#10824](https://github.com/dbt-labs/dbt-core/issues/10824))
+- Begin tracking execution time of microbatch model batches ([#10825](https://github.com/dbt-labs/dbt-core/issues/10825))
+- Allow instances of generic data tests to be documented ([#2578](https://github.com/dbt-labs/dbt-core/issues/2578))
+- Fix warnings for models referring to a deprecated model ([#10833](https://github.com/dbt-labs/dbt-core/issues/10833))
+- Change `lookback` default from `0` to `1` to ensure better data completeness ([#10867](https://github.com/dbt-labs/dbt-core/issues/10867))
+- Make `--event-time-start` and `--event-time-end` mutually required ([#10874](https://github.com/dbt-labs/dbt-core/issues/10874))
+- Exclude hook result from results in on-run-end context ([#7387](https://github.com/dbt-labs/dbt-core/issues/7387))
+- Implement partial parsing for all-yaml snapshots ([#10903](https://github.com/dbt-labs/dbt-core/issues/10903))
+- Restore source quoting behaviour when quoting config provided in dbt_project.yml ([#10892](https://github.com/dbt-labs/dbt-core/issues/10892))
+- Fix bug when referencing deprecated models ([#10915](https://github.com/dbt-labs/dbt-core/issues/10915))
+- Fix 'model' jinja context variable type to dict  ([#10927](https://github.com/dbt-labs/dbt-core/issues/10927))
+- Take `end_time` for batches to the ceiling to handle edge case where `event_time` column is a date ([#10868](https://github.com/dbt-labs/dbt-core/issues/10868))
+
+### Under the Hood
+
+- Remove support and testing for Python 3.8, which is now EOL. ([#10861](https://github.com/dbt-labs/dbt-core/issues/10861))
+
+### Dependencies
 
+- Bump minimnum allowed dbt-adapters version to 1.8.0 ([#N/A](https://github.com/dbt-labs/dbt-core/issues/N/A))
 
+### Contributors
+- [@danlsn](https://github.com/danlsn) ([#10915](https://github.com/dbt-labs/dbt-core/issues/10915))
+- [@plypaul](https://github.com/plypaul) ([#10531](https://github.com/dbt-labs/dbt-core/issues/10531))
+
+
+## dbt-core 1.9.0-b2 - October 07, 2024
 
 ## dbt-core 1.9.0-b1 - October 01, 2024
 

core/dbt/artifacts/resources/v1/config.py

@@ -81,7 +81,7 @@ class NodeConfig(NodeAndTestConfig):
     materialized: str = "view"
     incremental_strategy: Optional[str] = None
     batch_size: Any = None
-    lookback: Any = 0
+    lookback: Any = 1
     begin: Any = None
     persist_docs: Dict[str, Any] = field(default_factory=dict)
     post_hook: List[Hook] = field(

core/dbt/artifacts/resources/v1/snapshot.py

@@ -19,7 +19,7 @@ class SnapshotMetaColumnNames(dbtClassMixin):
 class SnapshotConfig(NodeConfig):
     materialized: str = "snapshot"
     strategy: Optional[str] = None
-    unique_key: Optional[str] = None
+    unique_key: Optional[Union[str, List[str]]] = None
     target_schema: Optional[str] = None
     target_database: Optional[str] = None
     updated_at: Optional[str] = None

core/dbt/artifacts/schemas/results.py

@@ -10,6 +10,12 @@
 
 @dataclass
 class TimingInfo(dbtClassMixin):
+    """
+    Represents a step in the execution of a node.
+    `name` should be one of: compile, execute, or other
+    Do not call directly, use `collect_timing_info` instead.
+    """
+
     name: str
     started_at: Optional[datetime] = None
     completed_at: Optional[datetime] = None
@@ -21,7 +27,7 @@ def end(self):
         self.completed_at = datetime.utcnow()
 
     def to_msg_dict(self):
-        msg_dict = {"name": self.name}
+        msg_dict = {"name": str(self.name)}
         if self.started_at:
             msg_dict["started_at"] = datetime_to_json_string(self.started_at)
         if self.completed_at:

core/dbt/cli/flags.py

@@ -7,7 +7,6 @@
 from pprint import pformat as pf
 from typing import Any, Callable, Dict, List, Optional, Set, Union
 
-import pytz
 from click import Context, Parameter, get_current_context
 from click.core import Command as ClickCommand
 from click.core import Group, ParameterSource
@@ -362,17 +361,26 @@ def _validate_event_time_configs(self) -> None:
             getattr(self, "EVENT_TIME_END") if hasattr(self, "EVENT_TIME_END") else None
         )
 
-        if event_time_start is not None and event_time_end is not None:
-            if event_time_start >= event_time_end:
+        # only do validations if at least one of `event_time_start` or `event_time_end` are specified
+        if event_time_start is not None or event_time_end is not None:
+
+            # These `ifs`, combined with the parent `if` make it so that `event_time_start` and
+            # `event_time_end` are mutually required
+            if event_time_start is None:
                 raise DbtUsageException(
-                    "Value for `--event-time-start` must be less than `--event-time-end`"
+                    "The flag `--event-time-end` was specified, but `--event-time-start` was not. "
+                    "When specifying `--event-time-end`, `--event-time-start` must also be present."
+                )
+            if event_time_end is None:
+                raise DbtUsageException(
+                    "The flag `--event-time-start` was specified, but `--event-time-end` was not. "
+                    "When specifying `--event-time-start`, `--event-time-end` must also be present."
                 )
-        elif event_time_start is not None:
-            utc_start = event_time_start.replace(tzinfo=pytz.UTC)
-            current_time = datetime.now(pytz.UTC)
-            if utc_start >= current_time:
+
+            # This `if` just is a sanity check that `event_time_start` is before `event_time_end`
+            if event_time_start >= event_time_end:
                 raise DbtUsageException(
-                    f"Value for `--event-time-start` ({utc_start}) must be less than the current time ({current_time}) if `--event-time-end` is not specififed"
+                    "Value for `--event-time-start` must be less than `--event-time-end`"
                 )
 
     def fire_deprecations(self):

core/dbt/context/providers.py

@@ -684,8 +684,15 @@ def resolve(self, source_name: str, table_name: str):
                 target_kind="source",
                 disabled=(isinstance(target_source, Disabled)),
             )
+
+        # Source quoting does _not_ respect global configs in dbt_project.yml, as documented here:
+        # https://docs.getdbt.com/reference/project-configs/quoting
+        # Use an object with an empty quoting field to bypass any settings in self.
+        class SourceQuotingBaseConfig:
+            quoting: Dict[str, Any] = {}
+
         return self.Relation.create_from(
-            self.config,
+            SourceQuotingBaseConfig(),
             target_source,
             limit=self.resolve_limit,
             event_time_filter=self.resolve_event_time_filter(target_source),

core/dbt/contracts/files.py

@@ -192,6 +192,7 @@ class SchemaSourceFile(BaseSourceFile):
     sources: List[str] = field(default_factory=list)
     exposures: List[str] = field(default_factory=list)
     metrics: List[str] = field(default_factory=list)
+    snapshots: List[str] = field(default_factory=list)
     # The following field will no longer be used. Leaving
     # here to avoid breaking existing projects. To be removed
     # later if possible.

core/dbt/contracts/graph/manifest.py

@@ -59,14 +59,15 @@
     SeedNode,
     SemanticModel,
     SingularTestNode,
+    SnapshotNode,
     SourceDefinition,
     UnitTestDefinition,
     UnitTestFileFixture,
     UnpatchedSourceDefinition,
 )
 from dbt.contracts.graph.unparsed import SourcePatch, UnparsedVersion
 from dbt.contracts.util import SourceKey
-from dbt.events.types import UnpinnedRefNewVersionAvailable
+from dbt.events.types import ArtifactWritten, UnpinnedRefNewVersionAvailable
 from dbt.exceptions import (
     AmbiguousResourceNameRefError,
     CompilationError,
@@ -1218,7 +1219,9 @@ def writable_manifest(self) -> "WritableManifest":
         )
 
     def write(self, path):
-        self.writable_manifest().write(path)
+        writable = self.writable_manifest()
+        writable.write(path)
+        fire_event(ArtifactWritten(artifact_type=writable.__class__.__name__, artifact_path=path))
 
     # Called in dbt.compilation.Linker.write_graph and
     # dbt.graph.queue.get and ._include_in_cost
@@ -1600,12 +1603,14 @@ def add_node(self, source_file: AnySourceFile, node: ManifestNode, test_from=Non
             if isinstance(node, GenericTestNode):
                 assert test_from
                 source_file.add_test(node.unique_id, test_from)
-            if isinstance(node, Metric):
+            elif isinstance(node, Metric):
                 source_file.metrics.append(node.unique_id)
-            if isinstance(node, Exposure):
+            elif isinstance(node, Exposure):
                 source_file.exposures.append(node.unique_id)
-            if isinstance(node, Group):
+            elif isinstance(node, Group):
                 source_file.groups.append(node.unique_id)
+            elif isinstance(node, SnapshotNode):
+                source_file.snapshots.append(node.unique_id)
         elif isinstance(source_file, FixtureSourceFile):
             pass
         else:

core/dbt/contracts/graph/semantic_manifest.py

@@ -7,7 +7,7 @@
 )
 from dbt.contracts.graph.manifest import Manifest
 from dbt.contracts.graph.nodes import ModelNode
-from dbt.events.types import SemanticValidationFailure
+from dbt.events.types import ArtifactWritten, SemanticValidationFailure
 from dbt.exceptions import ParsingError
 from dbt_common.clients.system import write_file
 from dbt_common.events.base_types import EventLevel
@@ -71,6 +71,7 @@ def write_json_to_file(self, file_path: str):
         semantic_manifest = self._get_pydantic_semantic_manifest()
         json = semantic_manifest.json()
         write_file(file_path, json)
+        fire_event(ArtifactWritten(artifact_type=self.__class__.__name__, artifact_path=file_path))
 
     def _get_pydantic_semantic_manifest(self) -> PydanticSemanticManifest:
         pydantic_time_spines: List[PydanticTimeSpine] = []

core/dbt/contracts/sql.py

@@ -7,7 +7,9 @@
 from dbt.artifacts.schemas.results import ExecutionResult, TimingInfo
 from dbt.artifacts.schemas.run import RunExecutionResult, RunResult, RunResultsArtifact
 from dbt.contracts.graph.nodes import ResultNode
+from dbt.events.types import ArtifactWritten
 from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.events.functions import fire_event
 
 TaskTags = Optional[Dict[str, Any]]
 TaskID = uuid.UUID
@@ -49,6 +51,7 @@ def write(self, path: str) -> None:
             args=self.args,
         )
         writable.write(path)
+        fire_event(ArtifactWritten(artifact_type=writable.__class__.__name__, artifact_path=path))
 
     @classmethod
     def from_local_result(