Merge branch 'main' into previous-state-manifest

.changes/unreleased/Dependencies-20231204-000945.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Bump freezegun from 0.3.12 to 1.3.0"
+time: 2023-12-04T00:09:45.00000Z
+custom:
+  Author: dependabot[bot]
+  PR: 9197

.changes/unreleased/Dependencies-20240212-011324.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Bump peter-evans/create-pull-request from 5 to 6"
+time: 2024-02-12T01:13:24.00000Z
+custom:
+  Author: dependabot[bot]
+  PR: 9552

.changes/unreleased/Dependencies-20240226-123502.yaml

@@ -0,0 +1,6 @@
+kind: Dependencies
+body: Cap dbt-semantic-interfaces version range to <0.6
+time: 2024-02-26T12:35:02.643779-08:00
+custom:
+  Author: tlento
+  PR: "9671"

.changes/unreleased/Fixes-20240223-162107.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Add target-path to retry
+time: 2024-02-23T16:21:07.83639Z
+custom:
+  Author: aranke
+  Issue: "8948"

.changes/unreleased/Security-20240222-152445.yaml

@@ -0,0 +1,6 @@
+kind: Security
+body: Update Jinja2 to >= 3.1.3 to address CVE-2024-22195
+time: 2024-02-22T15:24:45.158305-08:00
+custom:
+  Author: QMalcolm
+  PR: CVE-2024-22195

.changes/unreleased/Under the Hood-20240221-104518.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Restrict protobuf to major version 4.
+time: 2024-02-21T10:45:18.315195-05:00
+custom:
+  Author: peterallenwebb
+  Issue: "9566"

.changes/unreleased/Under the Hood-20240221-145058.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Make dbt-core compatible with Python 3.12
+time: 2024-02-21T14:50:58.983559Z
+custom:
+  Author: l1xnan aranke
+  Issue: "9007"

.changes/unreleased/Under the Hood-20240223-115021.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Implement primary key inference for model nodes
+time: 2024-02-23T11:50:21.257494-08:00
+custom:
+  Author: aliceliu
+  Issue: "9652"

.changes/unreleased/Under the Hood-20240226-141038.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Define UnitTestDefinition resource in dbt/artifacts/resources
+time: 2024-02-26T14:10:38.807154-05:00
+custom:
+  Author: michelleark
+  Issue: "9667"

.github/workflows/backport.yml

@@ -35,6 +35,6 @@ jobs:
       github.event.pull_request.merged
       && contains(github.event.label.name, 'backport')
     steps:
-      - uses: tibdex/[email protected].3
+      - uses: tibdex/[email protected].4
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/main.yml

@@ -74,7 +74,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
 
     env:
       TOXENV: "unit"
@@ -157,7 +157,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
         os: [ubuntu-20.04]
         split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
         include: ${{ fromJson(needs.integration-metadata.outputs.include) }}

.github/workflows/model_performance.yml

@@ -253,7 +253,7 @@ jobs:
           push: 'origin origin/${{ matrix.target-branch }}'
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v6
         with:
           author: 'Github Build Bot <[email protected]>'
           base: ${{ matrix.base-branch }}

Makefile

@@ -30,14 +30,19 @@ CI_FLAGS =\
 .PHONY: dev_req
 dev_req: ## Installs dbt-* packages in develop mode along with only development dependencies.
 	@\
-	pip install -r dev-requirements.txt
-	pip install -r editable-requirements.txt
+	pip install -r dev-requirements.txt -r editable-requirements.txt
 
 .PHONY: dev
 dev: dev_req ## Installs dbt-* packages in develop mode along with development dependencies and pre-commit.
 	@\
 	pre-commit install
 
+.PHONY: dev-uninstall
+dev-uninstall: ## Uninstall all packages in venv except for build tools
+	@\
+    pip freeze | grep -v "^-e" | cut -d "@" -f1 | xargs pip uninstall -y; \
+    pip uninstall -y dbt-core
+
 .PHONY: core_proto_types
 core_proto_types:  ## generates google protobuf python file from core_types.proto
 	protoc -I=./core/dbt/events --python_out=./core/dbt/events ./core/dbt/events/core_types.proto

core/MANIFEST.in

@@ -1,3 +1,3 @@
-recursive-include dbt/adapters/include *.py *.sql *.yml *.html *.md .gitkeep .gitignore
+recursive-include dbt/include *.py *.sql *.yml *.html *.md .gitkeep .gitignore
 include dbt/py.typed
 recursive-include dbt/task/docs *.html

core/dbt/artifacts/resources/__init__.py

@@ -89,3 +89,13 @@
     SourceDefinition,
     ParsedSourceMandatory,
 )
+
+from dbt.artifacts.resources.v1.unit_test_definition import (
+    UnitTestConfig,
+    UnitTestDefinition,
+    UnitTestInputFixture,
+    UnitTestOutputFixture,
+    UnitTestOverrides,
+    UnitTestNodeVersions,
+    UnitTestFormat,
+)

core/dbt/artifacts/resources/v1/unit_test_definition.py

@@ -0,0 +1,80 @@
+from dataclasses import dataclass, field
+import time
+from typing import Optional, Sequence, Dict, Any, List, Union
+
+from dbt_common.contracts.config.base import (
+    BaseConfig,
+    CompareBehavior,
+    MergeBehavior,
+)
+from dbt_common.contracts.config.metadata import ShowBehavior
+from dbt_common.dataclass_schema import dbtClassMixin, StrEnum
+
+from dbt.artifacts.resources.v1.config import metas, list_str
+from dbt.artifacts.resources.base import GraphResource
+from dbt.artifacts.resources import NodeVersion, DependsOn
+
+
+@dataclass
+class UnitTestConfig(BaseConfig):
+    tags: Union[str, List[str]] = field(
+        default_factory=list_str,
+        metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude),
+    )
+    meta: Dict[str, Any] = field(
+        default_factory=dict,
+        metadata=MergeBehavior.Update.meta(),
+    )
+
+
+class UnitTestFormat(StrEnum):
+    CSV = "csv"
+    Dict = "dict"
+
+
+@dataclass
+class UnitTestInputFixture(dbtClassMixin):
+    input: str
+    rows: Optional[Union[str, List[Dict[str, Any]]]] = None
+    format: UnitTestFormat = UnitTestFormat.Dict
+    fixture: Optional[str] = None
+
+
+@dataclass
+class UnitTestOverrides(dbtClassMixin):
+    macros: Dict[str, Any] = field(default_factory=dict)
+    vars: Dict[str, Any] = field(default_factory=dict)
+    env_vars: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class UnitTestNodeVersions(dbtClassMixin):
+    include: Optional[List[NodeVersion]] = None
+    exclude: Optional[List[NodeVersion]] = None
+
+
+@dataclass
+class UnitTestOutputFixture(dbtClassMixin):
+    rows: Optional[Union[str, List[Dict[str, Any]]]] = None
+    format: UnitTestFormat = UnitTestFormat.Dict
+    fixture: Optional[str] = None
+
+
+@dataclass
+class UnitTestDefinitionMandatory:
+    model: str
+    given: Sequence[UnitTestInputFixture]
+    expect: UnitTestOutputFixture
+
+
+@dataclass
+class UnitTestDefinition(GraphResource, UnitTestDefinitionMandatory):
+    description: str = ""
+    overrides: Optional[UnitTestOverrides] = None
+    depends_on: DependsOn = field(default_factory=DependsOn)
+    config: UnitTestConfig = field(default_factory=UnitTestConfig)
+    checksum: Optional[str] = None
+    schema: Optional[str] = None
+    created_at: float = field(default_factory=lambda: time.time())
+    versions: Optional[UnitTestNodeVersions] = None
+    version: Optional[NodeVersion] = None

core/dbt/artifacts/schemas/manifest/v12/manifest.py

@@ -17,14 +17,14 @@
     Metric,
     SavedQuery,
     SemanticModel,
+    SourceDefinition,
+    UnitTestDefinition,
 )
 
 # TODO: remove usage of dbt modules other than dbt.artifacts
 from dbt.contracts.graph.nodes import (
     GraphMemberNode,
     ManifestNode,
-    SourceDefinition,
-    UnitTestDefinition,
 )
 
 

core/dbt/cli/main.py

@@ -595,6 +595,7 @@ def run(ctx, **kwargs):
 @p.vars
 @p.profile
 @p.target
+@p.target_path
 @p.threads
 @p.full_refresh
 @requires.postflight

core/dbt/contracts/graph/manifest.py

@@ -1081,8 +1081,8 @@ def writable_manifest(self) -> "WritableManifest":
         return WritableManifest(
             nodes=self.nodes,
             sources=self._map_nodes_to_map_resources(self.sources),
-            macros=self.macros,
-            docs=self.docs,
+            macros=self._map_nodes_to_map_resources(self.macros),
+            docs=self._map_nodes_to_map_resources(self.docs),
             exposures=self._map_nodes_to_map_resources(self.exposures),
             metrics=self._map_nodes_to_map_resources(self.metrics),
             groups=self._map_nodes_to_map_resources(self.groups),
@@ -1093,7 +1093,7 @@ def writable_manifest(self) -> "WritableManifest":
             parent_map=self.parent_map,
             group_map=self.group_map,
             semantic_models=self._map_nodes_to_map_resources(self.semantic_models),
-            unit_tests=self.unit_tests,
+            unit_tests=self._map_nodes_to_map_resources(self.unit_tests),
             saved_queries=self._map_nodes_to_map_resources(self.saved_queries),
         )
 

core/dbt/contracts/graph/model_config.py

@@ -1,5 +1,5 @@
 from dataclasses import field, dataclass
-from typing import Any, List, Optional, Dict, Union, Type
+from typing import Any, List, Optional, Dict, Type
 
 from dbt.artifacts.resources import (
     ExposureConfig,
@@ -12,10 +12,10 @@
     SnapshotConfig,
     SourceConfig,
     ModelConfig,
+    UnitTestConfig,
 )
-from dbt_common.contracts.config.base import BaseConfig, MergeBehavior, CompareBehavior
-from dbt_common.contracts.config.metadata import Metadata, ShowBehavior
-from dbt.contracts.util import list_str
+from dbt_common.contracts.config.base import BaseConfig
+from dbt_common.contracts.config.metadata import Metadata
 from dbt.node_types import NodeType
 
 
@@ -44,18 +44,6 @@ class EmptySnapshotConfig(NodeConfig):
     unique_key: Optional[str] = None  # override NodeConfig unique_key definition
 
 
-@dataclass
-class UnitTestConfig(BaseConfig):
-    tags: Union[str, List[str]] = field(
-        default_factory=list_str,
-        metadata=metas(ShowBehavior.Hide, MergeBehavior.Append, CompareBehavior.Exclude),
-    )
-    meta: Dict[str, Any] = field(
-        default_factory=dict,
-        metadata=MergeBehavior.Update.meta(),
-    )
-
-
 RESOURCE_TYPES: Dict[NodeType, Type[BaseConfig]] = {
     NodeType.Metric: MetricConfig,
     NodeType.SemanticModel: SemanticModelConfig,