Merge remote-tracking branch 'origin/main' into paw/snapshots_new_rec…

…ord_mode_2

.changes/1.10.2.md

@@ -0,0 +1,5 @@
+## dbt-adapters 1.10.2 - October 01, 2024
+
+### Under the Hood
+
+- dbt-tests-adapters: Add required begin to microbatch model config to BaseMicrobatch test ([#315](https://github.com/dbt-labs/dbt-adapters/issues/315))

.changes/1.10.3.md

@@ -0,0 +1 @@
+## dbt-adapters 1.10.3 - October 29, 2024

.changes/1.7.1.md

@@ -0,0 +1,5 @@
+## dbt-adapters 1.7.1 - October 15, 2024
+
+### Features
+
+- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320))

.changes/1.7.2.md

@@ -0,0 +1,16 @@
+## dbt-adapters 1.7.2 - October 21, 2024
+
+### Breaking Changes
+
+- Drop support for Python 3.8 ([#332](https://github.com/dbt-labs/dbt-adapters/issues/332))
+
+### Features
+
+- Allows unique_key for snapshots to take a list ([#181](https://github.com/dbt-labs/dbt-adapters/issues/181))
+
+### Fixes
+
+- Always validate an incremental model's `incremental_strategy` ([#330](https://github.com/dbt-labs/dbt-adapters/issues/330))
+
+### Contributors
+- [@agpapa](https://github.com/agpapa) ([#181](https://github.com/dbt-labs/dbt-adapters/issues/181))

.changes/1.8.0.md

@@ -0,0 +1,9 @@
+## dbt-adapters 1.8.0 - October 29, 2024
+
+### Fixes
+
+- Always make behavior flags available for evaluation ([#338](https://github.com/dbt-labs/dbt-adapters/issues/338))
+
+### Under the Hood
+
+- Add adapter telemetry. ([#301](https://github.com/dbt-labs/dbt-adapters/issues/301))

.github/actions/publish-results/action.yml

@@ -5,7 +5,7 @@ inputs:
     description: File type for file name stub (e.g. "unit-tests")
     required: true
   python-version:
-    description: Python version for the file name stub (e.g. "3.8")
+    description: Python version for the file name stub (e.g. "3.9")
     required: true
   source-file:
     description: File to be uploaded

.github/workflows/unit-tests.yml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - name: Check out repository

.pre-commit-config.yaml

@@ -18,19 +18,19 @@ repos:
     -   id: dbt-core-in-adapters-check
 
 -   repo: https://github.com/psf/black
-    rev: 24.4.0
+    rev: 24.8.0
     hooks:
     -   id: black
         args:
         -   --line-length=99
-        -   --target-version=py38
         -   --target-version=py39
         -   --target-version=py310
         -   --target-version=py311
+        -   --target-version=py312
         -   --force-exclude=dbt/adapters/events/adapter_types_pb2.py
 
 -   repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
+    rev: 7.1.1
     hooks:
     -   id: flake8
         exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/
@@ -41,7 +41,7 @@ repos:
         -   --per-file-ignores=*/__init__.py:F401,*/conftest.py:F401
 
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.9.0
+    rev: v1.11.2
     hooks:
     -   id: mypy
         exclude: dbt/adapters/events/adapter_types_pb2.py|dbt-tests-adapter/dbt/__init__.py

CHANGELOG.md

@@ -5,17 +5,58 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
+## dbt-adapters 1.10.3 - October 29, 2024
+
+
+
+## dbt-adapters 1.10.2 - October 01, 2024
+
+### Under the Hood
+
+- dbt-tests-adapters: Add required begin to microbatch model config to BaseMicrobatch test ([#315](https://github.com/dbt-labs/dbt-adapters/issues/315))
+
 ## dbt-adapters 1.10.1 - September 16, 2024
 
 ## dbt-adapters 1.10.0 - September 12, 2024
 
-## dbt-adapters 1.7.0 - September 19, 2024
+## dbt-adapters 1.8.0 - October 29, 2024
+
+### Fixes
+
+- Always make behavior flags available for evaluation ([#338](https://github.com/dbt-labs/dbt-adapters/issues/338))
+
+### Under the Hood
+
+- Add adapter telemetry. ([#301](https://github.com/dbt-labs/dbt-adapters/issues/301))
+
+## dbt-adapters 1.7.2 - October 21, 2024
+
+### Breaking Changes
+
+- Drop support for Python 3.8 ([#332](https://github.com/dbt-labs/dbt-adapters/issues/332))
 
 ### Features
 
-- Allow configuring of snapshot column names ([#289](https://github.com/dbt-labs/dbt-adapters/issues/289))
+- Allows unique_key for snapshots to take a list ([#181](https://github.com/dbt-labs/dbt-adapters/issues/181))
+
+### Fixes
 
+- Always validate an incremental model's `incremental_strategy` ([#330](https://github.com/dbt-labs/dbt-adapters/issues/330))
 
+### Contributors
+- [@agpapa](https://github.com/agpapa) ([#181](https://github.com/dbt-labs/dbt-adapters/issues/181))
+
+## dbt-adapters 1.7.1 - October 15, 2024
+
+### Features
+
+- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320))
+
+## dbt-adapters 1.7.0 - September 19, 2024
+
+### Features
+
+- Allow configuring of snapshot column names ([#289](https://github.com/dbt-labs/dbt-adapters/issues/289))
 
 ## dbt-adapters 1.6.1 - September 16, 2024
 

dbt-tests-adapter/dbt/tests/__about__.py

@@ -1 +1 @@
-version = "1.10.1"
+version = "1.10.3"

dbt-tests-adapter/dbt/tests/adapter/basic/files.py

@@ -186,6 +186,10 @@
   {{ config(materialized="incremental") }}
 """
 
+config_materialized_incremental_invalid_strategy = """
+  {{ config(materialized="incremental", incremental_strategy="bad_strategy") }}
+"""
+
 config_materialized_var = """
   {{ config(materialized=var("materialized_var", "table"))}}
 """
@@ -217,3 +221,6 @@
 ephemeral_view_sql = config_materialized_view + model_ephemeral
 ephemeral_table_sql = config_materialized_table + model_ephemeral
 incremental_sql = config_materialized_incremental + model_incremental
+incremental_invalid_strategy_sql = (
+    config_materialized_incremental_invalid_strategy + model_incremental
+)

dbt-tests-adapter/dbt/tests/adapter/basic/test_incremental.py

@@ -86,6 +86,45 @@ def test_incremental_not_schema_change(self, project):
         assert run_result == RunStatus.Success
 
 
+class BaseIncrementalBadStrategy:
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {"name": "incremental"}
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "incremental.sql": files.incremental_invalid_strategy_sql,
+            "schema.yml": files.schema_base_yml,
+        }
+
+    @pytest.fixture(scope="class")
+    def seeds(self):
+        return {"base.csv": files.seeds_base_csv, "added.csv": files.seeds_added_csv}
+
+    @pytest.fixture(autouse=True)
+    def clean_up(self, project):
+        yield
+        with project.adapter.connection_named("__test"):
+            relation = project.adapter.Relation.create(
+                database=project.database, schema=project.test_schema
+            )
+            project.adapter.drop_schema(relation)
+
+    def test_incremental_invalid_strategy(self, project):
+        # seed command
+        results = run_dbt(["seed"])
+        assert len(results) == 2
+
+        # try to run the incremental model, it should fail on the first attempt
+        results = run_dbt(["run"], expect_pass=False)
+        assert len(results.results) == 1
+        assert (
+            'dbt could not find an incremental strategy macro with the name "get_incremental_bad_strategy_sql"'
+            in results.results[0].message
+        )
+
+
 class Testincremental(BaseIncremental):
     pass
 

dbt-tests-adapter/dbt/tests/adapter/hooks/test_run_hooks.py

@@ -3,6 +3,7 @@
 
 import pytest
 
+from dbt_common.exceptions import DbtDatabaseError
 from dbt.tests.adapter.hooks import fixtures
 from dbt.tests.util import check_table_does_not_exist, run_dbt
 
@@ -11,8 +12,8 @@ class BasePrePostRunHooks:
     @pytest.fixture(scope="function")
     def setUp(self, project):
         project.run_sql_file(project.test_data_dir / Path("seed_run.sql"))
-        project.run_sql(f"drop table if exists { project.test_schema }.schemas")
-        project.run_sql(f"drop table if exists { project.test_schema }.db_schemas")
+        project.run_sql(f"drop table if exists {project.test_schema}.schemas")
+        project.run_sql(f"drop table if exists {project.test_schema}.db_schemas")
         os.environ["TERM_TEST"] = "TESTING"
 
     @pytest.fixture(scope="class")
@@ -158,7 +159,8 @@ def project_config_update(self):
         }
 
     def test_missing_column_pre_hook(self, project):
-        run_dbt(["run"], expect_pass=False)
+        with pytest.raises(DbtDatabaseError):
+            run_dbt(["run"], expect_pass=False)
 
 
 class TestAfterRunHooks(BaseAfterRunHooks):

dbt-tests-adapter/pyproject.toml

@@ -4,7 +4,7 @@ name = "dbt-tests-adapter"
 description = "The set of reusable tests and test fixtures used to test common functionality"
 readme = "README.md"
 keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs"]
-requires-python = ">=3.8.0"
+requires-python = ">=3.9.0"
 authors = [
     { name = "dbt Labs", email = "[email protected]" },
 ]
@@ -17,7 +17,6 @@ classifiers = [
     "Operating System :: MacOS :: MacOS X",
     "Operating System :: Microsoft :: Windows",
     "Operating System :: POSIX :: Linux",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",

dbt/adapters/__about__.py

@@ -1 +1 @@
-version = "1.7.0"
+version = "1.8.0"

dbt/adapters/base/__init__.py

@@ -12,4 +12,5 @@
     BaseRelation,
     RelationType,
     SchemaSearchMap,
+    AdapterTrackingRelationInfo,
 )

dbt/adapters/base/impl.py

@@ -4,6 +4,7 @@
 from contextlib import contextmanager
 from datetime import datetime
 from enum import Enum
+from importlib import import_module
 from multiprocessing.context import SpawnContext
 from typing import (
     Any,
@@ -61,12 +62,14 @@
     ComponentName,
     InformationSchema,
     SchemaSearchMap,
+    AdapterTrackingRelationInfo,
 )
 from dbt.adapters.cache import RelationsCache, _make_ref_key_dict
 from dbt.adapters.capability import Capability, CapabilityDict
 from dbt.adapters.contracts.connection import Credentials
 from dbt.adapters.contracts.macros import MacroResolverProtocol
 from dbt.adapters.contracts.relation import RelationConfig
+
 from dbt.adapters.events.types import (
     CacheMiss,
     CatalogGenerationError,
@@ -300,12 +303,13 @@ def behavior(self) -> Behavior:
     @behavior.setter  # type: ignore
     def behavior(self, flags: List[BehaviorFlag]) -> None:
         flags.extend(self._behavior_flags)
-        try:
-            # we don't always get project flags, for example during `dbt debug`
-            self._behavior = Behavior(flags, self.config.flags)
-        except AttributeError:
-            # in that case, don't load any behavior to avoid unexpected defaults
-            self._behavior = Behavior([], {})
+
+        # we don't always get project flags, for example, the project file is not loaded during `dbt debug`
+        # in that case, load the default values for behavior flags to avoid compilation errors
+        # this mimics not loading a project file, or not specifying flags in a project file
+        user_overrides = getattr(self.config, "flags", {})
+
+        self._behavior = Behavior(flags, user_overrides)
 
     @property
     def _behavior_flags(self) -> List[BehaviorFlag]:
@@ -1743,6 +1747,30 @@ def capabilities(cls) -> CapabilityDict:
     def supports(cls, capability: Capability) -> bool:
         return bool(cls.capabilities()[capability])
 
+    @classmethod
+    def get_adapter_run_info(cls, config: RelationConfig) -> AdapterTrackingRelationInfo:
+        adapter_class_name, *_ = cls.__name__.split("Adapter")
+        adapter_name = adapter_class_name.lower()
+
+        if adapter_name == "base":
+            adapter_version = ""
+        else:
+            adapter_version = import_module(f"dbt.adapters.{adapter_name}.__version__").version
+
+        return AdapterTrackingRelationInfo(
+            adapter_name=adapter_name,
+            base_adapter_version=import_module("dbt.adapters.__about__").version,
+            adapter_version=adapter_version,
+            model_adapter_details=cls._get_adapter_specific_run_info(config),
+        )
+
+    @classmethod
+    def _get_adapter_specific_run_info(cls, config) -> Dict[str, Any]:
+        """
+        Adapter maintainers should overwrite this method to return any run metadata that should be captured during a run.
+        """
+        return {}
+
 
 COLUMNS_EQUAL_SQL = """
 with diff_count as (

dbt/adapters/base/relation.py

@@ -6,6 +6,7 @@
     Dict,
     FrozenSet,
     Iterator,
+    List,
     Optional,
     Set,
     Tuple,
@@ -341,6 +342,16 @@ def create(
         )
         return cls.from_dict(kwargs)
 
+    @classmethod
+    def scd_args(cls: Type[Self], primary_key: Union[str, List[str]], updated_at) -> List[str]:
+        scd_args = []
+        if isinstance(primary_key, list):
+            scd_args.extend(primary_key)
+        else:
+            scd_args.append(primary_key)
+        scd_args.append(updated_at)
+        return scd_args
+
     @property
     def can_be_renamed(self) -> bool:
         return self.type in self.renameable_relations
@@ -531,3 +542,11 @@ def flatten(self, allow_multiple_databases: bool = False) -> "SchemaSearchMap":
             )
 
         return new
+
+
+@dataclass(frozen=True, eq=False, repr=False)
+class AdapterTrackingRelationInfo(FakeAPIObject, Hashable):
+    adapter_name: str
+    base_adapter_version: str
+    adapter_version: str
+    model_adapter_details: Any