Merge branch 'main' into paw/types-tidy-2

.changes/unreleased/Features-20240716-102457.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add to and to_columns to ColumnLevelConstraint and ModelLevelConstraint contracts
+time: 2024-07-16T10:24:57.11251-04:00
+custom:
+  Author: michelleark
+  Issue: "168"

.changes/unreleased/Fixes-20240715-205355.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix case-insensitive env vars for Windows
+time: 2024-07-15T20:53:55.946355+01:00
+custom:
+    Author: peterallenwebb aranke
+    Issue: "166"

.changes/unreleased/Under the Hood-20240618-155025.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Deserialize Record objects on a just-in-time basis.
+time: 2024-06-18T15:50:25.985387-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "151"

.changes/unreleased/Under the Hood-20240716-125753.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add record grouping mechanism to record/replay.
+time: 2024-07-16T12:57:53.434099-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "169"

dbt_common/__about__.py

@@ -1 +1 @@
-version = "1.4.0"
+version = "1.7.0"

dbt_common/clients/system.py

@@ -38,6 +38,15 @@
     c_bool = None
 
 
+def _record_path(path: str) -> bool:
+    return (
+        # TODO: The first check here obviates the next two checks but is probably too coarse?
+        "dbt/include" not in path
+        and "dbt/include/global_project" not in path
+        and "/plugins/postgres/dbt/include/" not in path
+    )
+
+
 @dataclasses.dataclass
 class FindMatchingParams:
     root_path: str
@@ -61,10 +70,7 @@ def __init__(
     def _include(self) -> bool:
         # Do not record or replay filesystem searches that were performed against
         # files which are actually part of dbt's implementation.
-        return (
-            "dbt/include/global_project" not in self.root_path
-            and "/plugins/postgres/dbt/include/" not in self.root_path
-        )
+        return _record_path(self.root_path)
 
 
 @dataclasses.dataclass
@@ -148,10 +154,7 @@ class LoadFileParams:
     def _include(self) -> bool:
         # Do not record or replay file reads that were performed against files
         # which are actually part of dbt's implementation.
-        return (
-            "dbt/include/global_project" not in self.path
-            and "/plugins/postgres/dbt/include/" not in self.path
-        )
+        return _record_path(self.path)
 
 
 @dataclasses.dataclass
@@ -246,10 +249,7 @@ class WriteFileParams:
     def _include(self) -> bool:
         # Do not record or replay file reads that were performed against files
         # which are actually part of dbt's implementation.
-        return (
-            "dbt/include/global_project" not in self.path
-            and "/plugins/postgres/dbt/include/" not in self.path
-        )
+        return _record_path(self.path)
 
 
 @Recorder.register_record_type

dbt_common/context.py

@@ -1,15 +1,47 @@
+import os
 from contextvars import ContextVar, copy_context
-from typing import List, Mapping, Optional
+from typing import List, Mapping, Optional, Iterator
 
 from dbt_common.constants import PRIVATE_ENV_PREFIX, SECRET_ENV_PREFIX
+from dbt_common.record import Recorder
+
+
+class CaseInsensitiveMapping(Mapping):
+    def __init__(self, env: Mapping[str, str]):
+        self._env = {k.casefold(): (k, v) for k, v in env.items()}
+
+    def __getitem__(self, key: str) -> str:
+        return self._env[key.casefold()][1]
+
+    def __len__(self) -> int:
+        return len(self._env)
+
+    def __iter__(self) -> Iterator[str]:
+        for item in self._env.items():
+            yield item[0]
 
 
 class InvocationContext:
     def __init__(self, env: Mapping[str, str]):
-        self._env = {k: v for k, v in env.items() if not k.startswith(PRIVATE_ENV_PREFIX)}
+        self._env: Mapping[str, str]
+
+        env_public = {}
+        env_private = {}
+
+        for k, v in env.items():
+            if k.startswith(PRIVATE_ENV_PREFIX):
+                env_private[k] = v
+            else:
+                env_public[k] = v
+
+        if os.name == "nt":
+            self._env = CaseInsensitiveMapping(env_public)
+        else:
+            self._env = env_public
+
         self._env_secrets: Optional[List[str]] = None
-        self._env_private = {k: v for k, v in env.items() if k.startswith(PRIVATE_ENV_PREFIX)}
-        self.recorder = None
+        self._env_private = env_private
+        self.recorder: Optional[Recorder] = None
         # This class will also eventually manage the invocation_id, flags, event manager, etc.
 
     @property
@@ -32,7 +64,7 @@ def env_secrets(self) -> List[str]:
 _INVOCATION_CONTEXT_VAR: ContextVar[InvocationContext] = ContextVar("DBT_INVOCATION_CONTEXT_VAR")
 
 
-def reliably_get_invocation_var() -> ContextVar:
+def reliably_get_invocation_var() -> ContextVar[InvocationContext]:
     invocation_var: Optional[ContextVar] = next(
         (cv for cv in copy_context() if cv.name == _INVOCATION_CONTEXT_VAR.name), None
     )

dbt_common/contracts/constraints.py

@@ -36,6 +36,8 @@ class ColumnLevelConstraint(dbtClassMixin):
     warn_unsupported: bool = (
         True  # Warn if constraint is not supported by the platform and won't be in DDL
     )
+    to: Optional[str] = None
+    to_columns: List[str] = field(default_factory=list)
 
 
 @dataclass

dbt_common/dataclass_schema.py

@@ -1,4 +1,4 @@
-from typing import ClassVar, cast, get_type_hints, List, Tuple, Dict, Any, Optional
+from typing import Any, cast, ClassVar, Dict, get_type_hints, List, Optional, Tuple
 import re
 import jsonschema
 from dataclasses import fields, Field
@@ -26,7 +26,7 @@ class ValidationError(jsonschema.ValidationError):
 
 
 class DateTimeSerialization(SerializationStrategy):
-    def serialize(self, value) -> str:
+    def serialize(self, value: datetime) -> str:
         out = value.isoformat()
         # Assume UTC if timezone is missing
         if value.tzinfo is None:
@@ -127,7 +127,7 @@ def _get_fields(cls) -> List[Tuple[Field, str]]:
 
     # copied from hologram. Used in tests
     @classmethod
-    def _get_field_names(cls):
+    def _get_field_names(cls) -> List[str]:
         return [element[1] for element in cls._get_fields()]
 
 
@@ -152,7 +152,7 @@ def validate(cls, value):
 
 # These classes must be in this order or it doesn't work
 class StrEnum(str, SerializableType, Enum):
-    def __str__(self):
+    def __str__(self) -> str:
         return self.value
 
     # https://docs.python.org/3.6/library/enum.html#using-automatic-values

dbt_common/exceptions/base.py

@@ -1,5 +1,5 @@
 import builtins
-from typing import List, Any, Optional
+from typing import Any, List, Optional
 import os
 
 from dbt_common.constants import SECRET_ENV_PREFIX
@@ -37,7 +37,7 @@ def __init__(self, msg: str):
         self.msg = scrub_secrets(msg, env_secrets())
 
     @property
-    def type(self):
+    def type(self) -> str:
         return "Internal"
 
     def process_stack(self):
@@ -59,7 +59,7 @@ def process_stack(self):
 
         return lines
 
-    def __str__(self):
+    def __str__(self) -> str:
         if hasattr(self.msg, "split"):
             split_msg = self.msg.split("\n")
         else:

dbt_common/helper_types.py

@@ -19,7 +19,7 @@
 class NVEnum(StrEnum):
     novalue = "novalue"
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
         return isinstance(other, NVEnum)
 
 
@@ -59,7 +59,7 @@ def includes(self, item_name: str) -> bool:
             item_name in self.include or self.include in self.INCLUDE_ALL
         ) and item_name not in self.exclude
 
-    def _validate_items(self, items: List[str]):
+    def _validate_items(self, items: List[str]) -> None:
         pass