Merge remote-tracking branch 'origin/main' into paw/record-groups

.changes/unreleased/Features-20240716-102457.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add to and to_columns to ColumnLevelConstraint and ModelLevelConstraint contracts
+time: 2024-07-16T10:24:57.11251-04:00
+custom:
+  Author: michelleark
+  Issue: "168"

.changes/unreleased/Fixes-20240715-205355.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix case-insensitive env vars for Windows
+time: 2024-07-15T20:53:55.946355+01:00
+custom:
+    Author: peterallenwebb aranke
+    Issue: "166"

dbt_common/context.py

@@ -1,15 +1,46 @@
+import os
 from contextvars import ContextVar, copy_context
-from typing import List, Mapping, Optional
+from typing import List, Mapping, Optional, Iterator
 
 from dbt_common.constants import PRIVATE_ENV_PREFIX, SECRET_ENV_PREFIX
 from dbt_common.record import Recorder
 
 
+class CaseInsensitiveMapping(Mapping):
+    def __init__(self, env: Mapping[str, str]):
+        self._env = {k.casefold(): (k, v) for k, v in env.items()}
+
+    def __getitem__(self, key: str) -> str:
+        return self._env[key.casefold()][1]
+
+    def __len__(self) -> int:
+        return len(self._env)
+
+    def __iter__(self) -> Iterator[str]:
+        for item in self._env.items():
+            yield item[0]
+
+
 class InvocationContext:
     def __init__(self, env: Mapping[str, str]):
-        self._env = {k: v for k, v in env.items() if not k.startswith(PRIVATE_ENV_PREFIX)}
+        self._env: Mapping[str, str]
+
+        env_public = {}
+        env_private = {}
+
+        for k, v in env.items():
+            if k.startswith(PRIVATE_ENV_PREFIX):
+                env_private[k] = v
+            else:
+                env_public[k] = v
+
+        if os.name == "nt":
+            self._env = CaseInsensitiveMapping(env_public)
+        else:
+            self._env = env_public
+
         self._env_secrets: Optional[List[str]] = None
-        self._env_private = {k: v for k, v in env.items() if k.startswith(PRIVATE_ENV_PREFIX)}
+        self._env_private = env_private
         self.recorder: Optional[Recorder] = None
         # This class will also eventually manage the invocation_id, flags, event manager, etc.
 

dbt_common/contracts/constraints.py

@@ -36,6 +36,8 @@ class ColumnLevelConstraint(dbtClassMixin):
     warn_unsupported: bool = (
         True  # Warn if constraint is not supported by the platform and won't be in DDL
     )
+    to: Optional[str] = None
+    to_columns: List[str] = field(default_factory=list)
 
 
 @dataclass

dbt_common/record.py

@@ -10,7 +10,6 @@
 import json
 import os
 
-from deepdiff import DeepDiff  # type: ignore
 from enum import Enum
 from typing import Any, Callable, Dict, List, Mapping, Optional, Type
 
@@ -53,6 +52,11 @@ def from_dict(cls, dct: Mapping) -> "Record":
 
 class Diff:
     def __init__(self, current_recording_path: str, previous_recording_path: str) -> None:
+        # deepdiff is expensive to import, so we only do it here when we need it
+        from deepdiff import DeepDiff  # type: ignore
+
+        self.diff = DeepDiff
+
         self.current_recording_path = current_recording_path
         self.previous_recording_path = previous_recording_path
 
@@ -68,7 +72,7 @@ def diff_query_records(self, current: List, previous: List) -> Dict[str, Any]:
             if previous[i].get("result").get("table") is not None:
                 previous[i]["result"]["table"] = json.loads(previous[i]["result"]["table"])
 
-        return DeepDiff(previous, current, ignore_order=True, verbose_level=2)
+        return self.diff(previous, current, ignore_order=True, verbose_level=2)
 
     def diff_env_records(self, current: List, previous: List) -> Dict[str, Any]:
         # The mode and filepath may change.  Ignore them.
@@ -78,12 +82,12 @@ def diff_env_records(self, current: List, previous: List) -> Dict[str, Any]:
             "root[0]['result']['env']['DBT_RECORDER_MODE']",
         ]
 
-        return DeepDiff(
+        return self.diff(
             previous, current, ignore_order=True, verbose_level=2, exclude_paths=exclude_paths
         )
 
     def diff_default(self, current: List, previous: List) -> Dict[str, Any]:
-        return DeepDiff(previous, current, ignore_order=True, verbose_level=2)
+        return self.diff(previous, current, ignore_order=True, verbose_level=2)
 
     def calculate_diff(self) -> Dict[str, Any]:
         with open(self.current_recording_path) as current_recording:

pyproject.toml

@@ -126,6 +126,7 @@ ignore = ["E203", "E501", "E741", "W503", "W504"]
 exclude = [
     "dbt_common/events/types_pb2.py",
     "venv",
+    ".venv",
     "env*"
 ]
 per-file-ignores = ["*/__init__.py: F401"]

tests/unit/test_invocation_context.py

@@ -1,5 +1,9 @@
+import os
+
+import pytest
+
 from dbt_common.constants import PRIVATE_ENV_PREFIX, SECRET_ENV_PREFIX
-from dbt_common.context import InvocationContext
+from dbt_common.context import InvocationContext, CaseInsensitiveMapping
 
 
 def test_invocation_context_env() -> None:
@@ -8,6 +12,17 @@ def test_invocation_context_env() -> None:
     assert ic.env == test_env
 
 
+@pytest.mark.skipif(
+    os.name != "nt", reason="Test for case-insensitive env vars, only run on Windows"
+)
+def test_invocation_context_windows() -> None:
+    test_env = {"var_1": "lowercase", "vAr_2": "mixedcase", "VAR_3": "uppercase"}
+    ic = InvocationContext(env=test_env)
+    assert ic.env == CaseInsensitiveMapping(
+        {"var_1": "lowercase", "var_2": "mixedcase", "var_3": "uppercase"}
+    )
+
+
 def test_invocation_context_secrets() -> None:
     test_env = {
         f"{SECRET_ENV_PREFIX}_VAR_1": "secret1",