Merge branch 'main' into try_fastjsonschema

.changes/unreleased/Features-20240716-102457.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add to and to_columns to ColumnLevelConstraint and ModelLevelConstraint contracts
+time: 2024-07-16T10:24:57.11251-04:00
+custom:
+  Author: michelleark
+  Issue: "168"

.changes/unreleased/Fixes-20240715-205355.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix case-insensitive env vars for Windows
+time: 2024-07-15T20:53:55.946355+01:00
+custom:
+    Author: peterallenwebb aranke
+    Issue: "166"

.changes/unreleased/Under the Hood-20240603-123631.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Move CatalogKey, ColumnMetadata, ColumnMap, CatalogTable to dbt-common
+time: 2024-06-03T12:36:31.542118+02:00
+custom:
+    Author: aranke
+    Issue: "147"

.changes/unreleased/Under the Hood-20240617-204541.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add support for basic diff of run recordings.
+time: 2024-06-17T20:45:41.123374-05:00
+custom:
+  Author: emmyoop
+  Issue: "144"

.changes/unreleased/Under the Hood-20240618-155025.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Deserialize Record objects on a just-in-time basis.
+time: 2024-06-18T15:50:25.985387-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "151"

.changes/unreleased/Under the Hood-20240716-125753.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add record grouping mechanism to record/replay.
+time: 2024-07-16T12:57:53.434099-04:00
+custom:
+  Author: peterallenwebb
+  Issue: "169"

dbt_common/__about__.py

@@ -1 +1 @@
-version = "1.2.0"
+version = "1.7.0"

dbt_common/clients/system.py

@@ -38,6 +38,15 @@
     c_bool = None
 
 
+def _record_path(path: str) -> bool:
+    return (
+        # TODO: The first check here obviates the next two checks but is probably too coarse?
+        "dbt/include" not in path
+        and "dbt/include/global_project" not in path
+        and "/plugins/postgres/dbt/include/" not in path
+    )
+
+
 @dataclasses.dataclass
 class FindMatchingParams:
     root_path: str
@@ -61,10 +70,7 @@ def __init__(
     def _include(self) -> bool:
         # Do not record or replay filesystem searches that were performed against
         # files which are actually part of dbt's implementation.
-        return (
-            "dbt/include/global_project" not in self.root_path
-            and "/plugins/postgres/dbt/include/" not in self.root_path
-        )
+        return _record_path(self.root_path)
 
 
 @dataclasses.dataclass
@@ -148,10 +154,7 @@ class LoadFileParams:
     def _include(self) -> bool:
         # Do not record or replay file reads that were performed against files
         # which are actually part of dbt's implementation.
-        return (
-            "dbt/include/global_project" not in self.path
-            and "/plugins/postgres/dbt/include/" not in self.path
-        )
+        return _record_path(self.path)
 
 
 @dataclasses.dataclass
@@ -246,10 +249,7 @@ class WriteFileParams:
     def _include(self) -> bool:
         # Do not record or replay file reads that were performed against files
         # which are actually part of dbt's implementation.
-        return (
-            "dbt/include/global_project" not in self.path
-            and "/plugins/postgres/dbt/include/" not in self.path
-        )
+        return _record_path(self.path)
 
 
 @Recorder.register_record_type

dbt_common/context.py

@@ -1,15 +1,47 @@
+import os
 from contextvars import ContextVar, copy_context
-from typing import List, Mapping, Optional
+from typing import List, Mapping, Optional, Iterator
 
 from dbt_common.constants import PRIVATE_ENV_PREFIX, SECRET_ENV_PREFIX
+from dbt_common.record import Recorder
+
+
+class CaseInsensitiveMapping(Mapping):
+    def __init__(self, env: Mapping[str, str]):
+        self._env = {k.casefold(): (k, v) for k, v in env.items()}
+
+    def __getitem__(self, key: str) -> str:
+        return self._env[key.casefold()][1]
+
+    def __len__(self) -> int:
+        return len(self._env)
+
+    def __iter__(self) -> Iterator[str]:
+        for item in self._env.items():
+            yield item[0]
 
 
 class InvocationContext:
     def __init__(self, env: Mapping[str, str]):
-        self._env = {k: v for k, v in env.items() if not k.startswith(PRIVATE_ENV_PREFIX)}
+        self._env: Mapping[str, str]
+
+        env_public = {}
+        env_private = {}
+
+        for k, v in env.items():
+            if k.startswith(PRIVATE_ENV_PREFIX):
+                env_private[k] = v
+            else:
+                env_public[k] = v
+
+        if os.name == "nt":
+            self._env = CaseInsensitiveMapping(env_public)
+        else:
+            self._env = env_public
+
         self._env_secrets: Optional[List[str]] = None
-        self._env_private = {k: v for k, v in env.items() if k.startswith(PRIVATE_ENV_PREFIX)}
-        self.recorder = None
+        self._env_private = env_private
+        self.recorder: Optional[Recorder] = None
         # This class will also eventually manage the invocation_id, flags, event manager, etc.
 
     @property
@@ -32,7 +64,7 @@ def env_secrets(self) -> List[str]:
 _INVOCATION_CONTEXT_VAR: ContextVar[InvocationContext] = ContextVar("DBT_INVOCATION_CONTEXT_VAR")
 
 
-def reliably_get_invocation_var() -> ContextVar:
+def reliably_get_invocation_var() -> ContextVar[InvocationContext]:
     invocation_var: Optional[ContextVar] = next(
         (cv for cv in copy_context() if cv.name == _INVOCATION_CONTEXT_VAR.name), None
     )

dbt_common/contracts/config/base.py

@@ -17,11 +17,11 @@
 @dataclass
 class BaseConfig(AdditionalPropertiesAllowed, Replaceable):
     # enable syntax like: config['key']
-    def __getitem__(self, key):
+    def __getitem__(self, key: str) -> Any:
         return self.get(key)
 
     # like doing 'get' on a dictionary
-    def get(self, key, default=None):
+    def get(self, key: str, default: Any = None) -> Any:
         if hasattr(self, key):
             return getattr(self, key)
         elif key in self._extra:
@@ -30,13 +30,13 @@ def get(self, key, default=None):
             return default
 
     # enable syntax like: config['key'] = value
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: str, value) -> None:
         if hasattr(self, key):
             setattr(self, key, value)
         else:
             self._extra[key] = value
 
-    def __delitem__(self, key):
+    def __delitem__(self, key: str) -> None:
         if hasattr(self, key):
             msg = (
                 'Error, tried to delete config key "{}": Cannot delete ' "built-in keys"
@@ -60,7 +60,7 @@ def _content_iterator(self, include_condition: Callable[[Field], bool]):
     def __iter__(self):
         yield from self._content_iterator(include_condition=lambda f: True)
 
-    def __len__(self):
+    def __len__(self) -> int:
         return len(self._get_fields()) + len(self._extra)
 
     @staticmethod
@@ -221,7 +221,7 @@ def _merge_field_value(
     merge_behavior: MergeBehavior,
     self_value: Any,
     other_value: Any,
-):
+) -> Any:
     if merge_behavior == MergeBehavior.Clobber:
         return other_value
     elif merge_behavior == MergeBehavior.Append:

dbt_common/contracts/constraints.py

@@ -36,6 +36,8 @@ class ColumnLevelConstraint(dbtClassMixin):
     warn_unsupported: bool = (
         True  # Warn if constraint is not supported by the platform and won't be in DDL
     )
+    to: Optional[str] = None
+    to_columns: List[str] = field(default_factory=list)
 
 
 @dataclass

dbt_common/contracts/metadata.py

@@ -1,7 +1,8 @@
 from dataclasses import dataclass
-from typing import Dict, Optional, Union
+from typing import Dict, Optional, Union, NamedTuple
 
 from dbt_common.dataclass_schema import dbtClassMixin
+from dbt_common.utils.formatting import lowercase
 
 
 @dataclass
@@ -24,3 +25,35 @@ class TableMetadata(dbtClassMixin):
     database: Optional[str] = None
     comment: Optional[str] = None
     owner: Optional[str] = None
+
+
+CatalogKey = NamedTuple(
+    "CatalogKey", [("database", Optional[str]), ("schema", str), ("name", str)]
+)
+
+
+@dataclass
+class ColumnMetadata(dbtClassMixin):
+    type: str
+    index: int
+    name: str
+    comment: Optional[str] = None
+
+
+ColumnMap = Dict[str, ColumnMetadata]
+
+
+@dataclass
+class CatalogTable(dbtClassMixin):
+    metadata: TableMetadata
+    columns: ColumnMap
+    stats: StatsDict
+    # the same table with two unique IDs will just be listed two times
+    unique_id: Optional[str] = None
+
+    def key(self) -> CatalogKey:
+        return CatalogKey(
+            lowercase(self.metadata.database),
+            self.metadata.schema.lower(),
+            self.metadata.name.lower(),
+        )

dbt_common/dataclass_schema.py

@@ -1,4 +1,4 @@
-from typing import ClassVar, cast, get_type_hints, List, Tuple, Dict, Any, Optional
+from typing import Any, cast, ClassVar, Dict, get_type_hints, List, Optional, Tuple
 import re
 import fastjsonschema
 from dataclasses import fields, Field
@@ -42,7 +42,7 @@ def __str__(self):
 
 
 class DateTimeSerialization(SerializationStrategy):
-    def serialize(self, value) -> str:
+    def serialize(self, value: datetime) -> str:
         out = value.isoformat()
         # Assume UTC if timezone is missing
         if value.tzinfo is None:
@@ -153,7 +153,7 @@ def _get_fields(cls) -> List[Tuple[Field, str]]:
 
     # copied from hologram. Used in tests
     @classmethod
-    def _get_field_names(cls):
+    def _get_field_names(cls) -> List[str]:
         return [element[1] for element in cls._get_fields()]
 
 
@@ -178,7 +178,7 @@ def validate(cls, value):
 
 # These classes must be in this order or it doesn't work
 class StrEnum(str, SerializableType, Enum):
-    def __str__(self):
+    def __str__(self) -> str:
         return self.value
 
     # https://docs.python.org/3.6/library/enum.html#using-automatic-values

dbt_common/exceptions/base.py

@@ -1,5 +1,5 @@
 import builtins
-from typing import List, Any, Optional
+from typing import Any, List, Optional
 import os
 
 from dbt_common.constants import SECRET_ENV_PREFIX
@@ -36,7 +36,7 @@ def __init__(self, msg: str):
         self.msg = scrub_secrets(msg, env_secrets())
 
     @property
-    def type(self):
+    def type(self) -> str:
         return "Internal"
 
     def process_stack(self):
@@ -58,7 +58,7 @@ def process_stack(self):
 
         return lines
 
-    def __str__(self):
+    def __str__(self) -> str:
         if hasattr(self.msg, "split"):
             split_msg = self.msg.split("\n")
         else:

dbt_common/helper_types.py

@@ -19,7 +19,7 @@
 class NVEnum(StrEnum):
     novalue = "novalue"
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
         return isinstance(other, NVEnum)
 
 
@@ -59,7 +59,7 @@ def includes(self, item_name: str) -> bool:
             item_name in self.include or self.include in self.INCLUDE_ALL
         ) and item_name not in self.exclude
 
-    def _validate_items(self, items: List[str]):
+    def _validate_items(self, items: List[str]) -> None:
         pass
 
 

dbt_common/invocation.py

@@ -7,6 +7,6 @@ def get_invocation_id() -> str:
     return _INVOCATION_ID
 
 
-def reset_invocation_id():
+def reset_invocation_id() -> None:
     global _INVOCATION_ID
     _INVOCATION_ID = str(uuid.uuid4())