From 6e3828e0b0eced0ae4a42b2534748cf1b005a4aa Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 15 Jul 2024 09:32:37 -0500 Subject: [PATCH 1/3] conditionally import deepdiff (#164) only import deepdiff when needed --- dbt_common/record.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/dbt_common/record.py b/dbt_common/record.py index 8fe068bb..b33d4b5a 100644 --- a/dbt_common/record.py +++ b/dbt_common/record.py @@ -10,7 +10,6 @@ import json import os -from deepdiff import DeepDiff # type: ignore from enum import Enum from typing import Any, Callable, Dict, List, Mapping, Optional, Type @@ -52,6 +51,11 @@ def from_dict(cls, dct: Mapping) -> "Record": class Diff: def __init__(self, current_recording_path: str, previous_recording_path: str) -> None: + # deepdiff is expensive to import, so we only do it here when we need it + from deepdiff import DeepDiff # type: ignore + + self.diff = DeepDiff + self.current_recording_path = current_recording_path self.previous_recording_path = previous_recording_path @@ -67,7 +71,7 @@ def diff_query_records(self, current: List, previous: List) -> Dict[str, Any]: if previous[i].get("result").get("table") is not None: previous[i]["result"]["table"] = json.loads(previous[i]["result"]["table"]) - return DeepDiff(previous, current, ignore_order=True, verbose_level=2) + return self.diff(previous, current, ignore_order=True, verbose_level=2) def diff_env_records(self, current: List, previous: List) -> Dict[str, Any]: # The mode and filepath may change. Ignore them. @@ -77,12 +81,12 @@ def diff_env_records(self, current: List, previous: List) -> Dict[str, Any]: "root[0]['result']['env']['DBT_RECORDER_MODE']", ] - return DeepDiff( + return self.diff( previous, current, ignore_order=True, verbose_level=2, exclude_paths=exclude_paths ) def diff_default(self, current: List, previous: List) -> Dict[str, Any]: - return DeepDiff(previous, current, ignore_order=True, verbose_level=2) + return self.diff(previous, current, ignore_order=True, verbose_level=2) def calculate_diff(self) -> Dict[str, Any]: with open(self.current_recording_path) as current_recording: From 71f4d53c084e5960b4804c46eff435abc8718e57 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Mon, 15 Jul 2024 21:42:40 +0100 Subject: [PATCH 2/3] Fix case-insensitive env vars for Windows (#166) * Draft fix of case-insensitive env vars for Windows. * add test for windows * add test skip reason --------- Co-authored-by: Peter Allen Webb --- .../unreleased/Fixes-20240715-205355.yaml | 6 +++ dbt_common/context.py | 37 +++++++++++++++++-- pyproject.toml | 1 + tests/unit/test_invocation_context.py | 17 ++++++++- 4 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 .changes/unreleased/Fixes-20240715-205355.yaml diff --git a/.changes/unreleased/Fixes-20240715-205355.yaml b/.changes/unreleased/Fixes-20240715-205355.yaml new file mode 100644 index 00000000..780a6ad9 --- /dev/null +++ b/.changes/unreleased/Fixes-20240715-205355.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix case-insensitive env vars for Windows +time: 2024-07-15T20:53:55.946355+01:00 +custom: + Author: peterallenwebb aranke + Issue: "166" diff --git a/dbt_common/context.py b/dbt_common/context.py index d1775c55..947d409a 100644 --- a/dbt_common/context.py +++ b/dbt_common/context.py @@ -1,15 +1,46 @@ +import os from contextvars import ContextVar, copy_context -from typing import List, Mapping, Optional +from typing import List, Mapping, Optional, Iterator from dbt_common.constants import PRIVATE_ENV_PREFIX, SECRET_ENV_PREFIX from dbt_common.record import Recorder +class CaseInsensitiveMapping(Mapping): + def __init__(self, env: Mapping[str, str]): + self._env = {k.casefold(): (k, v) for k, v in env.items()} + + def __getitem__(self, key: str) -> str: + return self._env[key.casefold()][1] + + def __len__(self) -> int: + return len(self._env) + + def __iter__(self) -> Iterator[str]: + for item in self._env.items(): + yield item[0] + + class InvocationContext: def __init__(self, env: Mapping[str, str]): - self._env = {k: v for k, v in env.items() if not k.startswith(PRIVATE_ENV_PREFIX)} + self._env: Mapping[str, str] + + env_public = {} + env_private = {} + + for k, v in env.items(): + if k.startswith(PRIVATE_ENV_PREFIX): + env_private[k] = v + else: + env_public[k] = v + + if os.name == "nt": + self._env = CaseInsensitiveMapping(env_public) + else: + self._env = env_public + self._env_secrets: Optional[List[str]] = None - self._env_private = {k: v for k, v in env.items() if k.startswith(PRIVATE_ENV_PREFIX)} + self._env_private = env_private self.recorder: Optional[Recorder] = None # This class will also eventually manage the invocation_id, flags, event manager, etc. diff --git a/pyproject.toml b/pyproject.toml index 64fc04fc..ba306437 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,6 +126,7 @@ ignore = ["E203", "E501", "E741", "W503", "W504"] exclude = [ "dbt_common/events/types_pb2.py", "venv", + ".venv", "env*" ] per-file-ignores = ["*/__init__.py: F401"] diff --git a/tests/unit/test_invocation_context.py b/tests/unit/test_invocation_context.py index 3dc832d3..fbf060ba 100644 --- a/tests/unit/test_invocation_context.py +++ b/tests/unit/test_invocation_context.py @@ -1,5 +1,9 @@ +import os + +import pytest + from dbt_common.constants import PRIVATE_ENV_PREFIX, SECRET_ENV_PREFIX -from dbt_common.context import InvocationContext +from dbt_common.context import InvocationContext, CaseInsensitiveMapping def test_invocation_context_env() -> None: @@ -8,6 +12,17 @@ def test_invocation_context_env() -> None: assert ic.env == test_env +@pytest.mark.skipif( + os.name != "nt", reason="Test for case-insensitive env vars, only run on Windows" +) +def test_invocation_context_windows() -> None: + test_env = {"var_1": "lowercase", "vAr_2": "mixedcase", "VAR_3": "uppercase"} + ic = InvocationContext(env=test_env) + assert ic.env == CaseInsensitiveMapping( + {"var_1": "lowercase", "var_2": "mixedcase", "var_3": "uppercase"} + ) + + def test_invocation_context_secrets() -> None: test_env = { f"{SECRET_ENV_PREFIX}_VAR_1": "secret1", From 8edae92ac4e1c5814be028281215fd90ab29d9c6 Mon Sep 17 00:00:00 2001 From: Michelle Ark Date: Tue, 16 Jul 2024 10:28:23 -0400 Subject: [PATCH 3/3] add to and to_column(s) to ColumnLevelConstraint and ModelLevelConstraint contracts (#163) * add to and to_column(s) to ColumnLevelConstraint and ModelLevelConstraint contracts * fix schema * linting * changelog entry --- .changes/unreleased/Features-20240716-102457.yaml | 6 ++++++ dbt_common/contracts/constraints.py | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 .changes/unreleased/Features-20240716-102457.yaml diff --git a/.changes/unreleased/Features-20240716-102457.yaml b/.changes/unreleased/Features-20240716-102457.yaml new file mode 100644 index 00000000..096b4259 --- /dev/null +++ b/.changes/unreleased/Features-20240716-102457.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Add to and to_columns to ColumnLevelConstraint and ModelLevelConstraint contracts +time: 2024-07-16T10:24:57.11251-04:00 +custom: + Author: michelleark + Issue: "168" diff --git a/dbt_common/contracts/constraints.py b/dbt_common/contracts/constraints.py index c01ee6f8..4e2d9c7a 100644 --- a/dbt_common/contracts/constraints.py +++ b/dbt_common/contracts/constraints.py @@ -36,6 +36,8 @@ class ColumnLevelConstraint(dbtClassMixin): warn_unsupported: bool = ( True # Warn if constraint is not supported by the platform and won't be in DDL ) + to: Optional[str] = None + to_columns: List[str] = field(default_factory=list) @dataclass