Skip to content

Commit

Permalink
Move duplicate_cursor_warning_threshold to a ClassVar
Browse files Browse the repository at this point in the history
  • Loading branch information
burnash committed Nov 12, 2024
1 parent 9703879 commit ca3471c
Showing 1 changed file with 2 additions and 10 deletions.
12 changes: 2 additions & 10 deletions dlt/extract/incremental/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,13 +113,12 @@ class Incremental(ItemTransform[TDataItem], BaseConfiguration, Generic[TCursorVa
allow_external_schedulers: bool = False
on_cursor_value_missing: OnCursorValueMissing = "raise"
lag: Optional[float] = None
duplicate_cursor_warning_threshold: ClassVar[int] = 200

# incremental acting as empty
EMPTY: ClassVar["Incremental[Any]"] = None
placement_affinity: ClassVar[float] = 1 # stick to end

DEFAULT_DUPLICATE_CURSOR_WARNING_THRESHOLD: ClassVar[int] = 200

def __init__(
self,
cursor_path: str = None,
Expand All @@ -131,7 +130,6 @@ def __init__(
allow_external_schedulers: bool = False,
on_cursor_value_missing: OnCursorValueMissing = "raise",
lag: Optional[float] = None,
duplicate_cursor_warning_threshold: Optional[int] = None,
) -> None:
# make sure that path is valid
if cursor_path:
Expand Down Expand Up @@ -168,12 +166,6 @@ def __init__(
self._bound_pipe: SupportsPipe = None
"""Bound pipe"""

self.duplicate_cursor_warning_threshold = (
duplicate_cursor_warning_threshold
if duplicate_cursor_warning_threshold is not None
else self.DEFAULT_DUPLICATE_CURSOR_WARNING_THRESHOLD
)

@property
def primary_key(self) -> Optional[TTableHintTemplate[TColumnNames]]:
return self._primary_key
Expand Down Expand Up @@ -550,7 +542,7 @@ def __call__(self, rows: TDataItems, meta: Any = None) -> Optional[TDataItems]:
def _check_duplicate_cursor_threshold(
self, initial_hash_count: int, final_hash_count: int
) -> None:
if initial_hash_count <= self.duplicate_cursor_warning_threshold < final_hash_count:
if initial_hash_count <= Incremental.duplicate_cursor_warning_threshold < final_hash_count:
logger.warning(
f"Large number of records ({final_hash_count}) sharing the same value of "
f"cursor field '{self.cursor_path}'. This can happen if the cursor "
Expand Down

0 comments on commit ca3471c

Please sign in to comment.