Skip to content

Commit

Permalink
DT-3007: Give dispute stream a 90-day lookback window. (#6)
Browse files Browse the repository at this point in the history
  • Loading branch information
omarabed15 authored Jan 25, 2024
1 parent 5223900 commit 722f445
Show file tree
Hide file tree
Showing 2 changed files with 19 additions and 1 deletion.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,5 @@
/config.json
/.meltano/
/.secrets/config.json
.python-version
.vscode/
18 changes: 17 additions & 1 deletion tap_stripe/streams.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,18 @@ class StripeStream(Stream):
def get_starting_created_value(self, context: Optional[dict]) -> Optional[int]:
val = self.get_starting_replication_key_value(context)
if isinstance(val, str):
return pendulum.parse(val).int_timestamp
val = pendulum.parse(val).int_timestamp
assert isinstance(val, int)

if self.name == "disputes":
# For the disputes stream, increase the lookback window to 90 days (3 months).
# This is for several reasons:
# 1. Dispute volume is relatively small.
# 2. Dispute entities, not events, are being ingested. That means we are not capturing updates to dispute objects.
# This increased lookback window allows us to capture changes to any objects that occurred in the last 90 days.
# An ideal end state is to pass the lookback window as a config parameter to the Meltano stream,
# so it can be fine-tuned per stream.
val = val - (self.time_chunk_seconds * 90)
return val

@property
Expand Down Expand Up @@ -111,6 +121,12 @@ def _make_params(self, start_epoch: int, end_epoch: int, limit: int = 100) -> di

def _make_time_chunks(self, context) -> Iterable[Tuple[int, int]]:
step = self.time_chunk_seconds
if self.name == "disputes":
# Since dispute volume is so (relatively) low,
# we can process dispute in a single request that spans 90 days,
# rather than processing 90 one-day ingestion requests.
step = step * 90

return (
(i, i + step)
for i in range(
Expand Down

0 comments on commit 722f445

Please sign in to comment.