Skip to content

Commit

Permalink
feat: add task retries in get_data_source_max_date
Browse files Browse the repository at this point in the history
  • Loading branch information
folhesgabriel committed Nov 13, 2024
1 parent 674297a commit 4fc8067
Showing 1 changed file with 6 additions and 2 deletions.
8 changes: 6 additions & 2 deletions pipelines/datasets/br_me_cnpj/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import asyncio
import os
from typing import Union, List
from datetime import datetime
from datetime import datetime,timedelta
import basedosdados as bd
from prefect import task

Expand All @@ -20,13 +20,17 @@
process_csv_socios,
)
from pipelines.utils.utils import log
from pipelines.constants import constants

ufs = constants_cnpj.UFS.value
url = constants_cnpj.URL.value
headers = constants_cnpj.HEADERS.value


@task
@task(
max_retries=3,
retry_delay=timedelta(seconds=constants.TASK_RETRY_DELAY.value),
)
def get_data_source_max_date() -> tuple[datetime,datetime]:
"""
Checks if there are available updates for a specific dataset and table.
Expand Down

0 comments on commit 4fc8067

Please sign in to comment.