Skip to content

Commit

Permalink
[fix] add max_retries in crawl_cno
Browse files Browse the repository at this point in the history
  • Loading branch information
tricktx committed Aug 16, 2024
1 parent 905958c commit ee59295
Showing 1 changed file with 6 additions and 2 deletions.
8 changes: 6 additions & 2 deletions pipelines/datasets/br_rf_cno/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
"""
Tasks for br_rf_cno
"""

from datetime import datetime, timedelta
from pipelines.constants import constants
import os
from bs4 import BeautifulSoup
from datetime import datetime
Expand Down Expand Up @@ -106,7 +107,10 @@ def wrangling(input_dir: str, output_dir: str, partition_date: str) -> None:
log('----- Wrangling completed')


@task
@task(
max_retries=5,
retry_delay=timedelta(seconds=constants.TASK_RETRY_DELAY.value),
)
def crawl_cno(root: str, url: str) -> None:
"""
Downloads and unpacks a ZIP file from the given URL.
Expand Down

0 comments on commit ee59295

Please sign in to comment.