fix anatel time out

basedosdados · Dec 15, 2024 · beec487 · beec487
1 parent 48a974e
commit beec487
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 8 deletions.
diff --git a/pipelines/utils/crawler_anatel/banda_larga_fixa/flows.py b/pipelines/utils/crawler_anatel/banda_larga_fixa/flows.py
@@ -56,15 +56,16 @@
     # https://discourse.prefect.io/t/my-parameter-value-shows-the-same-date-every-day-how-can-i-set-parameter-value-dynamically/99
     #####
 
-    new_ano = get_year_and_unzip(day=ano)
+    new_ano = get_year_and_unzip(day=ano, upstream_tasks=[rename_flow_run])
 
     update_tables = get_max_date_in_table_microdados(ano=new_ano, table_id=table_id, upstream_tasks=[new_ano])
 
     get_max_date = check_if_data_is_outdated(
     dataset_id =  dataset_id,
     table_id =  table_id,
     data_source_max_date = update_tables,
-    date_format =  "%Y-%m")
+    date_format =  "%Y-%m",
+    upstream_tasks=[update_tables])
 
     with case(get_max_date, True):
         filepath = join_tables_in_function(

diff --git a/pipelines/utils/crawler_anatel/banda_larga_fixa/utils.py b/pipelines/utils/crawler_anatel/banda_larga_fixa/utils.py
@@ -55,18 +55,19 @@ def download_zip_file(path):
     driver.get(anatel_constants.URL.value)
 
     driver.maximize_window()
-    WebDriverWait(driver, 60).until(
+    WebDriverWait(driver, 300).until(
                 EC.element_to_be_clickable(
                     (By.XPATH, '/html/body/div/section/div/div[3]/div[2]/div[3]/div[2]/header/button')
                 )
             ).click()
 
-    WebDriverWait(driver, 60).until(
+    WebDriverWait(driver, 300).until(
                 EC.element_to_be_clickable(
                     (By.XPATH, '/html/body/div/section/div/div[3]/div[2]/div[3]/div[2]/div/div[1]/div[2]/div[2]/div/button')
                 )
             ).click()
-    time.sleep(300)
+    time.sleep(150)
+    log(os.listdir(path))
 
 def unzip_file():
     download_zip_file(path=anatel_constants.INPUT_PATH.value)

diff --git a/pipelines/utils/crawler_anatel/telefonia_movel/flows.py b/pipelines/utils/crawler_anatel/telefonia_movel/flows.py
@@ -59,7 +59,7 @@
     # Function dynamic parameters
     # https://discourse.prefect.io/t/my-parameter-value-shows-the-same-date-every-day-how-can-i-set-parameter-value-dynamically/99
     #####
-    unzip_task = unzip()
+    unzip_task = unzip(upstream_tasks=[rename_flow_run])
     new_year = get_year_full(ano, upstream_tasks=[unzip_task])
     new_semester = get_semester(semestre, upstream_tasks=[new_year])
 
@@ -72,6 +72,7 @@
     table_id =  table_id,
     data_source_max_date = update_tables,
     date_format =  "%Y-%m",
+    upstream_tasks=[update_tables]
 )
 
     with case(get_max_date, True):

diff --git a/pipelines/utils/crawler_anatel/telefonia_movel/utils.py b/pipelines/utils/crawler_anatel/telefonia_movel/utils.py
@@ -50,12 +50,12 @@ def download_zip_file(path):
     driver = webdriver.Chrome(options=options)
     driver.get(anatel_constants.URL.value)
     driver.maximize_window()
-    WebDriverWait(driver, 60).until(
+    WebDriverWait(driver, 300).until(
                 EC.element_to_be_clickable(
                     (By.XPATH, '/html/body/div/section/div/div[3]/div[2]/div[3]/div[2]/header/button')
                 )
             ).click()
-    WebDriverWait(driver, 60).until(
+    WebDriverWait(driver, 300).until(
                 EC.element_to_be_clickable(
                     (By.XPATH, '/html/body/div/section/div/div[3]/div[2]/div[3]/div[2]/div/div[1]/div[2]/div[2]/div/button')
                 )