register flow part 2

basedosdados · Nov 12, 2024 · 6f31009 · 6f31009
1 parent 47cbd1a
commit 6f31009
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 4 deletions.
diff --git a/pipelines/utils/crawler_anatel/banda_larga_fixa/flows.py b/pipelines/utils/crawler_anatel/banda_larga_fixa/flows.py
@@ -70,6 +70,7 @@
                 wait=filepath,
                 upstream_tasks=[filepath],  # Fix: Wrap filepath in a list to make it iterable
             )
+
         with case(materialize_after_dump, True):
             # Trigger DBT flow run
             current_flow_labels = get_current_flow_labels()

diff --git a/pipelines/utils/crawler_anatel/banda_larga_fixa/utils.py b/pipelines/utils/crawler_anatel/banda_larga_fixa/utils.py
@@ -214,12 +214,12 @@ def treatment_municipio(table_id:str):
 
 
 def get_year():
+    import os
     lista = []
     for x in os.listdir(anatel_constants.INPUT_PATH.value):
-        print(x)
         parts = x.split("_")
-        if len(parts) > 3:
-            x = parts[3]
+        if len(parts) > 4:
+            x = parts[4]
             if len(x) == 4:
                 lista.append(x)
 

diff --git a/pipelines/utils/crawler_anatel/telefonia_movel/flows.py b/pipelines/utils/crawler_anatel/telefonia_movel/flows.py
@@ -130,4 +130,4 @@
                 )
 
 flow_anatel_telefonia_movel.storage = GCS(constants.GCS_FLOWS_BUCKET.value)
-flow_anatel_telefonia_movel.run_config = KubernetesRun(image=constants.DOCKER_IMAGE.value)
+flow_anatel_telefonia_movel.run_config = KubernetesRun(image=constants.DOCKER_IMAGE.value)