register flow part 3

basedosdados · Nov 12, 2024 · 3eb86ea · 3eb86ea
1 parent 8a2ba5e
commit 3eb86ea
Show file tree

Hide file tree

Showing 5 changed files with 5 additions and 9 deletions.
diff --git a/pipelines/datasets/br_anatel_banda_larga_fixa/flows.py b/pipelines/datasets/br_anatel_banda_larga_fixa/flows.py
@@ -15,7 +15,6 @@
 )
 from pipelines.utils.crawler_anatel.banda_larga_fixa.flows import flow_anatel_banda_larga_fixa
 
-
 # ? Microdados
 br_anatel_banda_larga_fixa__microdados = deepcopy(flow_anatel_banda_larga_fixa)
 br_anatel_banda_larga_fixa__microdados.name = "br_anatel_banda_larga_fixa.microdados"

diff --git a/pipelines/datasets/br_anatel_telefonia_movel/flows.py b/pipelines/datasets/br_anatel_telefonia_movel/flows.py
@@ -15,7 +15,6 @@
             schedule_br_anatel_telefonia_movel__uf)
 
 from pipelines.utils.crawler_anatel.telefonia_movel.flows import flow_anatel_telefonia_movel
-
 # ? -------------------------------> Microdados
 br_anatel_telefonia_movel__microdados = deepcopy(flow_anatel_telefonia_movel)
 br_anatel_telefonia_movel__microdados.name = "br_anatel_telefonia_movel.microdados"

diff --git a/pipelines/utils/crawler_anatel/banda_larga_fixa/flows.py b/pipelines/utils/crawler_anatel/banda_larga_fixa/flows.py
@@ -70,7 +70,6 @@
                 wait=filepath,
                 upstream_tasks=[filepath],  # Fix: Wrap filepath in a list to make it iterable
             )
-
         with case(materialize_after_dump, True):
             # Trigger DBT flow run
             current_flow_labels = get_current_flow_labels()

diff --git a/pipelines/utils/crawler_anatel/banda_larga_fixa/utils.py b/pipelines/utils/crawler_anatel/banda_larga_fixa/utils.py
@@ -91,6 +91,7 @@ def check_and_create_column(df: pd.DataFrame, col_name: str) -> pd.DataFrame:
     # ! Pandas DataFrame: O DataFrame modificado.
     """
 
+
     if col_name not in df.columns:
         df[col_name] = ""
     return df
@@ -225,4 +226,4 @@ def get_year():
 
     max_year = max(lista)
     log(f"Ano máximo: {max_year}")
-    return max_year
+    return max_year
diff --git a/pipelines/utils/crawler_anatel/telefonia_movel/flows.py b/pipelines/utils/crawler_anatel/telefonia_movel/flows.py
@@ -43,11 +43,9 @@
         "materialize_after_dump", default=True, required=False
     )
     dbt_alias = Parameter("dbt_alias", default=True, required=False)
-    year = get_year_and_unzip()
-    semestre = get_semester()
-    ano = Parameter("ano", default=year, required=False)
 
-    semestre = Parameter("semestre", default=semestre, required=False)
+    ano = Parameter("ano", default=get_year_and_unzip(), required=False)
+    semestre = Parameter("semestre", default=get_semester(), required=False)
 
     update_metadata = Parameter("update_metadata", default=True, required=False)
 
@@ -130,4 +128,4 @@
                 )
 
 flow_anatel_telefonia_movel.storage = GCS(constants.GCS_FLOWS_BUCKET.value)
-flow_anatel_telefonia_movel.run_config = KubernetesRun(image=constants.DOCKER_IMAGE.value)
+flow_anatel_telefonia_movel.run_config = KubernetesRun(image=constants.DOCKER_IMAGE.value)