From 6f310090ccbf210db4027fa21431066635bd2b4c Mon Sep 17 00:00:00 2001 From: tricktx Date: Tue, 12 Nov 2024 16:44:00 -0300 Subject: [PATCH] register flow part 2 --- pipelines/utils/crawler_anatel/banda_larga_fixa/flows.py | 1 + pipelines/utils/crawler_anatel/banda_larga_fixa/utils.py | 6 +++--- pipelines/utils/crawler_anatel/telefonia_movel/flows.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pipelines/utils/crawler_anatel/banda_larga_fixa/flows.py b/pipelines/utils/crawler_anatel/banda_larga_fixa/flows.py index 7e01ce966..684d301a4 100644 --- a/pipelines/utils/crawler_anatel/banda_larga_fixa/flows.py +++ b/pipelines/utils/crawler_anatel/banda_larga_fixa/flows.py @@ -70,6 +70,7 @@ wait=filepath, upstream_tasks=[filepath], # Fix: Wrap filepath in a list to make it iterable ) + with case(materialize_after_dump, True): # Trigger DBT flow run current_flow_labels = get_current_flow_labels() diff --git a/pipelines/utils/crawler_anatel/banda_larga_fixa/utils.py b/pipelines/utils/crawler_anatel/banda_larga_fixa/utils.py index 31586c347..6b40e2559 100644 --- a/pipelines/utils/crawler_anatel/banda_larga_fixa/utils.py +++ b/pipelines/utils/crawler_anatel/banda_larga_fixa/utils.py @@ -214,12 +214,12 @@ def treatment_municipio(table_id:str): def get_year(): + import os lista = [] for x in os.listdir(anatel_constants.INPUT_PATH.value): - print(x) parts = x.split("_") - if len(parts) > 3: - x = parts[3] + if len(parts) > 4: + x = parts[4] if len(x) == 4: lista.append(x) diff --git a/pipelines/utils/crawler_anatel/telefonia_movel/flows.py b/pipelines/utils/crawler_anatel/telefonia_movel/flows.py index bf252f5a6..80c466bb8 100644 --- a/pipelines/utils/crawler_anatel/telefonia_movel/flows.py +++ b/pipelines/utils/crawler_anatel/telefonia_movel/flows.py @@ -130,4 +130,4 @@ ) flow_anatel_telefonia_movel.storage = GCS(constants.GCS_FLOWS_BUCKET.value) -flow_anatel_telefonia_movel.run_config = KubernetesRun(image=constants.DOCKER_IMAGE.value) +flow_anatel_telefonia_movel.run_config = KubernetesRun(image=constants.DOCKER_IMAGE.value) \ No newline at end of file