From 7c54995bb0a791fb1e810a4f22c8b2959fef902b Mon Sep 17 00:00:00 2001 From: tricktx Date: Thu, 14 Nov 2024 13:07:29 -0300 Subject: [PATCH 1/2] fix schedules anatel --- pipelines/datasets/br_anatel_banda_larga_fixa/schedules.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pipelines/datasets/br_anatel_banda_larga_fixa/schedules.py b/pipelines/datasets/br_anatel_banda_larga_fixa/schedules.py index aad46acbd..2f8f7386e 100644 --- a/pipelines/datasets/br_anatel_banda_larga_fixa/schedules.py +++ b/pipelines/datasets/br_anatel_banda_larga_fixa/schedules.py @@ -24,7 +24,6 @@ "materialize_after_dump": True, "dbt_alias": True, "update_metadata": True, - "ano": "2024", }, ), ] From 4cdba6b43005d9e53443e7ee3a3b71409abcf79c Mon Sep 17 00:00:00 2001 From: tricktx Date: Thu, 14 Nov 2024 13:31:52 -0300 Subject: [PATCH 2/2] fix read csv and schedules --- pipelines/datasets/br_anatel_telefonia_movel/schedules.py | 2 -- pipelines/utils/crawler_anatel/banda_larga_fixa/tasks.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pipelines/datasets/br_anatel_telefonia_movel/schedules.py b/pipelines/datasets/br_anatel_telefonia_movel/schedules.py index eb4ddfa48..308ae6117 100644 --- a/pipelines/datasets/br_anatel_telefonia_movel/schedules.py +++ b/pipelines/datasets/br_anatel_telefonia_movel/schedules.py @@ -23,8 +23,6 @@ "materialize_after_dump": True, "dbt_alias": True, "update_metadata": True, - "ano": "2024", - "semestre": "2", "update_metadata": True, }, ), diff --git a/pipelines/utils/crawler_anatel/banda_larga_fixa/tasks.py b/pipelines/utils/crawler_anatel/banda_larga_fixa/tasks.py index 564f411d8..224f91f44 100644 --- a/pipelines/utils/crawler_anatel/banda_larga_fixa/tasks.py +++ b/pipelines/utils/crawler_anatel/banda_larga_fixa/tasks.py @@ -61,10 +61,10 @@ def get_max_date_in_table_microdados(table_id: str, ano: int): return df['data'].max() else: - log(f"{anatel_constants.INPUT_PATH.value}Densidade_Telefonia_Movel.csv") + log(f"{anatel_constants.INPUT_PATH.value}Densidade_Banda_Larga_Fixa.csv") df = pd.read_csv( - f"{anatel_constants.INPUT_PATH.value}Densidade_Telefonia_Movel.csv", + f"{anatel_constants.INPUT_PATH.value}Densidade_Banda_Larga_Fixa.csv", sep=";", encoding="utf-8", dtype=str