From 201cbaf7632e803a6600cc2cce238d31aa804e1b Mon Sep 17 00:00:00 2001 From: tricktx Date: Sun, 24 Sep 2023 21:26:11 -0300 Subject: [PATCH 1/2] fix check for updates --- pipelines/datasets/br_anp_precos_combustiveis/tasks.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pipelines/datasets/br_anp_precos_combustiveis/tasks.py b/pipelines/datasets/br_anp_precos_combustiveis/tasks.py index 48c911278..0ce9012f5 100644 --- a/pipelines/datasets/br_anp_precos_combustiveis/tasks.py +++ b/pipelines/datasets/br_anp_precos_combustiveis/tasks.py @@ -38,9 +38,8 @@ def check_for_updates(dataset_id, table_id): # Obtém a data mais recente do site download_files(anatel_constants.URLS_DATA.value, anatel_constants.PATH_INPUT.value) df = pd.read_csv(anatel_constants.URL_GLP.value, sep=";", encoding="utf-8") - data_obj = df["Data da Coleta"].max() - data_obj = datetime.strptime(data_obj, "%d/%m/%Y").strftime("%Y-%m-%d") - data_obj = datetime.strptime(data_obj, "%Y-%m-%d").date() + data_obj = pd.to_datetime(df["Data da Coleta"]).max() + data_obj = data_obj.date() # Obtém a última data no site BD data_bq_obj = extract_last_date( From 28c505566dbcb58226089226580d4e3c58f7b60f Mon Sep 17 00:00:00 2001 From: tricktx Date: Thu, 28 Sep 2023 11:01:26 -0300 Subject: [PATCH 2/2] register now --- pipelines/datasets/br_anp_precos_combustiveis/flows.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pipelines/datasets/br_anp_precos_combustiveis/flows.py b/pipelines/datasets/br_anp_precos_combustiveis/flows.py index 1fb522cd5..1fda2e806 100644 --- a/pipelines/datasets/br_anp_precos_combustiveis/flows.py +++ b/pipelines/datasets/br_anp_precos_combustiveis/flows.py @@ -11,15 +11,11 @@ from prefect.tasks.prefect import create_flow_run, wait_for_flow_run from pipelines.constants import constants -from pipelines.datasets.br_anp_precos_combustiveis.constants import ( - constants as anatel_constants, -) from pipelines.datasets.br_anp_precos_combustiveis.schedules import ( every_week_anp_microdados, ) from pipelines.datasets.br_anp_precos_combustiveis.tasks import ( check_for_updates, - data_max_bd_mais, data_max_bd_pro, download_and_transform, make_partitions,