Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[fix] br_anp_precos_combustiveis #483

Merged
merged 9 commits into from
Sep 28, 2023
4 changes: 0 additions & 4 deletions pipelines/datasets/br_anp_precos_combustiveis/flows.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,11 @@
from prefect.tasks.prefect import create_flow_run, wait_for_flow_run

from pipelines.constants import constants
from pipelines.datasets.br_anp_precos_combustiveis.constants import (
constants as anatel_constants,
)
from pipelines.datasets.br_anp_precos_combustiveis.schedules import (
every_week_anp_microdados,
)
from pipelines.datasets.br_anp_precos_combustiveis.tasks import (
check_for_updates,
data_max_bd_mais,
data_max_bd_pro,
download_and_transform,
make_partitions,
Expand Down
5 changes: 2 additions & 3 deletions pipelines/datasets/br_anp_precos_combustiveis/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,8 @@ def check_for_updates(dataset_id, table_id):
# Obtém a data mais recente do site
download_files(anatel_constants.URLS_DATA.value, anatel_constants.PATH_INPUT.value)
df = pd.read_csv(anatel_constants.URL_GLP.value, sep=";", encoding="utf-8")
data_obj = df["Data da Coleta"].max()
data_obj = datetime.strptime(data_obj, "%d/%m/%Y").strftime("%Y-%m-%d")
data_obj = datetime.strptime(data_obj, "%Y-%m-%d").date()
data_obj = pd.to_datetime(df["Data da Coleta"]).max()
data_obj = data_obj.date()

# Obtém a última data no site BD
data_bq_obj = extract_last_date(
Expand Down
Loading