From 1bbdd881572ecaef54eb74bbd49a789caec930ab Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Sep 2023 23:43:01 +0000 Subject: [PATCH] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/scripts/register_flows.py | 1 - pipelines/datasets/br_anatel_banda_larga_fixa/schedules.py | 2 +- pipelines/datasets/br_anatel_banda_larga_fixa/tasks.py | 2 -- pipelines/datasets/br_anatel_banda_larga_fixa/utils.py | 1 - pipelines/datasets/br_anatel_telefonia_movel/schedules.py | 2 +- pipelines/datasets/br_anatel_telefonia_movel/utils.py | 1 - pipelines/datasets/br_anp_precos_combustiveis/schedules.py | 2 +- pipelines/datasets/br_ans_beneficiario/flows.py | 2 +- pipelines/datasets/br_ans_beneficiario/schedules.py | 2 +- pipelines/datasets/br_ans_beneficiario/tasks.py | 5 ----- pipelines/datasets/br_ans_beneficiario/utils.py | 4 ---- pipelines/datasets/br_b3_cotacoes/constants.py | 1 - pipelines/datasets/br_b3_cotacoes/schedules.py | 2 +- pipelines/datasets/br_bcb_agencia/tasks.py | 2 +- pipelines/datasets/br_bcb_estban/flows.py | 2 +- pipelines/datasets/br_bcb_estban/tasks.py | 3 +-- pipelines/datasets/br_bcb_taxa_cambio/tasks.py | 1 - pipelines/datasets/br_bcb_taxa_cambio/utils.py | 3 +-- pipelines/datasets/br_bd_indicadores/flows.py | 2 +- pipelines/datasets/br_cgu_pessoal_executivo_federal/flows.py | 2 +- pipelines/datasets/br_cvm_administradores_carteira/flows.py | 2 +- pipelines/datasets/br_cvm_administradores_carteira/tasks.py | 1 - pipelines/datasets/br_cvm_fi/schedules.py | 2 +- .../datasets/br_cvm_oferta_publica_distribuicao/flows.py | 2 +- pipelines/datasets/br_fgv_igp/flows.py | 2 +- pipelines/datasets/br_ibge_pnadc/tasks.py | 1 - pipelines/datasets/br_inmet_bdmep/schedules.py | 2 +- pipelines/datasets/br_inmet_bdmep/tasks.py | 1 - pipelines/datasets/br_me_cnpj/schedules.py | 2 +- pipelines/datasets/br_me_cnpj/tasks.py | 3 +-- pipelines/datasets/br_me_cnpj/utils.py | 2 -- pipelines/datasets/br_me_comex_stat/flows.py | 2 +- pipelines/datasets/br_me_comex_stat/schedules.py | 2 +- pipelines/datasets/br_me_comex_stat/tasks.py | 4 ---- pipelines/datasets/br_mg_belohorizonte_smfa_iptu/utils.py | 1 - pipelines/datasets/br_ms_cnes/flows.py | 2 +- pipelines/datasets/br_ms_cnes/tasks.py | 4 +--- pipelines/datasets/br_ms_cnes/utils.py | 1 - pipelines/datasets/br_ons_avaliacao_operacao/flows.py | 2 +- pipelines/datasets/br_ons_avaliacao_operacao/tasks.py | 1 - pipelines/datasets/br_ons_avaliacao_operacao/utils.py | 1 - pipelines/datasets/br_ons_estimativa_custos/flows.py | 2 +- pipelines/datasets/br_ons_estimativa_custos/schedules.py | 1 - pipelines/datasets/br_ons_estimativa_custos/utils.py | 1 - pipelines/datasets/br_poder360_pesquisas/flows.py | 2 +- pipelines/datasets/br_rf_cafir/flows.py | 2 +- pipelines/datasets/br_rf_cafir/tasks.py | 3 +-- pipelines/datasets/br_rj_isp_estatisticas_seguranca/utils.py | 1 - pipelines/datasets/br_sp_saopaulo_dieese_icv/flows.py | 2 +- pipelines/datasets/br_tse_eleicoes/flows.py | 2 +- pipelines/datasets/mundo_transfermarkt_competicoes/flows.py | 1 - .../datasets/mundo_transfermarkt_competicoes/schedules.py | 2 +- pipelines/datasets/mundo_transfermarkt_competicoes/tasks.py | 3 --- pipelines/datasets/mundo_transfermarkt_competicoes/utils.py | 1 - pipelines/utils/crawler_ibge_inflacao/flows.py | 2 +- pipelines/utils/metadata/tasks.py | 1 - pipelines/utils/metadata/utils.py | 3 +-- pipelines/utils/tasks.py | 1 - pipelines/utils/temporal_coverage_updater/tasks.py | 2 -- pipelines/utils/temporal_coverage_updater/utils.py | 1 - pipelines/utils/utils.py | 1 - 61 files changed, 33 insertions(+), 83 deletions(-) diff --git a/.github/workflows/scripts/register_flows.py b/.github/workflows/scripts/register_flows.py index 130323b44..3ee6a3a7a 100644 --- a/.github/workflows/scripts/register_flows.py +++ b/.github/workflows/scripts/register_flows.py @@ -10,7 +10,6 @@ ) import glob import hashlib -import importlib import json import os from pathlib import Path diff --git a/pipelines/datasets/br_anatel_banda_larga_fixa/schedules.py b/pipelines/datasets/br_anatel_banda_larga_fixa/schedules.py index e9ec786e9..586770201 100644 --- a/pipelines/datasets/br_anatel_banda_larga_fixa/schedules.py +++ b/pipelines/datasets/br_anatel_banda_larga_fixa/schedules.py @@ -3,7 +3,7 @@ Schedules for dataset br_anatel_banda_larga_fixa """ -from datetime import timedelta, datetime +from datetime import datetime from prefect.schedules import Schedule from prefect.schedules.clocks import CronClock from prefect.schedules.clocks import IntervalClock diff --git a/pipelines/datasets/br_anatel_banda_larga_fixa/tasks.py b/pipelines/datasets/br_anatel_banda_larga_fixa/tasks.py index 186e3d61c..655c53d35 100644 --- a/pipelines/datasets/br_anatel_banda_larga_fixa/tasks.py +++ b/pipelines/datasets/br_anatel_banda_larga_fixa/tasks.py @@ -4,8 +4,6 @@ """ import pandas as pd import numpy as np -from zipfile import ZipFile -from pathlib import Path import os from datetime import datetime, timedelta diff --git a/pipelines/datasets/br_anatel_banda_larga_fixa/utils.py b/pipelines/datasets/br_anatel_banda_larga_fixa/utils.py index 3465890c9..2fc71e3f4 100644 --- a/pipelines/datasets/br_anatel_banda_larga_fixa/utils.py +++ b/pipelines/datasets/br_anatel_banda_larga_fixa/utils.py @@ -8,7 +8,6 @@ import os import pandas as pd import numpy as np -from datetime import datetime, timedelta from pathlib import Path diff --git a/pipelines/datasets/br_anatel_telefonia_movel/schedules.py b/pipelines/datasets/br_anatel_telefonia_movel/schedules.py index 03f169872..7bdaeeb7a 100644 --- a/pipelines/datasets/br_anatel_telefonia_movel/schedules.py +++ b/pipelines/datasets/br_anatel_telefonia_movel/schedules.py @@ -2,7 +2,7 @@ """ Schedules for dataset br_anatel_telefonia_movel """ -from datetime import timedelta, datetime +from datetime import datetime from prefect.schedules import Schedule from prefect.schedules.clocks import IntervalClock from pipelines.constants import constants diff --git a/pipelines/datasets/br_anatel_telefonia_movel/utils.py b/pipelines/datasets/br_anatel_telefonia_movel/utils.py index 6f27b495f..c1d7071d2 100644 --- a/pipelines/datasets/br_anatel_telefonia_movel/utils.py +++ b/pipelines/datasets/br_anatel_telefonia_movel/utils.py @@ -10,7 +10,6 @@ import os import pandas as pd import numpy as np -from datetime import datetime, timedelta from pathlib import Path diff --git a/pipelines/datasets/br_anp_precos_combustiveis/schedules.py b/pipelines/datasets/br_anp_precos_combustiveis/schedules.py index 9c660ab04..b1891de93 100644 --- a/pipelines/datasets/br_anp_precos_combustiveis/schedules.py +++ b/pipelines/datasets/br_anp_precos_combustiveis/schedules.py @@ -3,7 +3,7 @@ Schedules for br_anp_precos_combustiveis """ -from datetime import timedelta, datetime +from datetime import datetime from prefect.schedules import Schedule from prefect.schedules.clocks import CronClock from pipelines.constants import constants diff --git a/pipelines/datasets/br_ans_beneficiario/flows.py b/pipelines/datasets/br_ans_beneficiario/flows.py index 64f6a8783..0b4b968fa 100644 --- a/pipelines/datasets/br_ans_beneficiario/flows.py +++ b/pipelines/datasets/br_ans_beneficiario/flows.py @@ -4,7 +4,7 @@ """ -from datetime import datetime, timedelta +from datetime import timedelta from prefect.run_configs import KubernetesRun from prefect.storage import GCS from pipelines.constants import constants diff --git a/pipelines/datasets/br_ans_beneficiario/schedules.py b/pipelines/datasets/br_ans_beneficiario/schedules.py index 564583556..e607a6702 100644 --- a/pipelines/datasets/br_ans_beneficiario/schedules.py +++ b/pipelines/datasets/br_ans_beneficiario/schedules.py @@ -4,7 +4,7 @@ """ -from datetime import timedelta, datetime +from datetime import datetime from prefect.schedules import Schedule from pipelines.constants import constants from prefect.schedules.clocks import CronClock diff --git a/pipelines/datasets/br_ans_beneficiario/tasks.py b/pipelines/datasets/br_ans_beneficiario/tasks.py index c178dfdfe..6149a93b4 100644 --- a/pipelines/datasets/br_ans_beneficiario/tasks.py +++ b/pipelines/datasets/br_ans_beneficiario/tasks.py @@ -5,17 +5,12 @@ from bs4 import BeautifulSoup import re import pandas as pd -from multiprocessing import Pool from datetime import datetime from loguru import logger -from pathlib import Path -from functools import reduce from prefect import task import os from tqdm import tqdm -import zipfile import requests -from glob import glob from pipelines.datasets.br_ans_beneficiario.utils import ( download_unzip_csv, get_url_from_template, diff --git a/pipelines/datasets/br_ans_beneficiario/utils.py b/pipelines/datasets/br_ans_beneficiario/utils.py index 700b4e515..37107da69 100644 --- a/pipelines/datasets/br_ans_beneficiario/utils.py +++ b/pipelines/datasets/br_ans_beneficiario/utils.py @@ -6,11 +6,8 @@ # from multiprocessing import Pool from loguru import logger -from pathlib import Path -from datetime import datetime -from io import BytesIO import os from tqdm import tqdm @@ -18,7 +15,6 @@ from pipelines.datasets.br_ans_beneficiario.constants import constants as ans_constants import zipfile import requests -from functools import reduce from dateutil.relativedelta import relativedelta from pipelines.utils.utils import ( log, diff --git a/pipelines/datasets/br_b3_cotacoes/constants.py b/pipelines/datasets/br_b3_cotacoes/constants.py index 9be80291e..982b6c2cb 100644 --- a/pipelines/datasets/br_b3_cotacoes/constants.py +++ b/pipelines/datasets/br_b3_cotacoes/constants.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- from enum import Enum -from datetime import datetime, timedelta class constants(Enum): # pylint: disable=c0103 diff --git a/pipelines/datasets/br_b3_cotacoes/schedules.py b/pipelines/datasets/br_b3_cotacoes/schedules.py index bb60c1d17..c2e3a93db 100644 --- a/pipelines/datasets/br_b3_cotacoes/schedules.py +++ b/pipelines/datasets/br_b3_cotacoes/schedules.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from datetime import timedelta, datetime +from datetime import datetime from prefect.schedules import Schedule from prefect.schedules.clocks import CronClock from pipelines.constants import constants diff --git a/pipelines/datasets/br_bcb_agencia/tasks.py b/pipelines/datasets/br_bcb_agencia/tasks.py index b02805c2a..dfb3c4cfb 100644 --- a/pipelines/datasets/br_bcb_agencia/tasks.py +++ b/pipelines/datasets/br_bcb_agencia/tasks.py @@ -6,7 +6,7 @@ import os import pandas as pd -from datetime import timedelta, datetime +from datetime import timedelta from pipelines.datasets.br_bcb_agencia.constants import ( constants as agencia_constants, diff --git a/pipelines/datasets/br_bcb_estban/flows.py b/pipelines/datasets/br_bcb_estban/flows.py index 324307a75..46aec245d 100644 --- a/pipelines/datasets/br_bcb_estban/flows.py +++ b/pipelines/datasets/br_bcb_estban/flows.py @@ -3,7 +3,7 @@ Flows for br_bcb_estban """ -from datetime import datetime, timedelta +from datetime import timedelta from prefect.run_configs import KubernetesRun from prefect.storage import GCS from prefect import Parameter, case diff --git a/pipelines/datasets/br_bcb_estban/tasks.py b/pipelines/datasets/br_bcb_estban/tasks.py index 5b331c0a7..f894a714c 100644 --- a/pipelines/datasets/br_bcb_estban/tasks.py +++ b/pipelines/datasets/br_bcb_estban/tasks.py @@ -7,13 +7,12 @@ import pandas as pd import os import basedosdados as bd -import glob from pipelines.constants import constants from pipelines.datasets.br_bcb_estban.constants import ( constants as br_bcb_estban_constants, ) -from datetime import datetime, timedelta +from datetime import timedelta from pipelines.utils.utils import ( clean_dataframe, diff --git a/pipelines/datasets/br_bcb_taxa_cambio/tasks.py b/pipelines/datasets/br_bcb_taxa_cambio/tasks.py index 2b28e34c0..1e745675c 100644 --- a/pipelines/datasets/br_bcb_taxa_cambio/tasks.py +++ b/pipelines/datasets/br_bcb_taxa_cambio/tasks.py @@ -3,7 +3,6 @@ Tasks for br_bcb_indicadores """ -import datetime from pipelines.utils.utils import log, to_partitions from prefect import task import pandas as pd diff --git a/pipelines/datasets/br_bcb_taxa_cambio/utils.py b/pipelines/datasets/br_bcb_taxa_cambio/utils.py index 5512316dc..1a822e33d 100644 --- a/pipelines/datasets/br_bcb_taxa_cambio/utils.py +++ b/pipelines/datasets/br_bcb_taxa_cambio/utils.py @@ -8,9 +8,8 @@ import pandas as pd import datetime import pytz -from datetime import timedelta import os -from io import BytesIO, StringIO +from io import BytesIO import time as tm from pipelines.datasets.br_bcb_taxa_cambio.constants import constants as bcb_constants from pipelines.utils.utils import log diff --git a/pipelines/datasets/br_bd_indicadores/flows.py b/pipelines/datasets/br_bd_indicadores/flows.py index 53fbabd69..4080de496 100755 --- a/pipelines/datasets/br_bd_indicadores/flows.py +++ b/pipelines/datasets/br_bd_indicadores/flows.py @@ -3,7 +3,7 @@ Flows for br_bd_indicadores """ # pylint: disable=invalid-name -from datetime import timedelta, datetime +from datetime import timedelta from prefect import Parameter, case from prefect.run_configs import KubernetesRun diff --git a/pipelines/datasets/br_cgu_pessoal_executivo_federal/flows.py b/pipelines/datasets/br_cgu_pessoal_executivo_federal/flows.py index fdc22b0b6..936f4a8f1 100644 --- a/pipelines/datasets/br_cgu_pessoal_executivo_federal/flows.py +++ b/pipelines/datasets/br_cgu_pessoal_executivo_federal/flows.py @@ -2,7 +2,7 @@ """ Flows for br_cgu_terceirizados """ -from datetime import datetime, timedelta +from datetime import timedelta from prefect.run_configs import KubernetesRun from prefect.storage import GCS diff --git a/pipelines/datasets/br_cvm_administradores_carteira/flows.py b/pipelines/datasets/br_cvm_administradores_carteira/flows.py index 04c7d9916..87a402e7c 100644 --- a/pipelines/datasets/br_cvm_administradores_carteira/flows.py +++ b/pipelines/datasets/br_cvm_administradores_carteira/flows.py @@ -3,7 +3,7 @@ Flows for br_cvm_administradores_carteira """ # pylint: disable=C0103, E1123, invalid-name -from datetime import datetime, timedelta +from datetime import timedelta from prefect import Parameter, case from prefect.run_configs import KubernetesRun diff --git a/pipelines/datasets/br_cvm_administradores_carteira/tasks.py b/pipelines/datasets/br_cvm_administradores_carteira/tasks.py index 8128d25b8..c202cc2aa 100644 --- a/pipelines/datasets/br_cvm_administradores_carteira/tasks.py +++ b/pipelines/datasets/br_cvm_administradores_carteira/tasks.py @@ -13,7 +13,6 @@ from prefect import task from unidecode import unidecode import basedosdados as bd -from datetime import datetime from pipelines.utils.utils import log diff --git a/pipelines/datasets/br_cvm_fi/schedules.py b/pipelines/datasets/br_cvm_fi/schedules.py index 7815da136..e6df000f7 100644 --- a/pipelines/datasets/br_cvm_fi/schedules.py +++ b/pipelines/datasets/br_cvm_fi/schedules.py @@ -3,7 +3,7 @@ Schedules for br_cvm_fi """ -from datetime import timedelta, datetime +from datetime import datetime from prefect.schedules import Schedule from prefect.schedules.clocks import IntervalClock from pipelines.constants import constants diff --git a/pipelines/datasets/br_cvm_oferta_publica_distribuicao/flows.py b/pipelines/datasets/br_cvm_oferta_publica_distribuicao/flows.py index 368820b61..20ef02fb8 100644 --- a/pipelines/datasets/br_cvm_oferta_publica_distribuicao/flows.py +++ b/pipelines/datasets/br_cvm_oferta_publica_distribuicao/flows.py @@ -3,7 +3,7 @@ Flows for br_cvm_oferta_publica_distribuicao """ # pylint: disable=C0103, E1123, invalid-name -from datetime import datetime, timedelta +from datetime import timedelta from prefect.run_configs import KubernetesRun from prefect.storage import GCS diff --git a/pipelines/datasets/br_fgv_igp/flows.py b/pipelines/datasets/br_fgv_igp/flows.py index e0fa21aae..6b241a01a 100644 --- a/pipelines/datasets/br_fgv_igp/flows.py +++ b/pipelines/datasets/br_fgv_igp/flows.py @@ -4,7 +4,7 @@ """ # pylint: disable=invalid-name -from datetime import timedelta, datetime +from datetime import timedelta from pathlib import Path from prefect import Parameter, case diff --git a/pipelines/datasets/br_ibge_pnadc/tasks.py b/pipelines/datasets/br_ibge_pnadc/tasks.py index 7f643f32a..4c9804a7e 100644 --- a/pipelines/datasets/br_ibge_pnadc/tasks.py +++ b/pipelines/datasets/br_ibge_pnadc/tasks.py @@ -12,7 +12,6 @@ import pandas as pd import numpy as np from prefect import task -from datetime import datetime from pipelines.utils.utils import log from pipelines.datasets.br_ibge_pnadc.constants import constants as pnad_constants diff --git a/pipelines/datasets/br_inmet_bdmep/schedules.py b/pipelines/datasets/br_inmet_bdmep/schedules.py index 00f791738..d89df7466 100644 --- a/pipelines/datasets/br_inmet_bdmep/schedules.py +++ b/pipelines/datasets/br_inmet_bdmep/schedules.py @@ -70,7 +70,7 @@ ############################################################################### -from datetime import timedelta, datetime +from datetime import datetime from prefect.schedules import Schedule from prefect.schedules.clocks import IntervalClock from pipelines.constants import constants diff --git a/pipelines/datasets/br_inmet_bdmep/tasks.py b/pipelines/datasets/br_inmet_bdmep/tasks.py index ddf155e87..bae6ad11a 100644 --- a/pipelines/datasets/br_inmet_bdmep/tasks.py +++ b/pipelines/datasets/br_inmet_bdmep/tasks.py @@ -16,7 +16,6 @@ import os import numpy as np import glob -from datetime import datetime, time from prefect import task from pipelines.datasets.br_inmet_bdmep.constants import constants as inmet_constants diff --git a/pipelines/datasets/br_me_cnpj/schedules.py b/pipelines/datasets/br_me_cnpj/schedules.py index 2dc5581b8..610e61d5d 100644 --- a/pipelines/datasets/br_me_cnpj/schedules.py +++ b/pipelines/datasets/br_me_cnpj/schedules.py @@ -5,7 +5,7 @@ ############################################################################### from prefect.schedules.clocks import CronClock -from datetime import timedelta, datetime +from datetime import datetime from prefect.schedules import Schedule from pipelines.constants import constants diff --git a/pipelines/datasets/br_me_cnpj/tasks.py b/pipelines/datasets/br_me_cnpj/tasks.py index 085081f5a..70c1ac83e 100644 --- a/pipelines/datasets/br_me_cnpj/tasks.py +++ b/pipelines/datasets/br_me_cnpj/tasks.py @@ -18,9 +18,8 @@ ) import os import requests -import zipfile import pandas as pd -from datetime import datetime, timedelta +from datetime import datetime from tqdm import tqdm ufs = constants_cnpj.UFS.value diff --git a/pipelines/datasets/br_me_cnpj/utils.py b/pipelines/datasets/br_me_cnpj/utils.py index f40b9bd22..7d201b4c3 100644 --- a/pipelines/datasets/br_me_cnpj/utils.py +++ b/pipelines/datasets/br_me_cnpj/utils.py @@ -15,8 +15,6 @@ from tqdm import tqdm import pyarrow.parquet as pq import pyarrow as pa -import csv -from typing import List ufs = constants_cnpj.UFS.value headers = constants_cnpj.HEADERS.value diff --git a/pipelines/datasets/br_me_comex_stat/flows.py b/pipelines/datasets/br_me_comex_stat/flows.py index 6c7475bce..7edf8b613 100644 --- a/pipelines/datasets/br_me_comex_stat/flows.py +++ b/pipelines/datasets/br_me_comex_stat/flows.py @@ -3,7 +3,7 @@ Flows for br_me_comex_stat """ # pylint: disable=invalid-name -from datetime import datetime, timedelta +from datetime import timedelta from prefect import Parameter, case from prefect.run_configs import KubernetesRun diff --git a/pipelines/datasets/br_me_comex_stat/schedules.py b/pipelines/datasets/br_me_comex_stat/schedules.py index e9c50e63f..5066f20c4 100644 --- a/pipelines/datasets/br_me_comex_stat/schedules.py +++ b/pipelines/datasets/br_me_comex_stat/schedules.py @@ -4,7 +4,7 @@ Schedules for br_me_comex_stat """ -from datetime import timedelta, datetime +from datetime import datetime from prefect.schedules import Schedule, filters, adjustments from prefect.schedules.clocks import CronClock diff --git a/pipelines/datasets/br_me_comex_stat/tasks.py b/pipelines/datasets/br_me_comex_stat/tasks.py index 8cf4a96b9..e0bb923e5 100644 --- a/pipelines/datasets/br_me_comex_stat/tasks.py +++ b/pipelines/datasets/br_me_comex_stat/tasks.py @@ -4,15 +4,11 @@ Tasks for br_me_comex_stat """ # pylint: disable=invalid-name,too-many-nested-blocks -from glob import glob from zipfile import ZipFile import time as tm -from datetime import timedelta import pandas as pd import numpy as np from prefect import task -from pathlib import Path -from typing import Union, List import basedosdados as bd import os diff --git a/pipelines/datasets/br_mg_belohorizonte_smfa_iptu/utils.py b/pipelines/datasets/br_mg_belohorizonte_smfa_iptu/utils.py index 9196a5a2f..7437bc12b 100644 --- a/pipelines/datasets/br_mg_belohorizonte_smfa_iptu/utils.py +++ b/pipelines/datasets/br_mg_belohorizonte_smfa_iptu/utils.py @@ -8,7 +8,6 @@ import requests from bs4 import BeautifulSoup import numpy as np -from datetime import datetime import geopandas as gpd from shapely import wkt from pipelines.utils.tasks import log diff --git a/pipelines/datasets/br_ms_cnes/flows.py b/pipelines/datasets/br_ms_cnes/flows.py index be11de58b..c65d8e69a 100644 --- a/pipelines/datasets/br_ms_cnes/flows.py +++ b/pipelines/datasets/br_ms_cnes/flows.py @@ -3,7 +3,7 @@ Flows for br_ms_cnes """ # pylint: disable=invalid-name -from datetime import datetime, timedelta +from datetime import timedelta from prefect import Parameter, case from pipelines.constants import constants diff --git a/pipelines/datasets/br_ms_cnes/tasks.py b/pipelines/datasets/br_ms_cnes/tasks.py index caa54ccd4..3ca592a83 100644 --- a/pipelines/datasets/br_ms_cnes/tasks.py +++ b/pipelines/datasets/br_ms_cnes/tasks.py @@ -5,12 +5,11 @@ from prefect import task -from datetime import timedelta, datetime +from datetime import timedelta from pipelines.utils.utils import log from pipelines.constants import constants import pandas as pd -from ftplib import FTP import wget import os from rpy2.robjects.packages import importr @@ -18,7 +17,6 @@ import rpy2.robjects as ro from rpy2.robjects import pandas2ri -import datetime as dt from pipelines.datasets.br_ms_cnes.constants import constants as cnes_constants from pipelines.datasets.br_ms_cnes.utils import ( diff --git a/pipelines/datasets/br_ms_cnes/utils.py b/pipelines/datasets/br_ms_cnes/utils.py index c105c44bf..277cb3a0f 100644 --- a/pipelines/datasets/br_ms_cnes/utils.py +++ b/pipelines/datasets/br_ms_cnes/utils.py @@ -6,7 +6,6 @@ from ftplib import FTP import pandas as pd import basedosdados as bd -from datetime import datetime from pipelines.utils.utils import log diff --git a/pipelines/datasets/br_ons_avaliacao_operacao/flows.py b/pipelines/datasets/br_ons_avaliacao_operacao/flows.py index 4144f267a..7c70f267c 100644 --- a/pipelines/datasets/br_ons_avaliacao_operacao/flows.py +++ b/pipelines/datasets/br_ons_avaliacao_operacao/flows.py @@ -3,7 +3,7 @@ Flows for br_ons_avaliacao_operacao """ # pylint: disable=invalid-name -from datetime import datetime, timedelta +from datetime import timedelta from prefect import Parameter, case from prefect.run_configs import KubernetesRun diff --git a/pipelines/datasets/br_ons_avaliacao_operacao/tasks.py b/pipelines/datasets/br_ons_avaliacao_operacao/tasks.py index 13921e94f..3d5ac5b8d 100644 --- a/pipelines/datasets/br_ons_avaliacao_operacao/tasks.py +++ b/pipelines/datasets/br_ons_avaliacao_operacao/tasks.py @@ -4,7 +4,6 @@ """ import os import pandas as pd -from datetime import datetime from prefect import task from pipelines.utils.utils import ( diff --git a/pipelines/datasets/br_ons_avaliacao_operacao/utils.py b/pipelines/datasets/br_ons_avaliacao_operacao/utils.py index c0d4775fb..2f3e614f6 100644 --- a/pipelines/datasets/br_ons_avaliacao_operacao/utils.py +++ b/pipelines/datasets/br_ons_avaliacao_operacao/utils.py @@ -7,7 +7,6 @@ import pandas as pd from io import StringIO from typing import List -from typing import Dict import time as tm import unicodedata diff --git a/pipelines/datasets/br_ons_estimativa_custos/flows.py b/pipelines/datasets/br_ons_estimativa_custos/flows.py index 1fe8d4c32..9fcf1d64b 100644 --- a/pipelines/datasets/br_ons_estimativa_custos/flows.py +++ b/pipelines/datasets/br_ons_estimativa_custos/flows.py @@ -3,7 +3,7 @@ Flows for br_ons_estimativa_custos """ # pylint: disable=invalid-name -from datetime import datetime, timedelta +from datetime import timedelta from prefect import Parameter, case from prefect.run_configs import KubernetesRun diff --git a/pipelines/datasets/br_ons_estimativa_custos/schedules.py b/pipelines/datasets/br_ons_estimativa_custos/schedules.py index ceeec6245..429e567d6 100644 --- a/pipelines/datasets/br_ons_estimativa_custos/schedules.py +++ b/pipelines/datasets/br_ons_estimativa_custos/schedules.py @@ -3,7 +3,6 @@ Schedules for br_ons_estimativa_custos """ -from datetime import timedelta, datetime from prefect.schedules import Schedule, filters, adjustments from prefect.schedules.clocks import CronClock diff --git a/pipelines/datasets/br_ons_estimativa_custos/utils.py b/pipelines/datasets/br_ons_estimativa_custos/utils.py index adc88d921..1782dd92a 100644 --- a/pipelines/datasets/br_ons_estimativa_custos/utils.py +++ b/pipelines/datasets/br_ons_estimativa_custos/utils.py @@ -10,7 +10,6 @@ import pandas as pd from io import StringIO from typing import List -from typing import Dict import time as tm import unicodedata diff --git a/pipelines/datasets/br_poder360_pesquisas/flows.py b/pipelines/datasets/br_poder360_pesquisas/flows.py index 7bc638321..00600eb8e 100644 --- a/pipelines/datasets/br_poder360_pesquisas/flows.py +++ b/pipelines/datasets/br_poder360_pesquisas/flows.py @@ -3,7 +3,7 @@ Flows for br_poder360_pesquisas """ -from datetime import datetime, timedelta +from datetime import timedelta from prefect.run_configs import KubernetesRun from prefect.storage import GCS diff --git a/pipelines/datasets/br_rf_cafir/flows.py b/pipelines/datasets/br_rf_cafir/flows.py index 63b9fdb60..e364acf8d 100644 --- a/pipelines/datasets/br_rf_cafir/flows.py +++ b/pipelines/datasets/br_rf_cafir/flows.py @@ -3,7 +3,7 @@ Flows for br_rf_cafir """ # pylint: disable=invalid-name -from datetime import datetime, timedelta +from datetime import timedelta from prefect import Parameter, case from pipelines.constants import constants diff --git a/pipelines/datasets/br_rf_cafir/tasks.py b/pipelines/datasets/br_rf_cafir/tasks.py index 8c4b9b5b9..b2565d967 100644 --- a/pipelines/datasets/br_rf_cafir/tasks.py +++ b/pipelines/datasets/br_rf_cafir/tasks.py @@ -7,8 +7,7 @@ from prefect import task from pipelines.utils.utils import log from pipelines.constants import constants -import re -from datetime import datetime, timedelta +from datetime import datetime import os import pandas as pd diff --git a/pipelines/datasets/br_rj_isp_estatisticas_seguranca/utils.py b/pipelines/datasets/br_rj_isp_estatisticas_seguranca/utils.py index 67fcee0a8..09954b35c 100644 --- a/pipelines/datasets/br_rj_isp_estatisticas_seguranca/utils.py +++ b/pipelines/datasets/br_rj_isp_estatisticas_seguranca/utils.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import pandas as pd from io import StringIO -import os import requests from typing import List from typing import Dict diff --git a/pipelines/datasets/br_sp_saopaulo_dieese_icv/flows.py b/pipelines/datasets/br_sp_saopaulo_dieese_icv/flows.py index 55ec12014..f61b42310 100644 --- a/pipelines/datasets/br_sp_saopaulo_dieese_icv/flows.py +++ b/pipelines/datasets/br_sp_saopaulo_dieese_icv/flows.py @@ -4,7 +4,7 @@ """ # pylint: disable=C0103, E1123 -from datetime import datetime, timedelta +from datetime import timedelta from prefect.run_configs import KubernetesRun from prefect.storage import GCS diff --git a/pipelines/datasets/br_tse_eleicoes/flows.py b/pipelines/datasets/br_tse_eleicoes/flows.py index f505f0401..09774bcf3 100644 --- a/pipelines/datasets/br_tse_eleicoes/flows.py +++ b/pipelines/datasets/br_tse_eleicoes/flows.py @@ -3,7 +3,7 @@ Flows for br_tse_eleicoes """ # pylint: disable=invalid-name,line-too-long -from datetime import datetime, timedelta +from datetime import timedelta from prefect import Parameter, case from prefect.tasks.prefect import create_flow_run, wait_for_flow_run diff --git a/pipelines/datasets/mundo_transfermarkt_competicoes/flows.py b/pipelines/datasets/mundo_transfermarkt_competicoes/flows.py index b1cfa0c5e..fc6815d2a 100644 --- a/pipelines/datasets/mundo_transfermarkt_competicoes/flows.py +++ b/pipelines/datasets/mundo_transfermarkt_competicoes/flows.py @@ -37,7 +37,6 @@ from pipelines.utils.constants import constants as utils_constants from pipelines.utils.metadata.tasks import update_django_metadata from datetime import timedelta -import asyncio with Flow( name="mundo_transfermarkt_competicoes.brasileirao_serie_a", diff --git a/pipelines/datasets/mundo_transfermarkt_competicoes/schedules.py b/pipelines/datasets/mundo_transfermarkt_competicoes/schedules.py index 6ca65e2a6..cc4e740a4 100644 --- a/pipelines/datasets/mundo_transfermarkt_competicoes/schedules.py +++ b/pipelines/datasets/mundo_transfermarkt_competicoes/schedules.py @@ -6,7 +6,7 @@ ############################################################################### from prefect.schedules.clocks import CronClock -from datetime import timedelta, datetime +from datetime import datetime from prefect.schedules import Schedule from pipelines.constants import constants diff --git a/pipelines/datasets/mundo_transfermarkt_competicoes/tasks.py b/pipelines/datasets/mundo_transfermarkt_competicoes/tasks.py index 07084eafb..e968be59a 100644 --- a/pipelines/datasets/mundo_transfermarkt_competicoes/tasks.py +++ b/pipelines/datasets/mundo_transfermarkt_competicoes/tasks.py @@ -13,12 +13,9 @@ ) from pipelines.utils.utils import log, to_partitions from prefect import task -import re import numpy as np import pandas as pd import asyncio -import os -from datetime import timedelta, datetime @task diff --git a/pipelines/datasets/mundo_transfermarkt_competicoes/utils.py b/pipelines/datasets/mundo_transfermarkt_competicoes/utils.py index dba0e9f62..8e6f79ce0 100644 --- a/pipelines/datasets/mundo_transfermarkt_competicoes/utils.py +++ b/pipelines/datasets/mundo_transfermarkt_competicoes/utils.py @@ -9,7 +9,6 @@ import requests import numpy as np import pandas as pd -import asyncio from pipelines.datasets.mundo_transfermarkt_competicoes.constants import ( constants as mundo_constants, ) diff --git a/pipelines/utils/crawler_ibge_inflacao/flows.py b/pipelines/utils/crawler_ibge_inflacao/flows.py index 601537bee..bb27deda1 100644 --- a/pipelines/utils/crawler_ibge_inflacao/flows.py +++ b/pipelines/utils/crawler_ibge_inflacao/flows.py @@ -3,7 +3,7 @@ Flows for ibge inflacao """ # pylint: disable=C0103, E1123, invalid-name, duplicate-code, R0801 -from datetime import datetime, timedelta +from datetime import timedelta from prefect import Parameter, case from prefect.tasks.prefect import create_flow_run, wait_for_flow_run diff --git a/pipelines/utils/metadata/tasks.py b/pipelines/utils/metadata/tasks.py index 40e99a8b0..b3593fe54 100644 --- a/pipelines/utils/metadata/tasks.py +++ b/pipelines/utils/metadata/tasks.py @@ -6,7 +6,6 @@ from prefect import task from datetime import datetime from pipelines.utils.utils import log, get_credentials_from_secret -from typing import Tuple from pipelines.utils.metadata.utils import ( get_ids, parse_temporal_coverage, diff --git a/pipelines/utils/metadata/utils.py b/pipelines/utils/metadata/utils.py index 61f30024e..ea004cad4 100644 --- a/pipelines/utils/metadata/utils.py +++ b/pipelines/utils/metadata/utils.py @@ -7,8 +7,7 @@ # pylint: disable=too-many-arguments from datetime import datetime -from os import getenv, walk -from typing import Any, Dict, List, Tuple +from typing import Tuple import basedosdados as bd import numpy as np diff --git a/pipelines/utils/tasks.py b/pipelines/utils/tasks.py index 381ccfb31..d39cc2d94 100644 --- a/pipelines/utils/tasks.py +++ b/pipelines/utils/tasks.py @@ -30,7 +30,6 @@ get_credentials_from_secret, get_token, ) -from typing import Tuple @task diff --git a/pipelines/utils/temporal_coverage_updater/tasks.py b/pipelines/utils/temporal_coverage_updater/tasks.py index 8ba76d908..2e305f1bb 100644 --- a/pipelines/utils/temporal_coverage_updater/tasks.py +++ b/pipelines/utils/temporal_coverage_updater/tasks.py @@ -16,9 +16,7 @@ extract_last_update, get_first_date, ) -from datetime import datetime from pipelines.utils.utils import log, get_credentials_from_secret -from typing import Tuple ## TODO: Transformar flow em task OK diff --git a/pipelines/utils/temporal_coverage_updater/utils.py b/pipelines/utils/temporal_coverage_updater/utils.py index 89a0711a9..02dcd1771 100644 --- a/pipelines/utils/temporal_coverage_updater/utils.py +++ b/pipelines/utils/temporal_coverage_updater/utils.py @@ -2,7 +2,6 @@ """ General purpose functions for the temporal_coverage_updater project """ -import os import json import requests from datetime import datetime diff --git a/pipelines/utils/utils.py b/pipelines/utils/utils.py index 57dc2758e..1c1147b99 100644 --- a/pipelines/utils/utils.py +++ b/pipelines/utils/utils.py @@ -31,7 +31,6 @@ from pipelines.constants import constants -import os import re