Merge pull request #446 from basedosdados/fix-action-pylint

fix: atualizar o workflow lint python

.flake8

@@ -1,5 +1,4 @@
 [flake8]
 ignore = E203,F401,F403,W503,E501,E266
-exclude =
-    *cookiecutter*
-max-line-length = 100
+exclude = *cookiecutter*
+max-line-length = 100

.github/workflows/lint_python.yaml

@@ -8,31 +8,24 @@ jobs:
     name: Lint Python
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v2
+
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: "3.8.x"
-      - name: Set up Poetry and upgrade pip
+          python-version: "3.9.x"
+
+      - name: Set up poetry and upgrade pip
         run: |
           pip install -U pip poetry
-      - name: Install flows
-        run: |
-          pip install --prefer-binary .
-      - name: Lint flows
-        uses: gabriel-milan/action-pylint@v1
-        with:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
-          path: "pipelines/"
-          fail: true
-          pr-message: true
-      - name: Install Pytest
-        run: |
-          pip install pytest
-      - name: Lint tests
-        uses: gabriel-milan/action-pylint@v1
-        with:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
-          path: "tests/"
-          fail: true
-          pr-message: true
+          pip install isort
+          pip install black
+          pip install autoflake
+          pip install flake8
+
+      - name: Install this package
+        run: poetry install
+
+      - name: Lint source code
+        run: poetry run lint

.github/workflows/scripts/register_flows.py

@@ -10,7 +10,6 @@
 )
 import glob
 import hashlib
-import importlib
 import json
 import os
 from pathlib import Path

.pre-commit-config.yaml

@@ -1,20 +1,23 @@
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:
-    - id: check-added-large-files # prevents adding large files
-    - id: detect-private-key # detects private keys
-    - id: fix-byte-order-marker # fixes BOM
-    - id: fix-encoding-pragma # fixes encoding pragma
-    - id: no-commit-to-branch # prevents committing to protected branches
-    - id: trailing-whitespace # prevents trailing whitespace
--   repo: https://github.com/psf/black
+      - id: check-added-large-files # prevents adding large files
+      - id: detect-private-key # detects private keys
+      - id: fix-byte-order-marker # fixes BOM
+      - id: fix-encoding-pragma # fixes encoding pragma
+      - id: no-commit-to-branch # prevents committing to protected branches
+      - id: trailing-whitespace # prevents trailing whitespace
+  - repo: https://github.com/psf/black
     rev: 23.9.1
     hooks:
-    -   id: black
-
+      - id: black
         exclude: 'pipelines\/\{\{cookiecutter\.project_name\}\}.*'
--   repo: https://github.com/PyCQA/flake8
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.2.1
+    hooks:
+      - id: autoflake
+  - repo: https://github.com/PyCQA/flake8
     rev: 6.1.0
     hooks:
-    - id: flake8
+      - id: flake8

Makefile

@@ -0,0 +1,13 @@
+# Get the executables we're using
+POETRY=$(shell which poetry)
+PYTHON=$(shell poetry run which python)
+
+# `make install`: installs dependencies
+.PHONY: install
+install:
+	$(POETRY) install
+
+# `make install`: installs pre-commit
+.PHONY: install_precommit
+install_precommit:
+	$(POETRY) run pre-commit install --install-hooks

pipelines/datasets/br_anatel_banda_larga_fixa/schedules.py

@@ -3,7 +3,7 @@
 Schedules for dataset br_anatel_banda_larga_fixa
 """
 
-from datetime import timedelta, datetime
+from datetime import datetime
 from prefect.schedules import Schedule
 from prefect.schedules.clocks import CronClock
 from prefect.schedules.clocks import IntervalClock

pipelines/datasets/br_anatel_banda_larga_fixa/tasks.py

@@ -4,8 +4,6 @@
 """
 import pandas as pd
 import numpy as np
-from zipfile import ZipFile
-from pathlib import Path
 import os
 
 from datetime import datetime, timedelta

pipelines/datasets/br_anatel_banda_larga_fixa/utils.py

@@ -8,7 +8,6 @@
 import os
 import pandas as pd
 import numpy as np
-from datetime import datetime, timedelta
 from pathlib import Path
 
 

pipelines/datasets/br_anatel_telefonia_movel/schedules.py

@@ -2,7 +2,7 @@
 """
 Schedules for dataset br_anatel_telefonia_movel
 """
-from datetime import timedelta, datetime
+from datetime import datetime
 from prefect.schedules import Schedule
 from prefect.schedules.clocks import IntervalClock
 from pipelines.constants import constants

pipelines/datasets/br_anatel_telefonia_movel/utils.py

@@ -10,7 +10,6 @@
 import os
 import pandas as pd
 import numpy as np
-from datetime import datetime, timedelta
 from pathlib import Path
 
 

pipelines/datasets/br_anp_precos_combustiveis/schedules.py

@@ -3,7 +3,7 @@
 Schedules for br_anp_precos_combustiveis
 """
 
-from datetime import timedelta, datetime
+from datetime import datetime
 from prefect.schedules import Schedule
 from prefect.schedules.clocks import CronClock
 from pipelines.constants import constants

pipelines/datasets/br_ans_beneficiario/flows.py

@@ -4,7 +4,7 @@
 """
 
 
-from datetime import datetime, timedelta
+from datetime import timedelta
 from prefect.run_configs import KubernetesRun
 from prefect.storage import GCS
 from pipelines.constants import constants

pipelines/datasets/br_ans_beneficiario/schedules.py

@@ -4,7 +4,7 @@
 """
 
 
-from datetime import timedelta, datetime
+from datetime import datetime
 from prefect.schedules import Schedule
 from pipelines.constants import constants
 from prefect.schedules.clocks import CronClock

pipelines/datasets/br_ans_beneficiario/tasks.py

@@ -5,17 +5,12 @@
 from bs4 import BeautifulSoup
 import re
 import pandas as pd
-from multiprocessing import Pool
 from datetime import datetime
 from loguru import logger
-from pathlib import Path
-from functools import reduce
 from prefect import task
 import os
 from tqdm import tqdm
-import zipfile
 import requests
-from glob import glob
 from pipelines.datasets.br_ans_beneficiario.utils import (
     download_unzip_csv,
     get_url_from_template,

pipelines/datasets/br_ans_beneficiario/utils.py

@@ -6,19 +6,15 @@
 
 # from multiprocessing import Pool
 from loguru import logger
-from pathlib import Path
 
 
-from datetime import datetime
-from io import BytesIO
 import os
 from tqdm import tqdm
 
 # import tempfile
 from pipelines.datasets.br_ans_beneficiario.constants import constants as ans_constants
 import zipfile
 import requests
-from functools import reduce
 from dateutil.relativedelta import relativedelta
 from pipelines.utils.utils import (
     log,

pipelines/datasets/br_b3_cotacoes/constants.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 from enum import Enum
-from datetime import datetime, timedelta
 
 
 class constants(Enum):  # pylint: disable=c0103

pipelines/datasets/br_b3_cotacoes/schedules.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from datetime import timedelta, datetime
+from datetime import datetime
 from prefect.schedules import Schedule
 from prefect.schedules.clocks import CronClock
 from pipelines.constants import constants

pipelines/datasets/br_bcb_agencia/tasks.py

@@ -6,7 +6,7 @@
 
 import os
 import pandas as pd
-from datetime import timedelta, datetime
+from datetime import timedelta
 
 from pipelines.datasets.br_bcb_agencia.constants import (
     constants as agencia_constants,

pipelines/datasets/br_bcb_estban/flows.py

@@ -3,7 +3,7 @@
 Flows for br_bcb_estban
 """
 
-from datetime import datetime, timedelta
+from datetime import timedelta
 from prefect.run_configs import KubernetesRun
 from prefect.storage import GCS
 from prefect import Parameter, case

pipelines/datasets/br_bcb_estban/tasks.py

@@ -7,13 +7,12 @@
 import pandas as pd
 import os
 import basedosdados as bd
-import glob
 
 from pipelines.constants import constants
 from pipelines.datasets.br_bcb_estban.constants import (
     constants as br_bcb_estban_constants,
 )
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from pipelines.utils.utils import (
     clean_dataframe,

pipelines/datasets/br_bcb_taxa_cambio/tasks.py

@@ -3,7 +3,6 @@
 Tasks for br_bcb_indicadores
 """
 
-import datetime
 from pipelines.utils.utils import log, to_partitions
 from prefect import task
 import pandas as pd

pipelines/datasets/br_bcb_taxa_cambio/utils.py

@@ -8,9 +8,8 @@
 import pandas as pd
 import datetime
 import pytz
-from datetime import timedelta
 import os
-from io import BytesIO, StringIO
+from io import BytesIO
 import time as tm
 from pipelines.datasets.br_bcb_taxa_cambio.constants import constants as bcb_constants
 from pipelines.utils.utils import log

pipelines/datasets/br_bd_indicadores/flows.py

@@ -3,7 +3,7 @@
 Flows for br_bd_indicadores
 """
 # pylint: disable=invalid-name
-from datetime import timedelta, datetime
+from datetime import timedelta
 
 from prefect import Parameter, case
 from prefect.run_configs import KubernetesRun

pipelines/datasets/br_cgu_pessoal_executivo_federal/flows.py

@@ -2,7 +2,7 @@
 """
 Flows for br_cgu_terceirizados
 """
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from prefect.run_configs import KubernetesRun
 from prefect.storage import GCS

pipelines/datasets/br_cvm_administradores_carteira/flows.py

@@ -3,7 +3,7 @@
 Flows for br_cvm_administradores_carteira
 """
 # pylint: disable=C0103, E1123, invalid-name
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from prefect import Parameter, case
 from prefect.run_configs import KubernetesRun

pipelines/datasets/br_cvm_administradores_carteira/tasks.py

@@ -13,7 +13,6 @@
 from prefect import task
 from unidecode import unidecode
 import basedosdados as bd
-from datetime import datetime
 from pipelines.utils.utils import log
 
 

pipelines/datasets/br_cvm_fi/schedules.py

@@ -3,7 +3,7 @@
 Schedules for br_cvm_fi
 """
 
-from datetime import timedelta, datetime
+from datetime import datetime
 from prefect.schedules import Schedule
 from prefect.schedules.clocks import IntervalClock
 from pipelines.constants import constants

pipelines/datasets/br_cvm_oferta_publica_distribuicao/flows.py

@@ -3,7 +3,7 @@
 Flows for br_cvm_oferta_publica_distribuicao
 """
 # pylint: disable=C0103, E1123, invalid-name
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 from prefect.run_configs import KubernetesRun
 from prefect.storage import GCS

pipelines/datasets/br_fgv_igp/flows.py

@@ -4,7 +4,7 @@
 """
 
 # pylint: disable=invalid-name
-from datetime import timedelta, datetime
+from datetime import timedelta
 from pathlib import Path
 
 from prefect import Parameter, case

pipelines/datasets/br_ibge_pnadc/tasks.py

@@ -12,7 +12,6 @@
 import pandas as pd
 import numpy as np
 from prefect import task
-from datetime import datetime
 
 from pipelines.utils.utils import log
 from pipelines.datasets.br_ibge_pnadc.constants import constants as pnad_constants

pipelines/datasets/br_inmet_bdmep/schedules.py

@@ -70,7 +70,7 @@
 ###############################################################################
 
 
-from datetime import timedelta, datetime
+from datetime import datetime
 from prefect.schedules import Schedule
 from prefect.schedules.clocks import IntervalClock
 from pipelines.constants import constants

pipelines/datasets/br_inmet_bdmep/tasks.py

@@ -16,7 +16,6 @@
 import os
 import numpy as np
 import glob
-from datetime import datetime, time
 from prefect import task
 from pipelines.datasets.br_inmet_bdmep.constants import constants as inmet_constants
 

pipelines/datasets/br_me_cnpj/schedules.py

@@ -5,7 +5,7 @@
 
 ###############################################################################
 from prefect.schedules.clocks import CronClock
-from datetime import timedelta, datetime
+from datetime import datetime
 from prefect.schedules import Schedule
 from pipelines.constants import constants
 

pipelines/datasets/br_me_cnpj/tasks.py

@@ -18,9 +18,8 @@
 )
 import os
 import requests
-import zipfile
 import pandas as pd
-from datetime import datetime, timedelta
+from datetime import datetime
 from tqdm import tqdm
 
 ufs = constants_cnpj.UFS.value