Staging/detran

.gitignore

@@ -148,8 +148,12 @@ notebooks/
 /tests/
 
 
+
+
 .DS_Store
 
 # Mac
 .DS_Store
 
+
+/DENATRAN_FILES

.pre-commit-config.yaml

@@ -31,4 +31,4 @@ repos:
     rev: 7.0.0
     hooks:
       - id: flake8
-        exclude: 'pipelines\/\{\{cookiecutter\.project_name\}\}.*'
+        exclude: 'pipelines\/\{\{cookiecutter\.project_name\}\}.*'

pipelines/datasets/__init__.py

@@ -25,6 +25,7 @@
 from pipelines.datasets.br_cvm_administradores_carteira.flows import *
 from pipelines.datasets.br_cvm_fi.flows import *
 from pipelines.datasets.br_cvm_oferta_publica_distribuicao.flows import *
+from pipelines.datasets.br_denatran_frota.flows import *
 from pipelines.datasets.br_fgv_igp.flows import *
 from pipelines.datasets.br_ibge_inpc.flows import *
 from pipelines.datasets.br_ibge_ipca15.flows import *

pipelines/datasets/br_denatran_frota/backfill.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+from pipelines.datasets.br_denatran_frota.constants import constants
+from pipelines.datasets.br_denatran_frota.handlers import (
+    crawl,
+    get_desired_file,
+    output_file_to_csv,
+    treat_uf_tipo,
+)
+
+# Fill for UF TIPO
+months = range(1, 13)
+years = range(2003, 2023)
+for year in years:
+    for month in months:
+        print(month)
+        crawl(month=month, year=year, temp_dir="DENATRAN_FILES")
+        file = get_desired_file(
+            year=year,
+            download_directory="DENATRAN_FILES",
+            filetype=f"{constants.UF_TIPO_BASIC_FILENAME.value}_{month}",
+        )
+        if year == 2004 and month == 3:
+            breakpoint()
+        df = treat_uf_tipo(file=file)
+        path = output_file_to_csv(
+            df=df, filename=constants.UF_TIPO_BASIC_FILENAME.value
+        )

pipelines/datasets/br_denatran_frota/constants.py

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+"""
+Constant values for the datasets projects
+"""
+
+from enum import Enum
+
+
+class constants(Enum):  # pylint: disable=c0103
+    """
+    Constant values for the br_denatran_frota project
+    """
+
+    MONTHS = {
+        "janeiro": 1,
+        "fevereiro": 2,
+        "marco": 3,
+        "abril": 4,
+        "maio": 5,
+        "junho": 6,
+        "julho": 7,
+        "agosto": 8,
+        "setembro": 9,
+        "outubro": 10,
+        "novembro": 11,
+        "dezembro": 12,
+        # some months have capital letters deppending on the year
+        "Janeiro": 1,
+        "Fevereiro": 2,
+        "Marco": 3,
+        "Abril": 4,
+        "Maio": 5,
+        "Junho": 6,
+        "Julho": 7,
+        "Agosto": 8,
+        "Setembro": 9,
+        "Outubro": 10,
+        "Novembro": 11,
+        "Dezembro": 12,
+    }
+
+    DATASET = "br_denatran_frota"
+
+    HEADERS = {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36"
+    }
+
+    DICT_UFS = {
+        "AC": "Acre",
+        "AL": "Alagoas",
+        "AP": "Amapá",
+        "AM": "Amazonas",
+        "BA": "Bahia",
+        "CE": "Ceará",
+        "DF": "Distrito Federal",
+        "ES": "Espírito Santo",
+        "GO": "Goiás",
+        "MA": "Maranhão",
+        "MT": "Mato Grosso",
+        "MS": "Mato Grosso do Sul",
+        "MG": "Minas Gerais",
+        "PA": "Pará",
+        "PB": "Paraíba",
+        "PR": "Paraná",
+        "PE": "Pernambuco",
+        "PI": "Piauí",
+        "RJ": "Rio de Janeiro",
+        "RN": "Rio Grande do Norte",
+        "RS": "Rio Grande do Sul",
+        "RO": "Rondônia",
+        "RR": "Roraima",
+        "SC": "Santa Catarina",
+        "SP": "São Paulo",
+        "SE": "Sergipe",
+        "TO": "Tocantins",
+    }
+
+    SUBSTITUTIONS = {
+        ("RN", "assu"): "acu",
+        ("PB", "sao domingos de pombal"): "sao domingos",
+        ("PB", "santarem"): "joca claudino",
+        ("SP", "embu"): "embu das artes",
+        ("TO", "sao valerio da natividade"): "sao valerio",
+        ("PB", "campo de santana"): "tacima",
+        ("AP", "amapari"): "pedra branca do amapari",
+        ("BA", "maracani"): "macarani",
+        ("BA", "livramento do brumado"): "livramento de nossa senhora",
+        ("PB", "sao bento de pombal"): "sao bentinho",
+        ("PB", "serido"): "sao vicente do serido",
+        ("PR", "vila alta"): "alto paraiso",
+        ("RN", "espirito santo do oeste"): "parau",
+        ("RO", "jamari"): "itapua do oeste",
+        ("SC", "picarras"): "balneario picarras",
+        ("SC", "barra do sul"): "balneario barra do sul",
+    }
+
+    DOWNLOAD_PATH = f"/tmp/input/{DATASET}"
+
+    OUTPUT_PATH = f"/tmp/output/{DATASET}"
+
+    UF_TIPO_BASIC_FILENAME = "frota_por_uf_e_tipo_de_veiculo"
+
+    MUNIC_TIPO_BASIC_FILENAME = "frota_por_municipio_e_tipo"
+
+    MONTHS_SHORT = {month[:3]: number for month, number in MONTHS.items()}
+
+    UF_TIPO_HEADER = [
+        "Grandes Regiões e\nUnidades da Federação",
+        "TOTAL",
+        "AUTOMÓVEL",
+        "BONDE",
+        "CAMINHÃO",
+        "CAMINHÃO TRATOR",
+        "CAMINHONETE",
+        "CAMIONETA",
+        "CHASSI PLATAFORMA",
+        "CICLOMOTOR",
+        "MICROÔNIBUS",
+        "MOTOCICLETA",
+        "MOTONETA",
+        "ÔNIBUS",
+        "QUADRICICLO",
+        "REBOQUE",
+        "SEMI-REBOQUE",
+        "SIDE-CAR",
+        "OUTROS",
+        "TRATOR ESTEIRA",
+        "TRATOR RODAS",
+        "TRICICLO",
+        "UTILITÁRIO",
+    ]
+
+    MUNICIPIO_TIPO_HEADER = [
+        "UF",
+        "MUNICIPIO",
+        "TOTAL",
+        "AUTOMÓVEL",
+        "BONDE",
+        "CAMINHÃO",
+        "CAMINHÃO TRATOR",
+        "CAMINHONETE",
+        "CAMIONETA",
+        "CHASSI PLATAFORMA",
+        "CICLOMOTOR",
+        "MICROÔNIBUS",
+        "MOTOCICLETA",
+        "MOTONETA",
+        "ÔNIBUS",
+        "QUADRICICLO",
+        "REBOQUE",
+        "SEMI-REBOQUE",
+        "SIDE-CAR",
+        "OUTROS",
+        "TRATOR ESTEIRA",
+        "TRATOR RODAS",
+        "TRICICLO",
+        "UTILITÁRIO",
+    ]