[dados]: br_cgu_servidores_executivo_federal (#489)

* br_cgu_servidores_executivo_federal: init

* move comment

* fix: no objects to concat

* fix: sheet url

* fix: remove empty tables

* fix: remove iter flows.py

* fix

* add cond tasks

* fix encoding

* rename columns

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix: convert cols to float

* add is_up_to_date task

* test

* simplify read_and_clean_csv

* move to dev mode

* fix api_mode

* update upload.py

* rename tables names

* rm upload.py

* api_mode to prod

* fix table name

* param to prod

* rm comment

* rm tests

* add params

* update shedule

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

pipelines/datasets/__init__.py

@@ -19,6 +19,7 @@
 from pipelines.datasets.br_bd_indicadores.flows import *
 from pipelines.datasets.br_bd_metadados.flows import *
 from pipelines.datasets.br_cgu_pessoal_executivo_federal.flows import *
+from pipelines.datasets.br_cgu_servidores_executivo_federal.flows import *
 from pipelines.datasets.br_cvm_administradores_carteira.flows import *
 from pipelines.datasets.br_cvm_fi.flows import *
 from pipelines.datasets.br_cvm_oferta_publica_distribuicao.flows import *

pipelines/datasets/br_cgu_servidores_executivo_federal/constants.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+"""
+Constant values for the datasets projects
+"""
+
+from enum import Enum
+
+# fmt: off
+__ALL_SHEETS__ = [
+    "Aposentados_BACEN",
+    "Aposentados_SIAPE",
+    "Militares",
+    "Pensionistas_BACEN",
+    "Pensionistas_DEFESA",
+    "Pensionistas_SIAPE",
+    "Reserva_Reforma_Militares",
+    "Servidores_BACEN",
+    "Servidores_SIAPE",
+]
+# fmt: on
+
+
+class constants(Enum):  # pylint: disable=c0103
+    """
+    Constant values for the br_cgu_servidores_executivo_federal project
+    """
+
+    URL = "http://portaldatransparencia.gov.br/download-de-dados/servidores"
+
+    # fmt: off
+    SHEETS = __ALL_SHEETS__,
+    # fmt: on
+
+    TABLES = {
+        "cadastro_aposentados": ["Aposentados_BACEN", "Aposentados_SIAPE"],
+        "cadastro_pensionistas": [
+            "Pensionistas_SIAPE",
+            "Pensionistas_DEFESA",
+            "Pensionistas_BACEN",
+        ],
+        "cadastro_servidores": ["Servidores_BACEN", "Servidores_SIAPE", "Militares"],
+        "cadastro_reserva_reforma_militares": ["Reserva_Reforma_Militares"],
+        "remuneracao": [
+            "Militares",
+            "Pensionistas_BACEN",
+            "Pensionistas_DEFESA",
+            "Reserva_Reforma_Militares",
+            "Servidores_BACEN",
+            "Servidores_SIAPE",
+        ],
+        "afastamentos": ["Servidores_BACEN", "Servidores_SIAPE"],
+        "observacoes": __ALL_SHEETS__,
+    }
+
+    ARCH = {
+        "afastamentos": "https://docs.google.com/spreadsheets/d/1NQ4t9l8znClnfM8NYBLBkI9PoWV5UAosUZ1KGvZe-T0/edit#gid=0",
+        "cadastro_aposentados": "https://docs.google.com/spreadsheets/d/1_t_JsWbuGlg8cz_2RYYNMuulzA4RHydfJ4TA-wH9Ch8/edit#gid=0",
+        "observacoes": "https://docs.google.com/spreadsheets/d/1BWt6yvKTfNW0XCDNIsIu8NhSKjhbDVJjnEwvnEmVkRc/edit#gid=0",
+        "cadastro_pensionistas": "https://docs.google.com/spreadsheets/d/1G_RPhSUZRrCqcQCP1WSjBiYbnjirqTp0yaLNUaPi_7U/edit#gid=0",
+        "remuneracao": "https://docs.google.com/spreadsheets/d/1LJ8_N53OoNEQQ1PMAeIPKq1asB29MUP5SRoFWnUI6Zg/edit#gid=0",
+        "cadastro_reserva_reforma_militares": "https://docs.google.com/spreadsheets/d/1vqWjATWjHK-6tbj_ilwbhNWReqe3AKd3VTpzBbPu4qI/edit#gid=0",
+        "cadastro_servidores": "https://docs.google.com/spreadsheets/d/1U57P5XhCw9gERD8sN24P0vDK0CjkUuOQZwOoELdE3Jg/edit#gid=0",
+    }
+
+    INPUT = "/tmp/input"
+
+    OUTPUT = "/tmp/output"