Merge branch 'main' into br-camara-dados

dbt_project.yml

@@ -111,12 +111,15 @@ models:
     br_cgu_beneficios_cidadao:
       +materialized: table
       +schema: br_cgu_beneficios_cidadao
-    br_cgu_servidores_executivo_federal:
+    br_cgu_dados_abertos:
       +materialized: table
-      +schema: br_cgu_servidores_executivo_federal
+      +schema: br_cgu_dados_abertos
     br_cgu_pessoal_executivo_federal:
       +materialized: table
       +schema: br_cgu_pessoal_executivo_federal
+    br_cgu_servidores_executivo_federal:
+      +materialized: table
+      +schema: br_cgu_servidores_executivo_federal
     br_cnj_improbidade_administrativa:
       +materialized: table
       +schema: br_cnj_improbidade_administrativa

models/br_bcb_agencia/schema.yml

@@ -7,31 +7,26 @@ models:
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns: [ano, mes, cnpj]
+      - not_null_proportion_multiple_columns:
+          at_least: 0.05      
     columns:
       - name: ano
         description: Ano
       - name: mes
         description: Mês
       - name: sigla_uf
         description: Sigla da Unidade da Federação
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: id_municipio
         description: ID Município - IBGE 7 Dígitos
         tests:
           - relationships:
               to: ref('br_bd_diretorios_brasil__municipio')
               field: id_municipio
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: data_inicio
         description: Data de criação da agência
       - name: cnpj
         description: Cadastro Nacional de Pessoa Júridica (CNPJ)
         tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
           - relationships:
               to: ref('br_bd_diretorios_brasil__empresa')
               field: cnpj
@@ -53,8 +48,6 @@ models:
           - relationships:
               to: ref('br_bd_diretorios_brasil__cep')
               field: cep.cep
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: endereco
         description: endereço da agência
       - name: complemento

models/br_bcb_estban/schema.yml

@@ -14,30 +14,25 @@ models:
             - id_municipio
             - cnpj_basico
             - id_verbete
+      - not_null_proportion_multiple_columns:
+          at_least: 0.05            
     columns:
       - name: ano
         description: Ano
       - name: mes
         description: Mês
       - name: sigla_uf
         description: Sigla da Unidade da Federação
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: id_municipio
         description: ID Município - IBGE 7 Dígitos
         tests:
           - relationships:
               to: ref('br_bd_diretorios_brasil__municipio')
               field: id_municipio
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: cnpj_basico
         description: Cadastro Nacional de Pessoa Jurídica (CNPJ) básico (8 primeiros
           dígitos)
         tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
           - relationships:
               to: ref('br_bd_diretorios_brasil__empresa')
               field: cnpj_basico
@@ -49,9 +44,6 @@ models:
         description: Quantidade de agências processadas da IF no município
       - name: id_verbete
         description: Código do verbete
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: valor
         description: Valores
   - name: br_bcb_estban__agencia
@@ -67,24 +59,21 @@ models:
             - id_municipio
             - cnpj_agencia
             - id_verbete
+      - not_null_proportion_multiple_columns:
+          at_least: 0.05            
     columns:
       - name: ano
         description: Ano
       - name: mes
         description: Mês
       - name: sigla_uf
         description: Sigla da Unidade da Federação
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: id_municipio
         description: ID Município - IBGE 7 Dígitos
         tests:
           - relationships:
               to: ref('br_bd_diretorios_brasil__municipio')
               field: id_municipio
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: cnpj_basico
         description: Cadastro Nacional de Pessoa Jurídica (CNPJ) básico (8 primeiros
           dígitos)
@@ -93,15 +82,10 @@ models:
       - name: cnpj_agencia
         description: CNPJ da agência bancária
         tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
           - relationships:
               to: ref('br_bd_diretorios_brasil__empresa')
               field: cnpj
       - name: id_verbete
         description: Código do verbete
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: valor
         description: Valores

models/br_bd_metadados/schema.yml

@@ -29,183 +29,85 @@ models:
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns: [flow_group_id]
+      - not_null_proportion_multiple_columns:
+          at_least: 0.05          
     columns:
       - name: flow_group_id
         description: Flow group identifier
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: name
         description: Flow's name
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: created
         description: Timestamp indicating when the flow was initially created
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: latest_version
         description: Version number of the latest iteration of the flow
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: last_update
         description: Timestamp of the last update made to the flow
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_type
         description: Type of scheduling employed for the flow (e.g. CronClock, IntervalClock)
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_cron
         description: Cron expression defining the flow's scheduling pattern
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_start_date
         description: Date when the scheduled execution of the flow starts
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_filters
         description: Filters applied to refine the scheduled execution criteria
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_adjustments
         description: Adjustments or configurations for the flow's scheduling
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_labels
         description: Labels in the flow's scheduling
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_all_parameters
         description: All parameters in the schedule
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_parameters_dataset_id
         description: Dataset used in scheduling parameters.
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_parameters_table_id
         description: Table used in scheduling parameters
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_parameters_dbt_alias
         description: Indicates if the model uses alias in DBT
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_parameters_materialization_mode
         description: Mode defining where data is materialized
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_parameters_materialize_after_dump
         description: Indicates if materialization occurs after data dump
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_parameters_update_metadata
         description: Indicates if metadata is updated
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
   - name: br_bd_metadados__prefect_flow_runs
     description: Insert `prefect_flow_runs` table description here
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns: [id]
+      - not_null_proportion_multiple_columns:
+          at_least: 0.05          
     columns:
       - name: id
         description: Unique identifier for the flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: flow_group_id
         description: Unique identifier for the flow group
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: name
         description: Name associated with the specific flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: labels
         description: Labels (determines the project the data will be materialized)
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: flow_project_name
         description: Name of the project linked to this flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: flow_name
         description: Name of the flow linked to this flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: flow_archived
         description: Boolean indicating whether the flow run is archived or active
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: dataset_id
         description: BigQuery dataset id relevant to this flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: table_id
         description: BigQuery table id relevant to this flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: start_time
         description: Date and time when the flow run was initiated
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: end_time
         description: Date and time when the flow run was completed or ended
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: state
         description: Current state of the flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: state_message
         description: Additional message or information related to the state/status
           of the flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: task_runs
         description: Information or details regarding individual task runs within
           this flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: skipped_upload_to_gcs
         description: Boolean indicating whether the upload to GCS (Google Cloud Storage)
           was skipped or not for this flow run.
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: error_logs
         description: Error log records
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05

models/br_camara_dados_abertos/schema.yml

@@ -1713,3 +1713,4 @@ models:
         description: Valor que o parlamentar devolveu à Câmara
       - name: url_documento
         description: URL do documento
+

models/br_cgu_dados_abertos/br_cgu_dados_abertos__conjunto.sql

@@ -0,0 +1,21 @@
+{{ config(alias="conjunto", schema="br_cgu_dados_abertos") }}
+--
+select
+    safe_cast(nullif(id, "") as string) id,
+    safe_cast(nullif(titulo, "") as string) nome,
+    safe_cast(nullif(nome, "") as string) nome_tokenizado,
+    safe_cast(nullif(descricao, "") as string) descricao,
+    safe_cast(nullif(mantenedor, "") as string) mantenedor,
+    safe_cast(nullif(email_mantenedor, "") as string) email_mantenedor,
+    safe_cast(
+        nullif(id_organizacao_responsavel, "") as string
+    ) id_organizacao_responsavel,
+    safe_cast(nullif(organizacao_responsavel, "") as string) organizacao_responsavel,
+    extract(date from safe.parse_datetime('%d/%m/%Y %T', data_criacao)) data_criacao,
+    extract(
+        date from safe.parse_datetime('%d/%m/%Y %T', data_atualizacao)
+    ) data_atualizacao,
+    safe_cast(quantidade_reusos as int64) quantidade_reusos,
+    safe_cast(quantidade_recursos as int64) quantidade_recursos,
+    safe_cast(quantidade_downloads as int64) quantidade_downloads,
+from `basedosdados-staging.br_cgu_dados_abertos_staging.conjunto` as c

models/br_cgu_dados_abertos/br_cgu_dados_abertos__organizacao.sql

@@ -0,0 +1,25 @@
+{{ config(alias="organizacao", schema="br_cgu_dados_abertos") }}
+--
+select
+    safe_cast(nullif(o.id, "") as string) id,
+    safe_cast(nullif(o.titulo, "") as string) nome,
+    safe_cast(nullif(o.nome, "") as string) nome_tokenizado,
+    safe_cast(nullif(o.descricao, "") as string) descricao,
+    case
+        when o.organizationesfera = "1"
+        then "Federal"
+        when o.organizationesfera = "2"
+        then "Estadual/Distrital"
+        when o.organizationesfera = "3"
+        then "Municipal"
+        else null
+    end tipo_esfera_administrativa,
+    safe_cast(nullif(o.organizationuf, "") as string) sigla_uf,
+    safe_cast(m.id_municipio as string) id_municipio,
+    safe_cast(o.qtdseguidores as int64) quantidade_seguidores,
+    safe_cast(o.qtdconjuntodedados as int64) quantidade_conjuntos
+from `basedosdados-staging.br_cgu_dados_abertos_staging.organizacao` as o
+left join
+    `basedosdados.br_bd_diretorios_brasil.municipio` as m
+    on o.organizationuf = m.sigla_uf
+    and o.organizationmunicipio = m.nome