Merge branch 'main' into update_br_inep_indicadores_educacionaiss

basedosdados · Mar 8, 2024 · a614a58 · a614a58
2 parents 2687a71 + 6fd2817
commit a614a58
Show file tree

Hide file tree

Showing 32 changed files with 52,857 additions and 2,783 deletions.
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -111,12 +111,15 @@ models:
     br_cgu_beneficios_cidadao:
       +materialized: table
       +schema: br_cgu_beneficios_cidadao
-    br_cgu_servidores_executivo_federal:
+    br_cgu_dados_abertos:
       +materialized: table
-      +schema: br_cgu_servidores_executivo_federal
+      +schema: br_cgu_dados_abertos
     br_cgu_pessoal_executivo_federal:
       +materialized: table
       +schema: br_cgu_pessoal_executivo_federal
+    br_cgu_servidores_executivo_federal:
+      +materialized: table
+      +schema: br_cgu_servidores_executivo_federal
     br_cnj_improbidade_administrativa:
       +materialized: table
       +schema: br_cnj_improbidade_administrativa
@@ -291,6 +294,9 @@ models:
     mundo_transfermarkt_competicoes_internacionais:
       +materialized: table
       +schema: mundo_transfermarkt_competicoes_internacionais
+    world_ampas_oscar:
+      +materialized: table
+      +schema: world_ampas_oscar    
     world_iea_pirls:
       +materialized: table
       +schema: world_iea_pirls

diff --git a/models/br_ans_beneficiario/informacao_consolidada.sql b/models/br_ans_beneficiario/informacao_consolidada.sql
@@ -5,7 +5,7 @@
         partition_by={
             "field": "ano",
             "data_type": "int64",
-            "range": {"start": 2014, "end": 2023, "interval": 1},
+            "range": {"start": 2014, "end": 2024, "interval": 1},
         },
         cluster_by=["id_municipio", "mes", "sigla_uf"],
         labels={"project_id": "basedosdados"},

diff --git a/models/br_bcb_agencia/schema.yml b/models/br_bcb_agencia/schema.yml
@@ -7,31 +7,26 @@ models:
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns: [ano, mes, cnpj]
+      - not_null_proportion_multiple_columns:
+          at_least: 0.05      
     columns:
       - name: ano
         description: Ano
       - name: mes
         description: Mês
       - name: sigla_uf
         description: Sigla da Unidade da Federação
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: id_municipio
         description: ID Município - IBGE 7 Dígitos
         tests:
           - relationships:
               to: ref('br_bd_diretorios_brasil__municipio')
               field: id_municipio
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: data_inicio
         description: Data de criação da agência
       - name: cnpj
         description: Cadastro Nacional de Pessoa Júridica (CNPJ)
         tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
           - relationships:
               to: ref('br_bd_diretorios_brasil__empresa')
               field: cnpj
@@ -53,8 +48,6 @@ models:
           - relationships:
               to: ref('br_bd_diretorios_brasil__cep')
               field: cep.cep
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: endereco
         description: endereço da agência
       - name: complemento

diff --git a/models/br_bcb_estban/schema.yml b/models/br_bcb_estban/schema.yml
@@ -14,30 +14,25 @@ models:
             - id_municipio
             - cnpj_basico
             - id_verbete
+      - not_null_proportion_multiple_columns:
+          at_least: 0.05            
     columns:
       - name: ano
         description: Ano
       - name: mes
         description: Mês
       - name: sigla_uf
         description: Sigla da Unidade da Federação
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: id_municipio
         description: ID Município - IBGE 7 Dígitos
         tests:
           - relationships:
               to: ref('br_bd_diretorios_brasil__municipio')
               field: id_municipio
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: cnpj_basico
         description: Cadastro Nacional de Pessoa Jurídica (CNPJ) básico (8 primeiros
           dígitos)
         tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
           - relationships:
               to: ref('br_bd_diretorios_brasil__empresa')
               field: cnpj_basico
@@ -49,9 +44,6 @@ models:
         description: Quantidade de agências processadas da IF no município
       - name: id_verbete
         description: Código do verbete
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: valor
         description: Valores
   - name: br_bcb_estban__agencia
@@ -67,24 +59,21 @@ models:
             - id_municipio
             - cnpj_agencia
             - id_verbete
+      - not_null_proportion_multiple_columns:
+          at_least: 0.05            
     columns:
       - name: ano
         description: Ano
       - name: mes
         description: Mês
       - name: sigla_uf
         description: Sigla da Unidade da Federação
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: id_municipio
         description: ID Município - IBGE 7 Dígitos
         tests:
           - relationships:
               to: ref('br_bd_diretorios_brasil__municipio')
               field: id_municipio
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: cnpj_basico
         description: Cadastro Nacional de Pessoa Jurídica (CNPJ) básico (8 primeiros
           dígitos)
@@ -93,15 +82,10 @@ models:
       - name: cnpj_agencia
         description: CNPJ da agência bancária
         tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
           - relationships:
               to: ref('br_bd_diretorios_brasil__empresa')
               field: cnpj
       - name: id_verbete
         description: Código do verbete
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: valor
         description: Valores
diff --git a/models/br_bd_metadados/schema.yml b/models/br_bd_metadados/schema.yml
@@ -29,183 +29,85 @@ models:
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns: [flow_group_id]
+      - not_null_proportion_multiple_columns:
+          at_least: 0.05          
     columns:
       - name: flow_group_id
         description: Flow group identifier
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: name
         description: Flow's name
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: created
         description: Timestamp indicating when the flow was initially created
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: latest_version
         description: Version number of the latest iteration of the flow
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: last_update
         description: Timestamp of the last update made to the flow
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_type
         description: Type of scheduling employed for the flow (e.g. CronClock, IntervalClock)
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_cron
         description: Cron expression defining the flow's scheduling pattern
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_start_date
         description: Date when the scheduled execution of the flow starts
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_filters
         description: Filters applied to refine the scheduled execution criteria
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_adjustments
         description: Adjustments or configurations for the flow's scheduling
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_labels
         description: Labels in the flow's scheduling
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_all_parameters
         description: All parameters in the schedule
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_parameters_dataset_id
         description: Dataset used in scheduling parameters.
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_parameters_table_id
         description: Table used in scheduling parameters
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_parameters_dbt_alias
         description: Indicates if the model uses alias in DBT
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_parameters_materialization_mode
         description: Mode defining where data is materialized
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_parameters_materialize_after_dump
         description: Indicates if materialization occurs after data dump
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: schedule_parameters_update_metadata
         description: Indicates if metadata is updated
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
   - name: br_bd_metadados__prefect_flow_runs
     description: Insert `prefect_flow_runs` table description here
     tests:
       - dbt_utils.unique_combination_of_columns:
           combination_of_columns: [id]
+      - not_null_proportion_multiple_columns:
+          at_least: 0.05          
     columns:
       - name: id
         description: Unique identifier for the flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: flow_group_id
         description: Unique identifier for the flow group
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: name
         description: Name associated with the specific flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: labels
         description: Labels (determines the project the data will be materialized)
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: flow_project_name
         description: Name of the project linked to this flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: flow_name
         description: Name of the flow linked to this flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: flow_archived
         description: Boolean indicating whether the flow run is archived or active
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: dataset_id
         description: BigQuery dataset id relevant to this flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: table_id
         description: BigQuery table id relevant to this flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: start_time
         description: Date and time when the flow run was initiated
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: end_time
         description: Date and time when the flow run was completed or ended
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: state
         description: Current state of the flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: state_message
         description: Additional message or information related to the state/status
           of the flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: task_runs
         description: Information or details regarding individual task runs within
           this flow run
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: skipped_upload_to_gcs
         description: Boolean indicating whether the upload to GCS (Google Cloud Storage)
           was skipped or not for this flow run.
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05
       - name: error_logs
         description: Error log records
-        tests:
-          - dbt_utils.not_null_proportion:
-              at_least: 0.05