Merge branch 'main' into br-sinan-dengue

models/br_camara_dados_abertos/schema.yml

@@ -39,7 +39,7 @@ models:
               to: ref('br_camara_dados_abertos__evento')
               field: id_evento
               ignore_values: ['0', '71957', '71921']  # O valor 5410 será ignorado
-              proportion_allowed_failures: 0
+              proportion_allowed_failures: 0.2
       - name: aprovacao
         description: Indicador se ocorreu a aprovação
       - name: voto_sim
@@ -157,9 +157,11 @@ models:
       - name: id_votacao
         description: ID da votação
         tests:
-          - relationships:
+          - custom_relationships:
               to: ref('br_camara_dados_abertos__votacao')
               field: id_votacao
+              ignore_values: ['235536312']
+              proportion_allowed_failures: 0.2
       - name: sigla_orgao
         description: Sigla do órgão
       - name: descricao
@@ -194,12 +196,6 @@ models:
         description: Descrição sobre a votação
       - name: id_proposicao
         description: ID proposição
-        tests:
-          - custom_relationships:
-              to: ref('br_camara_dados_abertos__proposicao_microdados')
-              field: id_proposicao
-              ignore_values: ['33463', '130292', '42069']
-              proportion_allowed_failures: 0
       - name: ano_proposicao
         description: Ano da proposição
         tests:
@@ -582,6 +578,7 @@ models:
             - nome
             - nome_deputado
             - cargo
+            - sigla_uf
             - data_inicio
             - data_final
             - sigla_partido
@@ -778,17 +775,21 @@ models:
       - name: id_frente
         description: ID da frente
         tests:
-          - relationships:
+          - custom_relationships:
               to: ref('br_camara_dados_abertos__frente')
               field: id_frente
+              ignore_values: ['55614', '55619', '55621', '55604']
+              proportion_allowed_failures: 0.2
       - name: titulo_deputado
         description: Título do deputado
       - name: id_deputado
         description: ID do deputado
         tests:
-          - relationships:
+          - custom_relationships:
               to: ref('br_camara_dados_abertos__deputado')
               field: id_deputado
+              ignore_values: ['223398']
+              proportion_allowed_failures: 0.2
       - name: nome_deputado
         description: Nome do deputado
       - name: url_foto_deputado
@@ -1209,7 +1210,7 @@ models:
               to: ref('br_camara_dados_abertos__deputado')
               field: id_deputado
               ignore_values: ['228042', '223398']  # O valor 5410 será ignorado
-              proportion_allowed_failures: 0
+              proportion_allowed_failures: 0.2
       - name: ano_legislatura
         description: Ano de início da legislatura
         tests:

models/br_ms_cnes/br_ms_cnes__dados_complementares.sql

@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:[email protected]", "group:[email protected]")                     FILTER USING (True)',
-        ],
     )
 }}
 with

models/br_ms_cnes/br_ms_cnes__dicionario.sql

@@ -5,6 +5,6 @@ select
     safe_cast(id_tabela as string) id_tabela,
     safe_cast(nome_coluna as string) nome_coluna,
     safe_cast(chave as string) chave,
-    safe_cast(cobertura_temporal as string) cobertura_temporal,
+    safe_cast(replace(cobertura_temporal, '-1', '(1)') as string) cobertura_temporal,
     safe_cast(valor as string) valor,
 from `basedosdados-staging.br_ms_cnes_staging.dicionario` as t

models/br_ms_cnes/br_ms_cnes__equipamento.sql

@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:[email protected]", "group:[email protected]")                     FILTER USING (True)',
-        ],
     )
 }}
 
@@ -45,7 +40,7 @@ select
     safe_cast(sigla_uf as string) as sigla_uf,
     safe_cast(id_municipio as string) as id_municipio,
     safe_cast(cnes as string) as id_estabelecimento_cnes,
-    safe_cast(codequip as string) as id_equipamento,
+    ltrim(safe_cast(codequip as string), '0') as id_equipamento,
     safe_cast(tipequip as string) as tipo_equipamento,
     safe_cast(qt_exist as string) as quantidade_equipamentos,
     safe_cast(qt_uso as string) as quantidade_equipamentos_ativos,

models/br_ms_cnes/br_ms_cnes__equipe.sql

@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:[email protected]", "group:[email protected]")                     FILTER USING (True)',
-        ],
     )
 }}
 
@@ -46,17 +41,41 @@ select
     safe_cast(id_municipio as string) as id_municipio,
     safe_cast(cnes as string) as id_estabelecimento_cnes,
     safe_cast(id_equipe as string) as id_equipe,
-    safe_cast(tipo_eqp as string) as tipo_equipe,
+    safe_cast(
+        case
+            when regexp_replace(tipo_eqp, '^0+', '') = ''
+            then '0'
+            else regexp_replace(tipo_eqp, '^0+', '')
+        end as string
+    ) as tipo_equipe,
     safe_cast(nome_eqp as string) as equipe,
     safe_cast(nomearea as string) as area,
     safe_cast(id_segm as string) as id_segmento,
-    safe_cast(tiposegm as string) as tipo_segmento,
+    safe_cast(
+        case
+            when regexp_replace(tiposegm, '^0+', '') = ''
+            then '0'
+            else regexp_replace(tiposegm, '^0+', '')
+        end as string
+    ) as tipo_segmento,
     safe_cast(descsegm as string) as descricao_segmento,
     -- - inserir subsrt para criar ano e mes
     safe_cast(substr(dt_ativa, 1, 4) as int64) as ano_ativacao_equipe,
     safe_cast(substr(dt_ativa, 5, 6) as int64) as mes_ativacao_equipe,
-    safe_cast(motdesat as string) as motivo_desativacao_equipe,
-    safe_cast(tp_desat as string) as tipo_desativacao_equipe,
+    safe_cast(
+        case
+            when regexp_replace(motdesat, '^0+', '') = ''
+            then '0'
+            else regexp_replace(motdesat, '^0+', '')
+        end as string
+    ) as motivo_desativacao_equipe,
+    safe_cast(
+        case
+            when regexp_replace(tp_desat, '^0+', '') = ''
+            then '0'
+            else regexp_replace(tp_desat, '^0+', '')
+        end as string
+    ) as tipo_desativacao_equipe,
     safe_cast(substr(dt_desat, 1, 4) as int64) as ano_desativacao_equipe,
     safe_cast(substr(dt_desat, 5, 6) as int64) as mes_desativacao_equipe,
     safe_cast(quilombo as int64) as indicador_atende_populacao_assistida_quilombolas,

models/br_ms_cnes/br_ms_cnes__estabelecimento.sql

@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:[email protected]", "group:[email protected]")                     FILTER USING (True)',
-        ],
     )
 }}
 with
@@ -61,17 +56,51 @@ select
     -- fazer replace em valores de linha com 14 zeros para null. 14 zeros é o tamanho
     -- de um cpf/cnpj nulo
     safe_cast(regexp_replace(cnpj_man, '0{14}', '') as string) cnpj_mantenedora,
-    safe_cast({{ clean_cols("COD_IR") }} as string) tipo_retencao_tributos_mantenedora,
+    replace(
+        safe_cast({{ clean_cols("COD_IR") }} as string), 'nan', null
+    ) tipo_retencao_tributos_mantenedora,
     safe_cast(vinc_sus as int64) indicador_vinculo_sus,
     safe_cast(tpgestao as string) tipo_gestao,
-    safe_cast({{ clean_cols("ESFERA_A") }} as string) tipo_esfera_administrativa,
-    safe_cast(retencao as string) tipo_retencao_tributos,
+    replace(
+        safe_cast({{ clean_cols("ESFERA_A") }} as string), 'nan', null
+    ) tipo_esfera_administrativa,
+    replace(
+        safe_cast(
+            case
+                when regexp_replace(safe_cast(retencao as string), '^0+', '') = ''
+                then '0'
+                else regexp_replace(safe_cast(retencao as string), '^0+', '')
+            end as string
+        ),
+        'nan',
+        null
+    ) as tipo_retencao_tributos,
     safe_cast({{ clean_cols("ATIVIDAD") }} as string) tipo_atividade_ensino_pesquisa,
-    safe_cast(natureza as string) tipo_natureza_administrativa,
-    safe_cast(nat_jur as string) id_natureza_juridica,
-    safe_cast(clientel as string) tipo_fluxo_atendimento,
+    replace(
+        safe_cast(
+            case
+                when regexp_replace(safe_cast(natureza as string), '^0+', '') = ''
+                then '0'
+                else regexp_replace(safe_cast(natureza as string), '^0+', '')
+            end as string
+        ),
+        'nan',
+        null
+    ) as tipo_natureza_administrativa,
+    nullif(safe_cast(nat_jur as string), '') id_natureza_juridica,
+    replace(
+        safe_cast(
+            case
+                when regexp_replace(safe_cast(clientel as string), '^0+', '') = ''
+                then '0'
+                else regexp_replace(safe_cast(clientel as string), '^0+', '')
+            end as string
+        ),
+        'nan',
+        null
+    ) as tipo_fluxo_atendimento,
     safe_cast({{ clean_cols("TP_UNID") }} as string) tipo_unidade,
-    safe_cast({{ clean_cols("TURNO_AT") }} as string) tipo_turno,
+    replace(safe_cast({{ clean_cols("TURNO_AT") }} as string), 'nan', null) tipo_turno,
     safe_cast({{ clean_cols("NIV_HIER") }} as string) tipo_nivel_hierarquia,
     safe_cast({{ clean_cols("TP_PREST") }} as string) tipo_prestador,
     safe_cast(co_banco as string) banco,
@@ -89,11 +118,23 @@ select
     safe_cast(
         safe.parse_date('%Y%m%d', cast(dt_exped as string)) as date
     ) data_expedicao_alvara,
-    safe_cast({{ clean_cols("ORGEXPED") }} as string) tipo_orgao_expedidor,
-    safe_cast(
-        {{ clean_cols("AV_ACRED") }} as string
+    replace(
+        safe_cast({{ clean_cols("ORGEXPED") }} as string), 'nan', null
+    ) tipo_orgao_expedidor,
+    replace(
+        safe_cast({{ clean_cols("AV_ACRED") }} as string), 'nan', null
     ) tipo_avaliacao_acreditacao_hospitalar,
-    safe_cast(clasaval as string) tipo_classificacao_acreditacao_hospitalar,
+    replace(
+        safe_cast(
+            case
+                when regexp_replace(safe_cast(clasaval as string), '^0+', '') = ''
+                then '0'
+                else regexp_replace(safe_cast(clasaval as string), '^0+', '')
+            end as string
+        ),
+        'nan',
+        null
+    ) as tipo_classificacao_acreditacao_hospitalar,
     safe_cast(substr(cast(dt_acred as string), 1, 4) as int64) as ano_acreditacao,
     safe_cast(substr(cast(dt_acred as string), 5, 2) as int64) as mes_acreditacao,
     safe_cast(

models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql

@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:[email protected]", "group:[email protected]")                     FILTER USING (True)',
-        ],
     )
 }}
 with
@@ -48,7 +43,7 @@ select
     cast(substr(cmpt_ini, 5, 2) as int64) as mes_competencia_inicial,
     cast(substr(cmpt_fim, 1, 4) as int64) as ano_competencia_final,
     cast(substr(cmpt_fim, 5, 2) as int64) as mes_competencia_final,
-    safe_cast(sgruphab as string) tipo_habilitacao,
+    ltrim(safe_cast(sgruphab as string), '0') as tipo_habilitacao,
     safe_cast(portaria as string) portaria,
     cast(
         concat(

models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql

@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:[email protected]", "group:[email protected]")                     FILTER USING (True)',
-        ],
     )
 }}
 with
@@ -49,7 +44,7 @@ select
     cast(substr(cmpt_ini, 5, 2) as int64) as mes_competencia_inicial,
     cast(substr(cmpt_fim, 1, 4) as int64) as ano_competencia_final,
     cast(substr(cmpt_fim, 5, 2) as int64) as mes_competencia_final,
-    safe_cast(sgruphab as string) tipo_habilitacao,
+    ltrim(safe_cast(sgruphab as string), '0') as tipo_habilitacao,
     safe_cast(portaria as string) portaria,
     cast(
         concat(
@@ -61,8 +56,9 @@ select
         ) as date
     ) data_portaria,
     cast(substr(maportar, 1, 4) as int64) as ano_portaria,
-    cast(substr(maportar, 5, 2) as int64) as mes_portaria,
+    cast(substr(maportar, 5, 2) as int64) as mes_portaria
 from cnes_add_muni
+
 {% if is_incremental() %}
     where
 

models/br_ms_cnes/br_ms_cnes__gestao_metas.sql

@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:[email protected]", "group:[email protected]")                     FILTER USING (True)',
-        ],
     )
 }}
 with
@@ -48,7 +43,7 @@ select
     cast(substr(cmpt_ini, 5, 2) as int64) as mes_competencia_inicial,
     cast(substr(cmpt_fim, 1, 4) as int64) as ano_competencia_final,
     cast(substr(cmpt_fim, 5, 2) as int64) as mes_competencia_final,
-    safe_cast(sgruphab as string) tipo_habilitacao,
+    ltrim(safe_cast(sgruphab as string), '0') tipo_habilitacao,
     case
         when safe_cast(sgruphab as string) in ("7003", "7004", "7005", "7006")
         then '1'

models/br_ms_cnes/br_ms_cnes__habilitacao.sql

@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:[email protected]", "group:[email protected]")                     FILTER USING (True)',
-        ],
     )
 }}
 with
@@ -33,7 +28,7 @@ with
         left join
             (
                 select id_municipio, id_municipio_6,
-                from `basedosdados-dev.br_bd_diretorios_brasil.municipio`
+                from `basedosdados.br_bd_diretorios_brasil.municipio`
             ) as mun
             on raw_cnes_habilitacao_without_duplicates.codufmun = mun.id_municipio_6
     )
@@ -49,7 +44,7 @@ select
     safe_cast(substr(cmpt_ini, 5, 2) as int64) as mes_competencia_inicial,
     safe_cast(substr(cmpt_fim, 1, 4) as int64) as ano_competencia_final,
     safe_cast(substr(cmpt_fim, 5, 2) as int64) as mes_competencia_final,
-    safe_cast(sgruphab as string) tipo_habilitacao,
+    ltrim(safe_cast(sgruphab as string), '0') as tipo_habilitacao,
     case
         when
             safe_cast(sgruphab as string) in (

models/br_ms_cnes/br_ms_cnes__incentivos.sql

@@ -8,11 +8,6 @@
             "data_type": "int64",
             "range": {"start": 2005, "end": 2024, "interval": 1},
         },
-        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-        post_hook=[
-            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter                     ON {{this}}                     GRANT TO ("allUsers")                     FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter        ON  {{this}}                     GRANT TO ("group:[email protected]", "group:[email protected]")                     FILTER USING (True)',
-        ],
     )
 }}
 with
@@ -48,7 +43,7 @@ select
     cast(substr(cmpt_ini, 5, 2) as int64) as mes_competencia_inicial,
     cast(substr(cmpt_fim, 1, 4) as int64) as ano_competencia_final,
     cast(substr(cmpt_fim, 5, 2) as int64) as mes_competencia_final,
-    safe_cast(sgruphab as string) tipo_habilitacao,
+    ltrim(safe_cast(sgruphab as string), '0') as tipo_habilitacao,
     case
         when safe_cast(sgruphab as string) in ("8105", "8106", "8107") then '2' else '1'
     end as tipo_incentivo,