Skip to content

Commit

Permalink
feat: add dbt tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@folhesgabriel
folhesgabriel committed Feb 23, 2024
1 parent 4a1d816 commit b947433
Show file tree
Hide file tree
Showing 15 changed files with 370 additions and 114 deletions.
44 changes: 0 additions & 44 deletions models/br_ms_cnes/br_ms_cnes___leito.sql
View file

This file was deleted.

9 changes: 5 additions & 4 deletions ...nes/br_ms_cnes___dados_complementares.sql → ...cnes/br_ms_cnes__dados_complementares.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,16 @@
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
],
)
}}
with
raw_cnes_dados_complementares as (
-- 1. Retirar linhas com id_estabelecimento_cnes nulo
select *
from `basedosdados-staging.br_ms_cnes_staging.dados_complementares`
from `basedosdados-dev.br_ms_cnes_staging.dados_complementares`
where cnes is not null
),
raw_cnes_dados_complementares_without_duplicates as (
Expand All @@ -33,7 +33,7 @@ with
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-dev.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_dados_complementares_without_duplicates.codufmun
= mun.id_municipio_6
Expand Down Expand Up @@ -137,6 +137,7 @@ select
safe_cast(dialise as int64) indicador_existencia_requisito_dialise,
safe_cast(quimradi as int64) indicador_existencia_requisito_quimio_radio
from cnes_add_muni as t

{% if is_incremental() %}
where
date(cast(ano as int64), cast(mes as int64), 1)
Expand Down
19 changes: 12 additions & 7 deletions ...s/br_ms_cnes/br_ms_cnes___equipamento.sql → ...ls/br_ms_cnes/br_ms_cnes__equipamento.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
],
)
}}
Expand All @@ -21,19 +21,23 @@ with
raw_cnes_equipamento as (
-- 1. Retirar linhas com id_estabelecimento_cnes nulo
select *
from `basedosdados-staging.br_ms_cnes_staging.equipamento`
from `basedosdados-dev.br_ms_cnes_staging.equipamento`
where cnes is not null
),
unique_raw_cnes_equipamento as (
-- 2. distinct nas linhas
select distinct * from raw_cnes_equipamento
),
cnes_add_muni as (
-- 2. Adicionar id_municipio de 7 dígitos
-- 3. Adicionar id_municipio de 7 dígitos
select *
from raw_cnes_equipamento
from unique_raw_cnes_equipamento
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-dev.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_equipamento.codufmun = mun.id_municipio_6
on unique_raw_cnes_equipamento.codufmun = mun.id_municipio_6
)
select
safe_cast(ano as int64) as ano,
Expand All @@ -48,6 +52,7 @@ select
safe_cast(ind_sus as int64) as indicador_equipamento_disponivel_sus,
safe_cast(ind_nsus as int64) as indicador_equipamento_indisponivel_sus
from cnes_add_muni

{% if is_incremental() %}
where
date(cast(ano as int64), cast(mes as int64), 1)
Expand Down
21 changes: 11 additions & 10 deletions models/br_ms_cnes/br_ms_cnes___equipe.sql → models/br_ms_cnes/br_ms_cnes__equipe.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
],
)
}}
Expand All @@ -20,7 +20,7 @@ with
raw_cnes_equipe as (
-- 1. Retirar linhas com id_estabelecimento_cnes nulo
select *
from `basedosdados-staging.br_ms_cnes_staging.equipe`
from `basedosdados-dev.br_ms_cnes_staging.equipe`
where cnes is not null
),
cnes_add_muni as (
Expand All @@ -30,7 +30,7 @@ with
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-dev.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_equipe.codufmun = mun.id_municipio_6
)
Expand Down Expand Up @@ -59,14 +59,15 @@ select
safe_cast(tp_desat as string) as tipo_desativacao_equipe,
safe_cast(substr(dt_desat, 1, 4) as int64) as ano_desativacao_equipe,
safe_cast(substr(dt_desat, 5, 6) as int64) as mes_desativacao_equipe,
safe_cast(quilombo as string) as indicador_atende_populacao_assistida_quilombolas,
safe_cast(assentad as string) as indicador_atende_populacao_assistida_assentados,
safe_cast(popgeral as string) as indicador_atende_populacao_assistida_geral,
safe_cast(escola as string) as indicador_atende_populacao_assistida_escolares,
safe_cast(indigena as string) as indicador_atende_populacao_assistida_indigena,
safe_cast(pronasci as string) as indicador_atende_populacao_assistida_pronasci,
safe_cast(quilombo as int64) as indicador_atende_populacao_assistida_quilombolas,
safe_cast(assentad as int64) as indicador_atende_populacao_assistida_assentados,
safe_cast(popgeral as int64) as indicador_atende_populacao_assistida_geral,
safe_cast(escola as int64) as indicador_atende_populacao_assistida_escolares,
safe_cast(indigena as int64) as indicador_atende_populacao_assistida_indigena,
safe_cast(pronasci as int64) as indicador_atende_populacao_assistida_pronasci,
from cnes_add_muni
{% if is_incremental() %}

where
date(cast(ano as int64), cast(mes as int64), 1)
> (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }})
Expand Down
8 changes: 4 additions & 4 deletions ..._ms_cnes/br_ms_cnes___estabelecimento.sql → ...r_ms_cnes/br_ms_cnes__estabelecimento.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
],
)
}}
Expand All @@ -33,7 +33,7 @@ with
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-dev.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_estabelecimento_without_duplicates.codufmun = mun.id_municipio_6
)
Expand Down Expand Up @@ -263,9 +263,9 @@ select
safe_cast(ap07cv04 as int64) indicador_atendimento_regulacao_plano_seguro_terceiro,
safe_cast(ap07cv05 as int64) indicador_atendimento_regulacao_plano_saude_publico,
safe_cast(ap07cv06 as int64) indicador_atendimento_regulacao_plano_saude_privado
from cnes_add_muni as t
{% if is_incremental() %}
where

date(cast(ano as int64), cast(mes as int64), 1)
> (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }})
{% endif %}
11 changes: 6 additions & 5 deletions ...s/br_ms_cnes___estabelecimento_ensino.sql → ...es/br_ms_cnes__estabelecimento_ensino.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,20 @@
partition_by={
"field": "ano",
"data_type": "int64",
"range": {"start": 2005, "end": 2023, "interval": 1},
"range": {"start": 2005, "end": 2024, "interval": 1},
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
],
)
}}
with
raw_cnes_estabelecimento_ensino as (
-- 1. Retirar linhas com id_estabelecimento_cnes nulo
select *
from `basedosdados-staging.br_ms_cnes_staging.estabelecimento_ensino`
from `basedosdados-dev.br_ms_cnes_staging.estabelecimento_ensino`
where cnes is not null
),
raw_cnes_estabelecimento_ensino_without_duplicates as (
Expand All @@ -33,7 +33,7 @@ with
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-dev.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_estabelecimento_ensino_without_duplicates.codufmun
= mun.id_municipio_6
Expand Down Expand Up @@ -64,6 +64,7 @@ select
from cnes_add_muni as t
{% if is_incremental() %}
where

date(cast(ano as int64), cast(mes as int64), 1)
> (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }})
{% endif %}
10 changes: 5 additions & 5 deletions ...s_cnes___estabelecimento_filantropico.sql → ...ms_cnes__estabelecimento_filantropico.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,16 @@
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
],
)
}}
with
raw_cnes_estabelecimento_filantropico as (
-- 1. Retirar linhas com id_estabelecimento_cnes nulo
select *
from `basedosdados-staging.br_ms_cnes_staging.estabelecimento_filantropico`
from `basedosdados-dev.br_ms_cnes_staging.estabelecimento_filantropico`
where cnes is not null
),
raw_cnes_estabelecimento_filantropico_without_duplicates as (
Expand All @@ -33,7 +33,7 @@ with
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-dev.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_estabelecimento_filantropico_without_duplicates.codufmun
= mun.id_municipio_6
Expand Down Expand Up @@ -62,9 +62,9 @@ select
) data_portaria,
cast(substr(maportar, 1, 4) as int64) as ano_portaria,
cast(substr(maportar, 5, 2) as int64) as mes_portaria,
from cnes_add_muni as t
{% if is_incremental() %}
where

date(cast(ano as int64), cast(mes as int64), 1)
> (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }})
{% endif %}
9 changes: 5 additions & 4 deletions .../br_ms_cnes/br_ms_cnes___gestao_metas.sql → ...s/br_ms_cnes/br_ms_cnes__gestao_metas.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,16 @@
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
],
)
}}
with
raw_cnes_gestao_metas as (
-- 1. Retirar linhas com id_estabelecimento_cnes nulo
select *
from `basedosdados-staging.br_ms_cnes_staging.gestao_metas`
from `basedosdados-dev.br_ms_cnes_staging.gestao_metas`
where cnes is not null
),
raw_cnes_gestao_metas_without_duplicates as (
Expand All @@ -33,7 +33,7 @@ with
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-dev.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_gestao_metas_without_duplicates.codufmun = mun.id_municipio_6
)
Expand Down Expand Up @@ -69,6 +69,7 @@ select
from cnes_add_muni as t
{% if is_incremental() %}
where

date(cast(ano as int64), cast(mes as int64), 1)
> (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }})
{% endif %}
Loading

0 comments on commit b947433

Please sign in to comment.