Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[dbt] br_ms_cnes #452

Merged
merged 4 commits into from
Feb 26, 2024
Merged

[dbt] br_ms_cnes #452

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b947433
feat: add dbt tests
folhesgabriel Feb 23, 2024
d8256d4
fix: set models to basedosdados-staging
folhesgabriel Feb 26, 2024
73062b9
Merge branch 'main' into staging/cnes
folhesgabriel Feb 26, 2024
22ed54d
fix: make requested changes
folhesgabriel Feb 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 0 additions & 44 deletions models/br_ms_cnes/br_ms_cnes___leito.sql
View file
Open in desktop

This file was deleted.

7 changes: 4 additions & 3 deletions ...nes/br_ms_cnes___dados_complementares.sql → ...cnes/br_ms_cnes__dados_complementares.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (True)',
],
)
}}
Expand All @@ -33,7 +33,7 @@ with
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-staging.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_dados_complementares_without_duplicates.codufmun
= mun.id_municipio_6
Expand Down Expand Up @@ -137,6 +137,7 @@ select
safe_cast(dialise as int64) indicador_existencia_requisito_dialise,
safe_cast(quimradi as int64) indicador_existencia_requisito_quimio_radio
from cnes_add_muni as t

{% if is_incremental() %}
where
date(cast(ano as int64), cast(mes as int64), 1)
Expand Down
17 changes: 11 additions & 6 deletions ...s/br_ms_cnes/br_ms_cnes___equipamento.sql → ...ls/br_ms_cnes/br_ms_cnes__equipamento.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (True)',
],
)
}}
Expand All @@ -24,16 +24,20 @@ with
from `basedosdados-staging.br_ms_cnes_staging.equipamento`
where cnes is not null
),
unique_raw_cnes_equipamento as (
-- 2. distinct nas linhas
select distinct * from raw_cnes_equipamento
),
cnes_add_muni as (
-- 2. Adicionar id_municipio de 7 dígitos
-- 3. Adicionar id_municipio de 7 dígitos
select *
from raw_cnes_equipamento
from unique_raw_cnes_equipamento
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-staging.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_equipamento.codufmun = mun.id_municipio_6
on unique_raw_cnes_equipamento.codufmun = mun.id_municipio_6
)
select
safe_cast(ano as int64) as ano,
Expand All @@ -48,6 +52,7 @@ select
safe_cast(ind_sus as int64) as indicador_equipamento_disponivel_sus,
safe_cast(ind_nsus as int64) as indicador_equipamento_indisponivel_sus
from cnes_add_muni

{% if is_incremental() %}
where
date(cast(ano as int64), cast(mes as int64), 1)
Expand Down
19 changes: 10 additions & 9 deletions models/br_ms_cnes/br_ms_cnes___equipe.sql → models/br_ms_cnes/br_ms_cnes__equipe.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (True)',
],
)
}}
Expand All @@ -30,7 +30,7 @@ with
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-staging.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_equipe.codufmun = mun.id_municipio_6
)
Expand Down Expand Up @@ -59,14 +59,15 @@ select
safe_cast(tp_desat as string) as tipo_desativacao_equipe,
safe_cast(substr(dt_desat, 1, 4) as int64) as ano_desativacao_equipe,
safe_cast(substr(dt_desat, 5, 6) as int64) as mes_desativacao_equipe,
safe_cast(quilombo as string) as indicador_atende_populacao_assistida_quilombolas,
safe_cast(assentad as string) as indicador_atende_populacao_assistida_assentados,
safe_cast(popgeral as string) as indicador_atende_populacao_assistida_geral,
safe_cast(escola as string) as indicador_atende_populacao_assistida_escolares,
safe_cast(indigena as string) as indicador_atende_populacao_assistida_indigena,
safe_cast(pronasci as string) as indicador_atende_populacao_assistida_pronasci,
safe_cast(quilombo as int64) as indicador_atende_populacao_assistida_quilombolas,
safe_cast(assentad as int64) as indicador_atende_populacao_assistida_assentados,
safe_cast(popgeral as int64) as indicador_atende_populacao_assistida_geral,
safe_cast(escola as int64) as indicador_atende_populacao_assistida_escolares,
safe_cast(indigena as int64) as indicador_atende_populacao_assistida_indigena,
safe_cast(pronasci as int64) as indicador_atende_populacao_assistida_pronasci,
from cnes_add_muni
{% if is_incremental() %}

where
date(cast(ano as int64), cast(mes as int64), 1)
> (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }})
Expand Down
10 changes: 5 additions & 5 deletions ..._ms_cnes/br_ms_cnes___estabelecimento.sql → ...r_ms_cnes/br_ms_cnes__estabelecimento.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (True)',
],
)
}}
Expand All @@ -33,7 +33,7 @@ with
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-staging.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_estabelecimento_without_duplicates.codufmun = mun.id_municipio_6
)
Expand Down Expand Up @@ -239,7 +239,7 @@ select
safe_cast(ap03cv04 as int64) indicador_atendimento_sadt_plano_seguro_terceiro,
safe_cast(ap03cv05 as int64) indicador_atendimento_sadt_plano_saude_publico,
safe_cast(ap03cv06 as int64) indicador_atendimento_sadt_plano_saude_privado,
safe_cast(ap04cv01 as string) indicador_atendimento_urgencia_sus,
safe_cast(ap04cv01 as int64) indicador_atendimento_urgencia_sus,
safe_cast(ap04cv02 as int64) indicador_atendimento_urgencia_privado,
safe_cast(ap04cv03 as int64) indicador_atendimento_urgencia_plano_seguro_proprio,
safe_cast(ap04cv04 as int64) indicador_atendimento_urgencia_plano_seguro_terceiro,
Expand All @@ -263,9 +263,9 @@ select
safe_cast(ap07cv04 as int64) indicador_atendimento_regulacao_plano_seguro_terceiro,
safe_cast(ap07cv05 as int64) indicador_atendimento_regulacao_plano_saude_publico,
safe_cast(ap07cv06 as int64) indicador_atendimento_regulacao_plano_saude_privado
from cnes_add_muni as t
{% if is_incremental() %}
where

date(cast(ano as int64), cast(mes as int64), 1)
> (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }})
{% endif %}
9 changes: 5 additions & 4 deletions ...s/br_ms_cnes___estabelecimento_ensino.sql → ...es/br_ms_cnes__estabelecimento_ensino.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@
partition_by={
"field": "ano",
"data_type": "int64",
"range": {"start": 2005, "end": 2023, "interval": 1},
"range": {"start": 2005, "end": 2024, "interval": 1},
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (True)',
],
)
}}
Expand All @@ -33,7 +33,7 @@ with
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-staging.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_estabelecimento_ensino_without_duplicates.codufmun
= mun.id_municipio_6
Expand Down Expand Up @@ -64,6 +64,7 @@ select
from cnes_add_muni as t
{% if is_incremental() %}
where

date(cast(ano as int64), cast(mes as int64), 1)
> (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }})
{% endif %}
8 changes: 4 additions & 4 deletions ...s_cnes___estabelecimento_filantropico.sql → ...ms_cnes__estabelecimento_filantropico.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (True)',
],
)
}}
Expand All @@ -33,7 +33,7 @@ with
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-staging.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_estabelecimento_filantropico_without_duplicates.codufmun
= mun.id_municipio_6
Expand Down Expand Up @@ -62,9 +62,9 @@ select
) data_portaria,
cast(substr(maportar, 1, 4) as int64) as ano_portaria,
cast(substr(maportar, 5, 2) as int64) as mes_portaria,
from cnes_add_muni as t
{% if is_incremental() %}
where

date(cast(ano as int64), cast(mes as int64), 1)
> (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }})
{% endif %}
7 changes: 4 additions & 3 deletions .../br_ms_cnes/br_ms_cnes___gestao_metas.sql → ...s/br_ms_cnes/br_ms_cnes__gestao_metas.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (True)',
],
)
}}
Expand All @@ -33,7 +33,7 @@ with
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-staging.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_gestao_metas_without_duplicates.codufmun = mun.id_municipio_6
)
Expand Down Expand Up @@ -69,6 +69,7 @@ select
from cnes_add_muni as t
{% if is_incremental() %}
where

date(cast(ano as int64), cast(mes as int64), 1)
> (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }})
{% endif %}
21 changes: 11 additions & 10 deletions ...s/br_ms_cnes/br_ms_cnes___habilitacao.sql → ...ls/br_ms_cnes/br_ms_cnes__habilitacao.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,16 @@
},
pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
post_hook=[
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)',
'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (True)',
],
)
}}
with
raw_cnes_habilitacaol as (
-- 1. Retirar linhas com id_estabelecimento_cnes nulo
select *
from `basedosdados-staging.br_ms_cnes_staging.habilitacao`
from `basedosdados-dev.br_ms_cnes_staging.habilitacao`
where cnes is not null
),
raw_cnes_habilitacao_without_duplicates as (
Expand All @@ -33,7 +33,7 @@ with
left join
(
select id_municipio, id_municipio_6,
from `basedosdados.br_bd_diretorios_brasil.municipio`
from `basedosdados-dev.br_bd_diretorios_brasil.municipio`
) as mun
on raw_cnes_habilitacao_without_duplicates.codufmun = mun.id_municipio_6
)
Expand All @@ -45,10 +45,10 @@ select
safe_cast(id_municipio as string) id_municipio,
safe_cast(cnes as string) id_estabelecimento_cnes,
safe_cast(nuleitos as int64) quantidade_leitos,
cast(substr(cmpt_ini, 1, 4) as int64) as ano_competencia_inicial,
cast(substr(cmpt_ini, 5, 2) as int64) as mes_competencia_inicial,
cast(substr(cmpt_fim, 1, 4) as int64) as ano_competencia_final,
cast(substr(cmpt_fim, 5, 2) as int64) as mes_competencia_final,
safe_cast(substr(cmpt_ini, 1, 4) as int64) as ano_competencia_inicial,
safe_cast(substr(cmpt_ini, 5, 2) as int64) as mes_competencia_inicial,
safe_cast(substr(cmpt_fim, 1, 4) as int64) as ano_competencia_final,
safe_cast(substr(cmpt_fim, 5, 2) as int64) as mes_competencia_final,
safe_cast(sgruphab as string) tipo_habilitacao,
case
when
Expand Down Expand Up @@ -78,8 +78,9 @@ select
substring(dtportar, 1, 2)
) as date
) data_portaria,
cast(substr(maportar, 1, 4) as int64) as ano_portaria,
cast(substr(maportar, 5, 2) as int64) as mes_portaria,

safe_cast(substr(maportar, 1, 4) as int64) as ano_portaria,
safe_cast(substr(maportar, 5, 2) as int64) as mes_portaria,
from cnes_add_muni as t
{% if is_incremental() %}
where
Expand Down
Loading
Loading