From b94743383e9c56696cf3317e4ea2ace409a266d8 Mon Sep 17 00:00:00 2001 From: folhesgabriel Date: Fri, 23 Feb 2024 13:09:10 -0300 Subject: [PATCH 1/3] feat: add dbt tests --- models/br_ms_cnes/br_ms_cnes___leito.sql | 44 ---- ...l => br_ms_cnes__dados_complementares.sql} | 9 +- ...amento.sql => br_ms_cnes__equipamento.sql} | 19 +- ...es___equipe.sql => br_ms_cnes__equipe.sql} | 21 +- ...to.sql => br_ms_cnes__estabelecimento.sql} | 8 +- ...=> br_ms_cnes__estabelecimento_ensino.sql} | 11 +- ...ms_cnes__estabelecimento_filantropico.sql} | 10 +- ...metas.sql => br_ms_cnes__gestao_metas.sql} | 9 +- ...itacao.sql => br_ms_cnes__habilitacao.sql} | 22 +- ...entivos.sql => br_ms_cnes__incentivos.sql} | 9 +- models/br_ms_cnes/br_ms_cnes__leito.sql | 64 ++++++ ...ional.sql => br_ms_cnes__profissional.sql} | 27 ++- ...l.sql => br_ms_cnes__regra_contratual.sql} | 9 +- ... => br_ms_cnes__servico_especializado.sql} | 9 +- models/br_ms_cnes/schema.yml | 213 +++++++++++++++++- 15 files changed, 370 insertions(+), 114 deletions(-) delete mode 100644 models/br_ms_cnes/br_ms_cnes___leito.sql rename models/br_ms_cnes/{br_ms_cnes___dados_complementares.sql => br_ms_cnes__dados_complementares.sql} (92%) rename models/br_ms_cnes/{br_ms_cnes___equipamento.sql => br_ms_cnes__equipamento.sql} (63%) rename models/br_ms_cnes/{br_ms_cnes___equipe.sql => br_ms_cnes__equipe.sql} (68%) rename models/br_ms_cnes/{br_ms_cnes___estabelecimento.sql => br_ms_cnes__estabelecimento.sql} (96%) rename models/br_ms_cnes/{br_ms_cnes___estabelecimento_ensino.sql => br_ms_cnes__estabelecimento_ensino.sql} (75%) rename models/br_ms_cnes/{br_ms_cnes___estabelecimento_filantropico.sql => br_ms_cnes__estabelecimento_filantropico.sql} (78%) rename models/br_ms_cnes/{br_ms_cnes___gestao_metas.sql => br_ms_cnes__gestao_metas.sql} (78%) rename models/br_ms_cnes/{br_ms_cnes___habilitacao.sql => br_ms_cnes__habilitacao.sql} (66%) rename models/br_ms_cnes/{br_ms_cnes___incentivos.sql => br_ms_cnes__incentivos.sql} (78%) create mode 100644 models/br_ms_cnes/br_ms_cnes__leito.sql rename models/br_ms_cnes/{br_ms_cnes___profissional.sql => br_ms_cnes__profissional.sql} (65%) rename models/br_ms_cnes/{br_ms_cnes___regra_contratual.sql => br_ms_cnes__regra_contratual.sql} (79%) rename models/br_ms_cnes/{br_ms_cnes___servico_especializado.sql => br_ms_cnes__servico_especializado.sql} (78%) diff --git a/models/br_ms_cnes/br_ms_cnes___leito.sql b/models/br_ms_cnes/br_ms_cnes___leito.sql deleted file mode 100644 index 42f13605..00000000 --- a/models/br_ms_cnes/br_ms_cnes___leito.sql +++ /dev/null @@ -1,44 +0,0 @@ -{{ - config( - schema="br_ms_cnes", - alias="leito", - materialized="incremental", - partition_by={ - "field": "ano", - "data_type": "int64", - "range": {"start": 2007, "end": 2024, "interval": 1}, - }, - pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", - post_hook=[ - 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', - ], - ) -}} - - -with - raw_cnes_leito as ( - -- 1. Retirar linhas com id_estabelecimento_cnes nulo - select * - from `basedosdados-staging.br_ms_cnes_staging.leito` - where cnes is not null - ), - cnes_leito_without_duplicates as (select distinct * from raw_cnes_leito) - -select - safe_cast(ano as int64) as ano, - safe_cast(mes as int64) as mes, - safe_cast(sigla_uf as string) as sigla_uf, - safe_cast(cnes as string) as id_estabelecimento_cnes, - safe_cast(codleito as string) as tipo_especialidade_leito, - safe_cast(tp_leito as string) as tipo_leito, - safe_cast(qt_exist as string) as quantidade_total, - safe_cast(qt_contr as string) as quantidade_contratado, - safe_cast(qt_sus as string) as quantidade_sus -from cnes_leito_without_duplicates -{% if is_incremental() %} - where - date(cast(ano as int64), cast(mes as int64), 1) - > (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) -{% endif %} diff --git a/models/br_ms_cnes/br_ms_cnes___dados_complementares.sql b/models/br_ms_cnes/br_ms_cnes__dados_complementares.sql similarity index 92% rename from models/br_ms_cnes/br_ms_cnes___dados_complementares.sql rename to models/br_ms_cnes/br_ms_cnes__dados_complementares.sql index 8152e79b..13ed89dc 100644 --- a/models/br_ms_cnes/br_ms_cnes___dados_complementares.sql +++ b/models/br_ms_cnes/br_ms_cnes__dados_complementares.sql @@ -10,8 +10,8 @@ }, pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ - 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', ], ) }} @@ -19,7 +19,7 @@ with raw_cnes_dados_complementares as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-staging.br_ms_cnes_staging.dados_complementares` + from `basedosdados-dev.br_ms_cnes_staging.dados_complementares` where cnes is not null ), raw_cnes_dados_complementares_without_duplicates as ( @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados.br_bd_diretorios_brasil.municipio` + from `basedosdados-dev.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_dados_complementares_without_duplicates.codufmun = mun.id_municipio_6 @@ -137,6 +137,7 @@ select safe_cast(dialise as int64) indicador_existencia_requisito_dialise, safe_cast(quimradi as int64) indicador_existencia_requisito_quimio_radio from cnes_add_muni as t + {% if is_incremental() %} where date(cast(ano as int64), cast(mes as int64), 1) diff --git a/models/br_ms_cnes/br_ms_cnes___equipamento.sql b/models/br_ms_cnes/br_ms_cnes__equipamento.sql similarity index 63% rename from models/br_ms_cnes/br_ms_cnes___equipamento.sql rename to models/br_ms_cnes/br_ms_cnes__equipamento.sql index 4d98bb3e..3162bd7f 100644 --- a/models/br_ms_cnes/br_ms_cnes___equipamento.sql +++ b/models/br_ms_cnes/br_ms_cnes__equipamento.sql @@ -10,8 +10,8 @@ }, pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ - 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', ], ) }} @@ -21,19 +21,23 @@ with raw_cnes_equipamento as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-staging.br_ms_cnes_staging.equipamento` + from `basedosdados-dev.br_ms_cnes_staging.equipamento` where cnes is not null ), + unique_raw_cnes_equipamento as ( + -- 2. distinct nas linhas + select distinct * from raw_cnes_equipamento + ), cnes_add_muni as ( - -- 2. Adicionar id_municipio de 7 dígitos + -- 3. Adicionar id_municipio de 7 dígitos select * - from raw_cnes_equipamento + from unique_raw_cnes_equipamento left join ( select id_municipio, id_municipio_6, - from `basedosdados.br_bd_diretorios_brasil.municipio` + from `basedosdados-dev.br_bd_diretorios_brasil.municipio` ) as mun - on raw_cnes_equipamento.codufmun = mun.id_municipio_6 + on unique_raw_cnes_equipamento.codufmun = mun.id_municipio_6 ) select safe_cast(ano as int64) as ano, @@ -48,6 +52,7 @@ select safe_cast(ind_sus as int64) as indicador_equipamento_disponivel_sus, safe_cast(ind_nsus as int64) as indicador_equipamento_indisponivel_sus from cnes_add_muni + {% if is_incremental() %} where date(cast(ano as int64), cast(mes as int64), 1) diff --git a/models/br_ms_cnes/br_ms_cnes___equipe.sql b/models/br_ms_cnes/br_ms_cnes__equipe.sql similarity index 68% rename from models/br_ms_cnes/br_ms_cnes___equipe.sql rename to models/br_ms_cnes/br_ms_cnes__equipe.sql index fca572c9..b06f3759 100644 --- a/models/br_ms_cnes/br_ms_cnes___equipe.sql +++ b/models/br_ms_cnes/br_ms_cnes__equipe.sql @@ -10,8 +10,8 @@ }, pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ - 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', ], ) }} @@ -20,7 +20,7 @@ with raw_cnes_equipe as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-staging.br_ms_cnes_staging.equipe` + from `basedosdados-dev.br_ms_cnes_staging.equipe` where cnes is not null ), cnes_add_muni as ( @@ -30,7 +30,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados.br_bd_diretorios_brasil.municipio` + from `basedosdados-dev.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_equipe.codufmun = mun.id_municipio_6 ) @@ -59,14 +59,15 @@ select safe_cast(tp_desat as string) as tipo_desativacao_equipe, safe_cast(substr(dt_desat, 1, 4) as int64) as ano_desativacao_equipe, safe_cast(substr(dt_desat, 5, 6) as int64) as mes_desativacao_equipe, - safe_cast(quilombo as string) as indicador_atende_populacao_assistida_quilombolas, - safe_cast(assentad as string) as indicador_atende_populacao_assistida_assentados, - safe_cast(popgeral as string) as indicador_atende_populacao_assistida_geral, - safe_cast(escola as string) as indicador_atende_populacao_assistida_escolares, - safe_cast(indigena as string) as indicador_atende_populacao_assistida_indigena, - safe_cast(pronasci as string) as indicador_atende_populacao_assistida_pronasci, + safe_cast(quilombo as int64) as indicador_atende_populacao_assistida_quilombolas, + safe_cast(assentad as int64) as indicador_atende_populacao_assistida_assentados, + safe_cast(popgeral as int64) as indicador_atende_populacao_assistida_geral, + safe_cast(escola as int64) as indicador_atende_populacao_assistida_escolares, + safe_cast(indigena as int64) as indicador_atende_populacao_assistida_indigena, + safe_cast(pronasci as int64) as indicador_atende_populacao_assistida_pronasci, from cnes_add_muni {% if is_incremental() %} + where date(cast(ano as int64), cast(mes as int64), 1) > (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) diff --git a/models/br_ms_cnes/br_ms_cnes___estabelecimento.sql b/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql similarity index 96% rename from models/br_ms_cnes/br_ms_cnes___estabelecimento.sql rename to models/br_ms_cnes/br_ms_cnes__estabelecimento.sql index 5c21e242..c1bf0825 100644 --- a/models/br_ms_cnes/br_ms_cnes___estabelecimento.sql +++ b/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql @@ -10,8 +10,8 @@ }, pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ - 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', ], ) }} @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados.br_bd_diretorios_brasil.municipio` + from `basedosdados-dev.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_estabelecimento_without_duplicates.codufmun = mun.id_municipio_6 ) @@ -263,9 +263,9 @@ select safe_cast(ap07cv04 as int64) indicador_atendimento_regulacao_plano_seguro_terceiro, safe_cast(ap07cv05 as int64) indicador_atendimento_regulacao_plano_saude_publico, safe_cast(ap07cv06 as int64) indicador_atendimento_regulacao_plano_saude_privado -from cnes_add_muni as t {% if is_incremental() %} where + date(cast(ano as int64), cast(mes as int64), 1) > (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) {% endif %} diff --git a/models/br_ms_cnes/br_ms_cnes___estabelecimento_ensino.sql b/models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql similarity index 75% rename from models/br_ms_cnes/br_ms_cnes___estabelecimento_ensino.sql rename to models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql index bd7dcd6d..7c2b7132 100644 --- a/models/br_ms_cnes/br_ms_cnes___estabelecimento_ensino.sql +++ b/models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql @@ -6,12 +6,12 @@ partition_by={ "field": "ano", "data_type": "int64", - "range": {"start": 2005, "end": 2023, "interval": 1}, + "range": {"start": 2005, "end": 2024, "interval": 1}, }, pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ - 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', ], ) }} @@ -19,7 +19,7 @@ with raw_cnes_estabelecimento_ensino as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-staging.br_ms_cnes_staging.estabelecimento_ensino` + from `basedosdados-dev.br_ms_cnes_staging.estabelecimento_ensino` where cnes is not null ), raw_cnes_estabelecimento_ensino_without_duplicates as ( @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados.br_bd_diretorios_brasil.municipio` + from `basedosdados-dev.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_estabelecimento_ensino_without_duplicates.codufmun = mun.id_municipio_6 @@ -64,6 +64,7 @@ select from cnes_add_muni as t {% if is_incremental() %} where + date(cast(ano as int64), cast(mes as int64), 1) > (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) {% endif %} diff --git a/models/br_ms_cnes/br_ms_cnes___estabelecimento_filantropico.sql b/models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql similarity index 78% rename from models/br_ms_cnes/br_ms_cnes___estabelecimento_filantropico.sql rename to models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql index 0479e83c..32b6804c 100644 --- a/models/br_ms_cnes/br_ms_cnes___estabelecimento_filantropico.sql +++ b/models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql @@ -10,8 +10,8 @@ }, pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ - 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', ], ) }} @@ -19,7 +19,7 @@ with raw_cnes_estabelecimento_filantropico as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-staging.br_ms_cnes_staging.estabelecimento_filantropico` + from `basedosdados-dev.br_ms_cnes_staging.estabelecimento_filantropico` where cnes is not null ), raw_cnes_estabelecimento_filantropico_without_duplicates as ( @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados.br_bd_diretorios_brasil.municipio` + from `basedosdados-dev.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_estabelecimento_filantropico_without_duplicates.codufmun = mun.id_municipio_6 @@ -62,9 +62,9 @@ select ) data_portaria, cast(substr(maportar, 1, 4) as int64) as ano_portaria, cast(substr(maportar, 5, 2) as int64) as mes_portaria, -from cnes_add_muni as t {% if is_incremental() %} where + date(cast(ano as int64), cast(mes as int64), 1) > (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) {% endif %} diff --git a/models/br_ms_cnes/br_ms_cnes___gestao_metas.sql b/models/br_ms_cnes/br_ms_cnes__gestao_metas.sql similarity index 78% rename from models/br_ms_cnes/br_ms_cnes___gestao_metas.sql rename to models/br_ms_cnes/br_ms_cnes__gestao_metas.sql index c6649484..53179720 100644 --- a/models/br_ms_cnes/br_ms_cnes___gestao_metas.sql +++ b/models/br_ms_cnes/br_ms_cnes__gestao_metas.sql @@ -10,8 +10,8 @@ }, pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ - 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', ], ) }} @@ -19,7 +19,7 @@ with raw_cnes_gestao_metas as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-staging.br_ms_cnes_staging.gestao_metas` + from `basedosdados-dev.br_ms_cnes_staging.gestao_metas` where cnes is not null ), raw_cnes_gestao_metas_without_duplicates as ( @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados.br_bd_diretorios_brasil.municipio` + from `basedosdados-dev.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_gestao_metas_without_duplicates.codufmun = mun.id_municipio_6 ) @@ -69,6 +69,7 @@ select from cnes_add_muni as t {% if is_incremental() %} where + date(cast(ano as int64), cast(mes as int64), 1) > (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) {% endif %} diff --git a/models/br_ms_cnes/br_ms_cnes___habilitacao.sql b/models/br_ms_cnes/br_ms_cnes__habilitacao.sql similarity index 66% rename from models/br_ms_cnes/br_ms_cnes___habilitacao.sql rename to models/br_ms_cnes/br_ms_cnes__habilitacao.sql index 018ecd3d..ed5fc2c3 100644 --- a/models/br_ms_cnes/br_ms_cnes___habilitacao.sql +++ b/models/br_ms_cnes/br_ms_cnes__habilitacao.sql @@ -8,18 +8,13 @@ "data_type": "int64", "range": {"start": 2005, "end": 2024, "interval": 1}, }, - pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", - post_hook=[ - 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', - ], ) }} with raw_cnes_habilitacaol as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-staging.br_ms_cnes_staging.habilitacao` + from `basedosdados-dev.br_ms_cnes_staging.habilitacao` where cnes is not null ), raw_cnes_habilitacao_without_duplicates as ( @@ -33,7 +28,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados.br_bd_diretorios_brasil.municipio` + from `basedosdados-dev.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_habilitacao_without_duplicates.codufmun = mun.id_municipio_6 ) @@ -45,10 +40,10 @@ select safe_cast(id_municipio as string) id_municipio, safe_cast(cnes as string) id_estabelecimento_cnes, safe_cast(nuleitos as int64) quantidade_leitos, - cast(substr(cmpt_ini, 1, 4) as int64) as ano_competencia_inicial, - cast(substr(cmpt_ini, 5, 2) as int64) as mes_competencia_inicial, - cast(substr(cmpt_fim, 1, 4) as int64) as ano_competencia_final, - cast(substr(cmpt_fim, 5, 2) as int64) as mes_competencia_final, + safe_cast(substr(cmpt_ini, 1, 4) as int64) as ano_competencia_inicial, + safe_cast(substr(cmpt_ini, 5, 2) as int64) as mes_competencia_inicial, + safe_cast(substr(cmpt_fim, 1, 4) as int64) as ano_competencia_final, + safe_cast(substr(cmpt_fim, 5, 2) as int64) as mes_competencia_final, safe_cast(sgruphab as string) tipo_habilitacao, case when @@ -78,8 +73,9 @@ select substring(dtportar, 1, 2) ) as date ) data_portaria, - cast(substr(maportar, 1, 4) as int64) as ano_portaria, - cast(substr(maportar, 5, 2) as int64) as mes_portaria, + + safe_cast(substr(maportar, 1, 4) as int64) as ano_portaria, + safe_cast(substr(maportar, 5, 2) as int64) as mes_portaria, from cnes_add_muni as t {% if is_incremental() %} where diff --git a/models/br_ms_cnes/br_ms_cnes___incentivos.sql b/models/br_ms_cnes/br_ms_cnes__incentivos.sql similarity index 78% rename from models/br_ms_cnes/br_ms_cnes___incentivos.sql rename to models/br_ms_cnes/br_ms_cnes__incentivos.sql index e893ba73..6683b499 100644 --- a/models/br_ms_cnes/br_ms_cnes___incentivos.sql +++ b/models/br_ms_cnes/br_ms_cnes__incentivos.sql @@ -10,8 +10,8 @@ }, pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ - 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', ], ) }} @@ -19,7 +19,7 @@ with raw_cnes_incentivos as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-staging.br_ms_cnes_staging.incentivos` + from `basedosdados-dev.br_ms_cnes_staging.incentivos` where cnes is not null ), raw_cnes_incentivos_without_duplicates as ( @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados.br_bd_diretorios_brasil.municipio` + from `basedosdados-dev.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_incentivos_without_duplicates.codufmun = mun.id_municipio_6 ) @@ -67,6 +67,7 @@ select from cnes_add_muni as t {% if is_incremental() %} where + date(cast(ano as int64), cast(mes as int64), 1) > (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) {% endif %} diff --git a/models/br_ms_cnes/br_ms_cnes__leito.sql b/models/br_ms_cnes/br_ms_cnes__leito.sql new file mode 100644 index 00000000..265a15a5 --- /dev/null +++ b/models/br_ms_cnes/br_ms_cnes__leito.sql @@ -0,0 +1,64 @@ +{{ + config( + schema="br_ms_cnes", + alias="leito", + materialized="incremental", + partition_by={ + "field": "ano", + "data_type": "int64", + "range": {"start": 2007, "end": 2024, "interval": 1}, + }, + pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=[ + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + ], + ) +}} + + +with + raw_cnes_leito as ( + -- 1. Retirar linhas com id_estabelecimento_cnes nulo + select * from `basedosdados-dev.br_ms_cnes_staging.leito` where cnes is not null + ), + cnes_leito_without_duplicates as (select distinct * from raw_cnes_leito), + leito_x_estabelecimento as ( + -- 3. Adicionar id_municipio de 7 dígitos fazendo join com a tabela + -- estabalecimento + -- ps: a coluna id_municipio não vem por padrão na tabela leito extraída do + -- FTP do Datasus + select * + from cnes_leito_without_duplicates as lt + left join + ( + select + id_municipio, + cast(ano as string) ano1, + cast(mes as string) mes1, + id_estabelecimento_cnes as iddd + from `basedosdados.br_ms_cnes.estabelecimento` + ) as st + on lt.cnes = st.iddd + and lt.ano = st.ano1 + and lt.mes = st.mes1 + ) + +select + safe_cast(ano as int64) as ano, + safe_cast(mes as int64) as mes, + safe_cast(sigla_uf as string) as sigla_uf, + safe_cast(id_municipio as string) as id_municipio, + safe_cast(cnes as string) as id_estabelecimento_cnes, + safe_cast(codleito as string) as tipo_especialidade_leito, + safe_cast(tp_leito as string) as tipo_leito, + safe_cast(qt_exist as string) as quantidade_total, + safe_cast(qt_contr as string) as quantidade_contratado, + safe_cast(qt_sus as string) as quantidade_sus +from leito_x_estabelecimento +{% if is_incremental() %} + where + + date(cast(ano as int64), cast(mes as int64), 1) + > (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) +{% endif %} diff --git a/models/br_ms_cnes/br_ms_cnes___profissional.sql b/models/br_ms_cnes/br_ms_cnes__profissional.sql similarity index 65% rename from models/br_ms_cnes/br_ms_cnes___profissional.sql rename to models/br_ms_cnes/br_ms_cnes__profissional.sql index a4dbd22f..0121a723 100644 --- a/models/br_ms_cnes/br_ms_cnes___profissional.sql +++ b/models/br_ms_cnes/br_ms_cnes__profissional.sql @@ -10,8 +10,8 @@ }, pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ - 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', ], ) }} @@ -19,14 +19,30 @@ with raw_cnes_profissional as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-staging.br_ms_cnes_staging.profissional` + from `basedosdados-dev.br_ms_cnes_staging.profissional` where cnes is not null + ), + profissional_x_estabelecimento as ( + select * + from raw_cnes_profissional as pf + left join + ( + select + id_municipio, + cast(ano as string) as ano1, + cast(mes as string) as mes1, + id_estabelecimento_cnes as iddd + from `basedosdados-dev.br_ms_cnes.estabelecimento` + ) as st + on pf.cnes = st.iddd + and pf.ano = st.ano1 + and pf.mes = st.mes1 ) - select cast(substr(competen, 1, 4) as int64) as ano, cast(substr(competen, 5, 2) as int64) as mes, safe_cast(sigla_uf as string) sigla_uf, + safe_cast(id_municipio as string) id_municipio, safe_cast(cnes as string) id_estabelecimento_cnes, -- replace de valores de linha com 6 zeros para null. 6 zeros é valor do campo -- UFMUNRES que indica null @@ -48,9 +64,10 @@ select safe_cast(horaoutr as int64) carga_horaria_outros, safe_cast(horahosp as int64) carga_horaria_hospitalar, safe_cast(hora_amb as int64) carga_horaria_ambulatorial -from raw_cnes_profissional +from profissional_x_estabelecimento {% if is_incremental() %} where + date(cast(ano as int64), cast(mes as int64), 1) > (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) {% endif %} diff --git a/models/br_ms_cnes/br_ms_cnes___regra_contratual.sql b/models/br_ms_cnes/br_ms_cnes__regra_contratual.sql similarity index 79% rename from models/br_ms_cnes/br_ms_cnes___regra_contratual.sql rename to models/br_ms_cnes/br_ms_cnes__regra_contratual.sql index 945cb175..636d0f3d 100644 --- a/models/br_ms_cnes/br_ms_cnes___regra_contratual.sql +++ b/models/br_ms_cnes/br_ms_cnes__regra_contratual.sql @@ -10,8 +10,8 @@ }, pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ - 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', ], ) }} @@ -19,7 +19,7 @@ with raw_cnes_regra_contratual as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-staging.br_ms_cnes_staging.regra_contratual` + from `basedosdados-dev.br_ms_cnes_staging.regra_contratual` where cnes is not null ), raw_cnes_regra_contratual_without_duplicates as ( @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados.br_bd_diretorios_brasil.municipio` + from `basedosdados-dev.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_regra_contratual_without_duplicates.codufmun = mun.id_municipio_6 @@ -72,6 +72,7 @@ select from cnes_add_muni as t {% if is_incremental() %} where + date(cast(ano as int64), cast(mes as int64), 1) > (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) {% endif %} diff --git a/models/br_ms_cnes/br_ms_cnes___servico_especializado.sql b/models/br_ms_cnes/br_ms_cnes__servico_especializado.sql similarity index 78% rename from models/br_ms_cnes/br_ms_cnes___servico_especializado.sql rename to models/br_ms_cnes/br_ms_cnes__servico_especializado.sql index 8b83327b..2c5344ca 100644 --- a/models/br_ms_cnes/br_ms_cnes___servico_especializado.sql +++ b/models/br_ms_cnes/br_ms_cnes__servico_especializado.sql @@ -10,8 +10,8 @@ }, pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ - 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', ], ) }} @@ -19,7 +19,7 @@ with raw_cnes_servico_especializado as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-staging.br_ms_cnes_staging.servico_especializado` + from `basedosdados-dev.br_ms_cnes_staging.servico_especializado` where cnes is not null ), raw_cnes_servico_especializado_without_duplicates as ( @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados.br_bd_diretorios_brasil.municipio` + from `basedosdados-dev.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_servico_especializado_without_duplicates.codufmun = mun.id_municipio_6 @@ -59,6 +59,7 @@ select from cnes_add_muni as t {% if is_incremental() %} where + date(cast(ano as int64), cast(mes as int64), 1) > (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) {% endif %} diff --git a/models/br_ms_cnes/schema.yml b/models/br_ms_cnes/schema.yml index 0492012c..0da9364d 100644 --- a/models/br_ms_cnes/schema.yml +++ b/models/br_ms_cnes/schema.yml @@ -2,19 +2,36 @@ version: 2 models: - name: equipamento - description: Esta tabela contém o catálogo de equipamentos em atividade ou não + description: Esta tabela contém o catálogo de equipamentos em atividade ou não nos estabelecimentos de saúde + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ano + - mes + - id_estabelecimento_cnes + - id_equipamento + - tipo_equipamento columns: - name: ano description: Ano + tests: [not_null] - name: mes description: Mes + tests: [not_null] - name: sigla_uf description: Sigla da Unidade da Federação - name: id_municipio description: ID Município IBGE - 7 Dígitos + tests: + - relationships: + to: ref('br_bd_diretorios_brasil__municipio') + field: id_municipio + - dbt_utils.not_null_proportion: + at_least: 0.05 - name: id_estabelecimento_cnes description: ID Estabelecimento - CNES + tests: [not_null] - name: id_equipamento description: Identificador do equipamento - name: tipo_equipamento @@ -30,17 +47,35 @@ models: - name: equipe description: Esta tabela contém o cadastro de equipes em atividade ou não nos estabelecimentos de saúde. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ano + - mes + - sigla_uf + - id_estabelecimento_cnes + - id_equipe + - tipo_equipe columns: - name: ano description: Ano + tests: [not_null] - name: mes description: Mês + tests: [not_null] - name: sigla_uf description: Sigla da Unidade da Federação - name: id_municipio description: ID Município IBGE - 7 Dígitos + tests: + - relationships: + to: ref('br_bd_diretorios_brasil__municipio') + field: id_municipio + - dbt_utils.not_null_proportion: + at_least: 0.05 - name: id_estabelecimento_cnes description: ID Estabelecimento - CNES + tests: [not_null] - name: id_equipe description: ID da equipe dado pelo código do Município, area e sequencial da equipe @@ -82,15 +117,37 @@ models: description: Indicador de atendimento a população Pronasci - name: profissional description: A tabela fornece o cadastro dos profissionais de saúde nos estabelecimentos. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ano + - mes + - id_estabelecimento_cnes + - cartao_nacional_saude + - nome + - cbo_2002 + - id_vinculo + - indicador_atende_sus columns: - name: ano description: Ano + tests: [not_null] - name: mes description: Mês + tests: [not_null] - name: sigla_uf description: Sigla da Unidade da Federação + - name: id_municipio + description: ID Município IBGE - 7 Dígitos + tests: + - relationships: + to: ref('br_bd_diretorios_brasil__municipio') + field: id_municipio + - dbt_utils.not_null_proportion: + at_least: 0.05 - name: id_estabelecimento_cnes description: ID Estabelecimento - CNES + tests: [not_null] - name: id_municipio_6_residencia description: ID Município IBGE - 6 Dígitos do local de residência do profissional - name: nome @@ -127,11 +184,16 @@ models: - name: estabelecimento description: A tabela fornece informações sobre a gestão e operação dos estabelecimentos de saúde brasileiros. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: [ano, mes, id_estabelecimento_cnes] columns: - name: ano description: Ano + tests: [not_null] - name: mes description: Mês + tests: [not_null] - name: sigla_uf description: Sigla da Unidade da Federação - name: ano_atualizacao @@ -140,6 +202,12 @@ models: description: Mês de atualização da informação pelo estabelecimento - name: id_municipio description: ID Município - IBGE 7 Dígitos + tests: + - relationships: + to: ref('br_bd_diretorios_brasil__municipio') + field: id_municipio + - dbt_utils.not_null_proportion: + at_least: 0.05 - name: id_municipio_6 description: ID Município - IBGE 6 Dígitos - name: id_regiao_saude @@ -154,6 +222,7 @@ models: description: Código do CEP - name: id_estabelecimento_cnes description: ID estabelecimento - CNES + tests: [not_null] - name: tipo_pessoa description: Indicador de pessoa física ou jurídica - name: cpf_cnpj @@ -610,6 +679,14 @@ models: plano de saúde privado - name: leito description: leitos disponíveis nos estabelecimentos de saúde + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ano + - mes + - id_estabelecimento_cnes + - tipo_especialidade_leito + - tipo_leito columns: - name: ano description: Ano @@ -617,8 +694,17 @@ models: description: Mês - name: sigla_uf description: Sigla da Unidade da Federação + - name: id_municipio + description: ID Município IBGE - 7 Dígitos + tests: + - relationships: + to: ref('br_bd_diretorios_brasil__municipio') + field: id_municipio + - dbt_utils.not_null_proportion: + at_least: 0.05 - name: id_estabelecimento_cnes description: ID Estabelecimento - CNES + tests: [not_null] - name: tipo_especialidade_leito description: Especialidade do leito - name: tipo_leito @@ -631,6 +717,9 @@ models: description: Quantidade de leitos para o SUS - name: dados_complementares description: Dados complementares dos estabelecimentos de saúde + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: [ano, mes, id_estabelecimento_cnes] columns: - name: ano description: Ano @@ -640,8 +729,15 @@ models: description: Sigla da Unidade da Federação - name: id_municipio description: ID Município IBGE - 7 Dígitos + tests: + - relationships: + to: ref('br_bd_diretorios_brasil__municipio') + field: id_municipio + - dbt_utils.not_null_proportion: + at_least: 0.05 - name: id_estabelecimento_cnes description: ID Estabelecimento - CNES + tests: [not_null] - name: cns_medico_responsavel_administrador_responsavel_tecnico description: CNS do médico responsável - Administrador ou Responsável Técnico - name: cns_medico_responsavel_oncologista_pediatrico @@ -824,6 +920,9 @@ models: CNES - name: estabelecimento_filantropico description: Esta tabela apresenta um cadastro de estabelecimentos de saúde filantrópicos. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: [ano, mes, id_estabelecimento_cnes] columns: - name: ano description: Ano @@ -833,8 +932,15 @@ models: description: Sigla da Unidade da Federação - name: id_municipio description: ID Município IBGE - 7 Dígitos + tests: + - relationships: + to: ref('br_bd_diretorios_brasil__municipio') + field: id_municipio + - dbt_utils.not_null_proportion: + at_least: 0.05 - name: id_estabelecimento_cnes description: ID Estabelecimento - CNES + tests: [not_null] - name: ano_competencia_inicial description: Ano Competência inicial - name: mes_competencia_inicial @@ -858,6 +964,9 @@ models: - name: estabelecimento_ensino description: Esta tabela apresenta informações sobre os estabelecimentos habilitados para atividades de ensino. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: [ano, mes, id_estabelecimento_cnes] columns: - name: ano description: Ano @@ -867,8 +976,15 @@ models: description: Sigla da Unidade da Federação - name: id_municipio description: ID Município IBGE - 7 Dígitos + tests: + - relationships: + to: ref('br_bd_diretorios_brasil__municipio') + field: id_municipio + - dbt_utils.not_null_proportion: + at_least: 0.05 - name: id_estabelecimento_cnes description: ID Estabelecimento - CNES + tests: [not_null] - name: ano_competencia_inicial description: Ano Competência inicial - name: mes_competencia_inicial @@ -890,17 +1006,35 @@ models: - name: gestao_metas description: Esta tabela apresenta informações sobre os contratos de gestão de metas dos estabelecimentos de sáude. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ano + - mes + - id_estabelecimento_cnes + - tipo_habilitacao + - portaria + - data_portaria columns: - name: ano description: Ano + tests: [not_null] - name: mes description: Mês + tests: [not_null] - name: sigla_uf description: Sigla da Unidade da Federação - name: id_municipio description: ID Município IBGE - 7 Dígitos + tests: + - relationships: + to: ref('br_bd_diretorios_brasil__municipio') + field: id_municipio + - dbt_utils.not_null_proportion: + at_least: 0.05 - name: id_estabelecimento_cnes description: ID Estabelecimento - CNES + tests: [not_null] - name: ano_competencia_inicial description: Ano Competência inicial - name: mes_competencia_inicial @@ -923,17 +1057,35 @@ models: description: Mês da portaria - name: incentivos description: A tabela fornece informações sobre os incentivos do estabelecimento. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ano + - mes + - id_estabelecimento_cnes + - tipo_habilitacao + - portaria + - data_portaria columns: - name: ano description: Ano + tests: [not_null] - name: mes description: Mês + tests: [not_null] - name: sigla_uf description: Sigla da Unidade da Federação - name: id_municipio description: ID Município IBGE - 7 Dígitos + tests: + - relationships: + to: ref('br_bd_diretorios_brasil__municipio') + field: id_municipio + - dbt_utils.not_null_proportion: + at_least: 0.05 - name: id_estabelecimento_cnes description: ID Estabelecimento - CNES + tests: [not_null] - name: ano_competencia_inicial description: Ano Competência inicial - name: mes_competencia_inicial @@ -956,17 +1108,38 @@ models: description: Mês da portaria - name: habilitacao description: A tabela fornece informações sobre as habilitações do estabelecimento. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ano + - mes + - id_estabelecimento_cnes + - tipo_habilitacao + - nivel_habilitacao + - ano_competencia_inicial + - mes_competencia_inicial + - data_portaria + - portaria columns: - name: ano description: Ano + tests: [not_null] - name: mes description: Mês + tests: [not_null] - name: sigla_uf description: Sigla da Unidade da Federação - name: id_municipio description: ID Município IBGE - 7 Dígitos + tests: + - relationships: + to: ref('br_bd_diretorios_brasil__municipio') + field: id_municipio + - dbt_utils.not_null_proportion: + at_least: 0.05 - name: id_estabelecimento_cnes description: ID Estabelecimento - CNES + tests: [not_null] - name: quantidade_leitos description: Quantidade de leitos - name: ano_competencia_inicial @@ -991,17 +1164,34 @@ models: description: Mês da portaria - name: regra_contratual description: A tabela fornece informações sobre os contratos dos estabelecimentos. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ano + - mes + - id_estabelecimento_cnes + - tipo_habilitacao + - data_portaria columns: - name: ano description: Ano + tests: [not_null] - name: mes description: Mês + tests: [not_null] - name: sigla_uf description: Sigla da Unidade da Federação - name: id_municipio description: ID Município IBGE - 7 Dígitos + tests: + - relationships: + to: ref('br_bd_diretorios_brasil__municipio') + field: id_municipio + - dbt_utils.not_null_proportion: + at_least: 0.05 - name: id_estabelecimento_cnes description: ID Estabelecimento - CNES + tests: [not_null] - name: ano_competencia_inicial description: Ano Competência inicial - name: mes_competencia_inicial @@ -1023,17 +1213,38 @@ models: - name: servico_especializado description: A tabela fornece informações sobre serviços especializados ofertados pelo estabelecimento. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - ano + - mes + - id_estabelecimento_cnes + - tipo_classificacao_bd + - tipo_servico_especializado_unico + - indicador_servico_ambulatorial_sus + - indicador_servico_nao_sus + - indicador_servico_hospitalar_nao_sus + - indicador_servico_hospitalar_sus columns: - name: ano description: Ano + tests: [not_null] - name: mes description: Mês + tests: [not_null] - name: sigla_uf description: Sigla da Unidade da Federação - name: id_municipio description: ID Município IBGE - 7 Dígitos + tests: + - relationships: + to: ref('br_bd_diretorios_brasil__municipio') + field: id_municipio + - dbt_utils.not_null_proportion: + at_least: 0.05 - name: id_estabelecimento_cnes description: ID Estabelecimento - CNES + tests: [not_null] - name: tipo_servico_especializado description: Código do serviço especializado - name: tipo_classificacao From d8256d4f455ffcbeb12db1d2ed9fc06dd54d6f72 Mon Sep 17 00:00:00 2001 From: folhesgabriel Date: Sun, 25 Feb 2024 21:58:21 -0300 Subject: [PATCH 2/3] fix: set models to basedosdados-staging --- .../br_ms_cnes__dados_complementares.sql | 4 ++-- models/br_ms_cnes/br_ms_cnes__equipamento.sql | 4 ++-- models/br_ms_cnes/br_ms_cnes__equipe.sql | 4 ++-- .../br_ms_cnes/br_ms_cnes__estabelecimento.sql | 4 ++-- .../br_ms_cnes__estabelecimento_ensino.sql | 4 ++-- .../br_ms_cnes__estabelecimento_filantropico.sql | 4 ++-- models/br_ms_cnes/br_ms_cnes__gestao_metas.sql | 4 ++-- models/br_ms_cnes/br_ms_cnes__habilitacao.sql | 5 +++++ models/br_ms_cnes/br_ms_cnes__incentivos.sql | 4 ++-- models/br_ms_cnes/br_ms_cnes__leito.sql | 10 ++++++---- models/br_ms_cnes/br_ms_cnes__profissional.sql | 16 ++++++++-------- .../br_ms_cnes/br_ms_cnes__regra_contratual.sql | 4 ++-- .../br_ms_cnes__servico_especializado.sql | 4 ++-- 13 files changed, 39 insertions(+), 32 deletions(-) diff --git a/models/br_ms_cnes/br_ms_cnes__dados_complementares.sql b/models/br_ms_cnes/br_ms_cnes__dados_complementares.sql index 13ed89dc..b721e75f 100644 --- a/models/br_ms_cnes/br_ms_cnes__dados_complementares.sql +++ b/models/br_ms_cnes/br_ms_cnes__dados_complementares.sql @@ -19,7 +19,7 @@ with raw_cnes_dados_complementares as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-dev.br_ms_cnes_staging.dados_complementares` + from `basedosdados-staging.br_ms_cnes_staging.dados_complementares` where cnes is not null ), raw_cnes_dados_complementares_without_duplicates as ( @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados-dev.br_bd_diretorios_brasil.municipio` + from `basedosdados-staging.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_dados_complementares_without_duplicates.codufmun = mun.id_municipio_6 diff --git a/models/br_ms_cnes/br_ms_cnes__equipamento.sql b/models/br_ms_cnes/br_ms_cnes__equipamento.sql index 3162bd7f..95a4bb9c 100644 --- a/models/br_ms_cnes/br_ms_cnes__equipamento.sql +++ b/models/br_ms_cnes/br_ms_cnes__equipamento.sql @@ -21,7 +21,7 @@ with raw_cnes_equipamento as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-dev.br_ms_cnes_staging.equipamento` + from `basedosdados-staging.br_ms_cnes_staging.equipamento` where cnes is not null ), unique_raw_cnes_equipamento as ( @@ -35,7 +35,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados-dev.br_bd_diretorios_brasil.municipio` + from `basedosdados-staging.br_bd_diretorios_brasil.municipio` ) as mun on unique_raw_cnes_equipamento.codufmun = mun.id_municipio_6 ) diff --git a/models/br_ms_cnes/br_ms_cnes__equipe.sql b/models/br_ms_cnes/br_ms_cnes__equipe.sql index b06f3759..a6c7a721 100644 --- a/models/br_ms_cnes/br_ms_cnes__equipe.sql +++ b/models/br_ms_cnes/br_ms_cnes__equipe.sql @@ -20,7 +20,7 @@ with raw_cnes_equipe as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-dev.br_ms_cnes_staging.equipe` + from `basedosdados-staging.br_ms_cnes_staging.equipe` where cnes is not null ), cnes_add_muni as ( @@ -30,7 +30,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados-dev.br_bd_diretorios_brasil.municipio` + from `basedosdados-staging.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_equipe.codufmun = mun.id_municipio_6 ) diff --git a/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql b/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql index c1bf0825..3ce4e611 100644 --- a/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql +++ b/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados-dev.br_bd_diretorios_brasil.municipio` + from `basedosdados-staging.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_estabelecimento_without_duplicates.codufmun = mun.id_municipio_6 ) @@ -239,7 +239,7 @@ select safe_cast(ap03cv04 as int64) indicador_atendimento_sadt_plano_seguro_terceiro, safe_cast(ap03cv05 as int64) indicador_atendimento_sadt_plano_saude_publico, safe_cast(ap03cv06 as int64) indicador_atendimento_sadt_plano_saude_privado, - safe_cast(ap04cv01 as string) indicador_atendimento_urgencia_sus, + safe_cast(ap04cv01 as int64) indicador_atendimento_urgencia_sus, safe_cast(ap04cv02 as int64) indicador_atendimento_urgencia_privado, safe_cast(ap04cv03 as int64) indicador_atendimento_urgencia_plano_seguro_proprio, safe_cast(ap04cv04 as int64) indicador_atendimento_urgencia_plano_seguro_terceiro, diff --git a/models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql b/models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql index 7c2b7132..f46c7d0d 100644 --- a/models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql +++ b/models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql @@ -19,7 +19,7 @@ with raw_cnes_estabelecimento_ensino as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-dev.br_ms_cnes_staging.estabelecimento_ensino` + from `basedosdados-staging.br_ms_cnes_staging.estabelecimento_ensino` where cnes is not null ), raw_cnes_estabelecimento_ensino_without_duplicates as ( @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados-dev.br_bd_diretorios_brasil.municipio` + from `basedosdados-staging.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_estabelecimento_ensino_without_duplicates.codufmun = mun.id_municipio_6 diff --git a/models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql b/models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql index 32b6804c..7849451a 100644 --- a/models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql +++ b/models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql @@ -19,7 +19,7 @@ with raw_cnes_estabelecimento_filantropico as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-dev.br_ms_cnes_staging.estabelecimento_filantropico` + from `basedosdados-staging.br_ms_cnes_staging.estabelecimento_filantropico` where cnes is not null ), raw_cnes_estabelecimento_filantropico_without_duplicates as ( @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados-dev.br_bd_diretorios_brasil.municipio` + from `basedosdados-staging.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_estabelecimento_filantropico_without_duplicates.codufmun = mun.id_municipio_6 diff --git a/models/br_ms_cnes/br_ms_cnes__gestao_metas.sql b/models/br_ms_cnes/br_ms_cnes__gestao_metas.sql index 53179720..7d95d356 100644 --- a/models/br_ms_cnes/br_ms_cnes__gestao_metas.sql +++ b/models/br_ms_cnes/br_ms_cnes__gestao_metas.sql @@ -19,7 +19,7 @@ with raw_cnes_gestao_metas as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-dev.br_ms_cnes_staging.gestao_metas` + from `basedosdados-staging.br_ms_cnes_staging.gestao_metas` where cnes is not null ), raw_cnes_gestao_metas_without_duplicates as ( @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados-dev.br_bd_diretorios_brasil.municipio` + from `basedosdados-staging.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_gestao_metas_without_duplicates.codufmun = mun.id_municipio_6 ) diff --git a/models/br_ms_cnes/br_ms_cnes__habilitacao.sql b/models/br_ms_cnes/br_ms_cnes__habilitacao.sql index ed5fc2c3..9e7ade77 100644 --- a/models/br_ms_cnes/br_ms_cnes__habilitacao.sql +++ b/models/br_ms_cnes/br_ms_cnes__habilitacao.sql @@ -8,6 +8,11 @@ "data_type": "int64", "range": {"start": 2005, "end": 2024, "interval": 1}, }, + pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=[ + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + ], ) }} with diff --git a/models/br_ms_cnes/br_ms_cnes__incentivos.sql b/models/br_ms_cnes/br_ms_cnes__incentivos.sql index 6683b499..27dbbbb7 100644 --- a/models/br_ms_cnes/br_ms_cnes__incentivos.sql +++ b/models/br_ms_cnes/br_ms_cnes__incentivos.sql @@ -19,7 +19,7 @@ with raw_cnes_incentivos as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-dev.br_ms_cnes_staging.incentivos` + from `basedosdados-staging.br_ms_cnes_staging.incentivos` where cnes is not null ), raw_cnes_incentivos_without_duplicates as ( @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados-dev.br_bd_diretorios_brasil.municipio` + from `basedosdados-staging.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_incentivos_without_duplicates.codufmun = mun.id_municipio_6 ) diff --git a/models/br_ms_cnes/br_ms_cnes__leito.sql b/models/br_ms_cnes/br_ms_cnes__leito.sql index 265a15a5..5b21730f 100644 --- a/models/br_ms_cnes/br_ms_cnes__leito.sql +++ b/models/br_ms_cnes/br_ms_cnes__leito.sql @@ -20,7 +20,9 @@ with raw_cnes_leito as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo - select * from `basedosdados-dev.br_ms_cnes_staging.leito` where cnes is not null + select * + from `basedosdados-staging.br_ms_cnes_staging.leito` + where cnes is not null ), cnes_leito_without_duplicates as (select distinct * from raw_cnes_leito), leito_x_estabelecimento as ( @@ -52,9 +54,9 @@ select safe_cast(cnes as string) as id_estabelecimento_cnes, safe_cast(codleito as string) as tipo_especialidade_leito, safe_cast(tp_leito as string) as tipo_leito, - safe_cast(qt_exist as string) as quantidade_total, - safe_cast(qt_contr as string) as quantidade_contratado, - safe_cast(qt_sus as string) as quantidade_sus + safe_cast(qt_exist as int64) as quantidade_total, + safe_cast(qt_contr as int64) as quantidade_contratado, + safe_cast(qt_sus as int64) as quantidade_sus from leito_x_estabelecimento {% if is_incremental() %} where diff --git a/models/br_ms_cnes/br_ms_cnes__profissional.sql b/models/br_ms_cnes/br_ms_cnes__profissional.sql index 0121a723..9cc5e9e3 100644 --- a/models/br_ms_cnes/br_ms_cnes__profissional.sql +++ b/models/br_ms_cnes/br_ms_cnes__profissional.sql @@ -19,7 +19,7 @@ with raw_cnes_profissional as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-dev.br_ms_cnes_staging.profissional` + from `basedosdados-staging.br_ms_cnes_staging.profissional` where cnes is not null ), profissional_x_estabelecimento as ( @@ -32,7 +32,7 @@ with cast(ano as string) as ano1, cast(mes as string) as mes1, id_estabelecimento_cnes as iddd - from `basedosdados-dev.br_ms_cnes.estabelecimento` + from `basedosdados-staging.br_ms_cnes.estabelecimento` ) as st on pf.cnes = st.iddd and pf.ano = st.ano1 @@ -55,12 +55,12 @@ select -- CNS_PROF que indica null safe_cast(regexp_replace(cns_prof, '0{15}', '') as string) cartao_nacional_saude, safe_cast(cbo as string) cbo_2002, - safe_cast(terceiro as string) indicador_estabelecimento_terceiro, - safe_cast(vincul_c as string) indicador_vinculo_contratado_sus, - safe_cast(vincul_a as string) indicador_vinculo_autonomo_sus, - safe_cast(vincul_n as string) indicador_vinculo_outros, - safe_cast(prof_sus as string) indicador_atende_sus, - safe_cast(profnsus as string) indicador_atende_nao_sus, + safe_cast(terceiro as int64) indicador_estabelecimento_terceiro, + safe_cast(vincul_c as int64) indicador_vinculo_contratado_sus, + safe_cast(vincul_a as int64) indicador_vinculo_autonomo_sus, + safe_cast(vincul_n as int64) indicador_vinculo_outros, + safe_cast(prof_sus as int64) indicador_atende_sus, + safe_cast(profnsus as int64) indicador_atende_nao_sus, safe_cast(horaoutr as int64) carga_horaria_outros, safe_cast(horahosp as int64) carga_horaria_hospitalar, safe_cast(hora_amb as int64) carga_horaria_ambulatorial diff --git a/models/br_ms_cnes/br_ms_cnes__regra_contratual.sql b/models/br_ms_cnes/br_ms_cnes__regra_contratual.sql index 636d0f3d..58354d24 100644 --- a/models/br_ms_cnes/br_ms_cnes__regra_contratual.sql +++ b/models/br_ms_cnes/br_ms_cnes__regra_contratual.sql @@ -19,7 +19,7 @@ with raw_cnes_regra_contratual as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-dev.br_ms_cnes_staging.regra_contratual` + from `basedosdados-staging.br_ms_cnes_staging.regra_contratual` where cnes is not null ), raw_cnes_regra_contratual_without_duplicates as ( @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados-dev.br_bd_diretorios_brasil.municipio` + from `basedosdados-staging.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_regra_contratual_without_duplicates.codufmun = mun.id_municipio_6 diff --git a/models/br_ms_cnes/br_ms_cnes__servico_especializado.sql b/models/br_ms_cnes/br_ms_cnes__servico_especializado.sql index 2c5344ca..fcf2125d 100644 --- a/models/br_ms_cnes/br_ms_cnes__servico_especializado.sql +++ b/models/br_ms_cnes/br_ms_cnes__servico_especializado.sql @@ -19,7 +19,7 @@ with raw_cnes_servico_especializado as ( -- 1. Retirar linhas com id_estabelecimento_cnes nulo select * - from `basedosdados-dev.br_ms_cnes_staging.servico_especializado` + from `basedosdados-staging.br_ms_cnes_staging.servico_especializado` where cnes is not null ), raw_cnes_servico_especializado_without_duplicates as ( @@ -33,7 +33,7 @@ with left join ( select id_municipio, id_municipio_6, - from `basedosdados-dev.br_bd_diretorios_brasil.municipio` + from `basedosdados-staging.br_bd_diretorios_brasil.municipio` ) as mun on raw_cnes_servico_especializado_without_duplicates.codufmun = mun.id_municipio_6 From 22ed54dc28fb76d1790f2df76c2bf6506856a4f9 Mon Sep 17 00:00:00 2001 From: folhesgabriel Date: Mon, 26 Feb 2024 09:31:09 -0300 Subject: [PATCH 3/3] fix: make requested changes --- models/br_ms_cnes/br_ms_cnes__dados_complementares.sql | 2 +- models/br_ms_cnes/br_ms_cnes__equipamento.sql | 2 +- models/br_ms_cnes/br_ms_cnes__equipe.sql | 2 +- models/br_ms_cnes/br_ms_cnes__estabelecimento.sql | 2 +- models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql | 2 +- models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql | 2 +- models/br_ms_cnes/br_ms_cnes__gestao_metas.sql | 2 +- models/br_ms_cnes/br_ms_cnes__habilitacao.sql | 2 +- models/br_ms_cnes/br_ms_cnes__incentivos.sql | 2 +- models/br_ms_cnes/br_ms_cnes__leito.sql | 2 +- models/br_ms_cnes/br_ms_cnes__profissional.sql | 2 +- models/br_ms_cnes/br_ms_cnes__regra_contratual.sql | 2 +- models/br_ms_cnes/br_ms_cnes__servico_especializado.sql | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/models/br_ms_cnes/br_ms_cnes__dados_complementares.sql b/models/br_ms_cnes/br_ms_cnes__dados_complementares.sql index b721e75f..0f79b901 100644 --- a/models/br_ms_cnes/br_ms_cnes__dados_complementares.sql +++ b/models/br_ms_cnes/br_ms_cnes__dados_complementares.sql @@ -11,7 +11,7 @@ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', ], ) }} diff --git a/models/br_ms_cnes/br_ms_cnes__equipamento.sql b/models/br_ms_cnes/br_ms_cnes__equipamento.sql index 95a4bb9c..d9de5e32 100644 --- a/models/br_ms_cnes/br_ms_cnes__equipamento.sql +++ b/models/br_ms_cnes/br_ms_cnes__equipamento.sql @@ -11,7 +11,7 @@ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', ], ) }} diff --git a/models/br_ms_cnes/br_ms_cnes__equipe.sql b/models/br_ms_cnes/br_ms_cnes__equipe.sql index a6c7a721..71475617 100644 --- a/models/br_ms_cnes/br_ms_cnes__equipe.sql +++ b/models/br_ms_cnes/br_ms_cnes__equipe.sql @@ -11,7 +11,7 @@ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', ], ) }} diff --git a/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql b/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql index 3ce4e611..0d26e30f 100644 --- a/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql +++ b/models/br_ms_cnes/br_ms_cnes__estabelecimento.sql @@ -11,7 +11,7 @@ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', ], ) }} diff --git a/models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql b/models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql index f46c7d0d..62b42df6 100644 --- a/models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql +++ b/models/br_ms_cnes/br_ms_cnes__estabelecimento_ensino.sql @@ -11,7 +11,7 @@ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', ], ) }} diff --git a/models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql b/models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql index 7849451a..33f0b623 100644 --- a/models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql +++ b/models/br_ms_cnes/br_ms_cnes__estabelecimento_filantropico.sql @@ -11,7 +11,7 @@ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', ], ) }} diff --git a/models/br_ms_cnes/br_ms_cnes__gestao_metas.sql b/models/br_ms_cnes/br_ms_cnes__gestao_metas.sql index 7d95d356..78c9f666 100644 --- a/models/br_ms_cnes/br_ms_cnes__gestao_metas.sql +++ b/models/br_ms_cnes/br_ms_cnes__gestao_metas.sql @@ -11,7 +11,7 @@ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', ], ) }} diff --git a/models/br_ms_cnes/br_ms_cnes__habilitacao.sql b/models/br_ms_cnes/br_ms_cnes__habilitacao.sql index 9e7ade77..36ef2eae 100644 --- a/models/br_ms_cnes/br_ms_cnes__habilitacao.sql +++ b/models/br_ms_cnes/br_ms_cnes__habilitacao.sql @@ -11,7 +11,7 @@ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', ], ) }} diff --git a/models/br_ms_cnes/br_ms_cnes__incentivos.sql b/models/br_ms_cnes/br_ms_cnes__incentivos.sql index 27dbbbb7..5413820b 100644 --- a/models/br_ms_cnes/br_ms_cnes__incentivos.sql +++ b/models/br_ms_cnes/br_ms_cnes__incentivos.sql @@ -11,7 +11,7 @@ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', ], ) }} diff --git a/models/br_ms_cnes/br_ms_cnes__leito.sql b/models/br_ms_cnes/br_ms_cnes__leito.sql index 5b21730f..2d86ab0c 100644 --- a/models/br_ms_cnes/br_ms_cnes__leito.sql +++ b/models/br_ms_cnes/br_ms_cnes__leito.sql @@ -11,7 +11,7 @@ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', ], ) }} diff --git a/models/br_ms_cnes/br_ms_cnes__profissional.sql b/models/br_ms_cnes/br_ms_cnes__profissional.sql index 9cc5e9e3..93b1d402 100644 --- a/models/br_ms_cnes/br_ms_cnes__profissional.sql +++ b/models/br_ms_cnes/br_ms_cnes__profissional.sql @@ -11,7 +11,7 @@ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', ], ) }} diff --git a/models/br_ms_cnes/br_ms_cnes__regra_contratual.sql b/models/br_ms_cnes/br_ms_cnes__regra_contratual.sql index 58354d24..e185e4f6 100644 --- a/models/br_ms_cnes/br_ms_cnes__regra_contratual.sql +++ b/models/br_ms_cnes/br_ms_cnes__regra_contratual.sql @@ -11,7 +11,7 @@ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', ], ) }} diff --git a/models/br_ms_cnes/br_ms_cnes__servico_especializado.sql b/models/br_ms_cnes/br_ms_cnes__servico_especializado.sql index fcf2125d..554e3767 100644 --- a/models/br_ms_cnes/br_ms_cnes__servico_especializado.sql +++ b/models/br_ms_cnes/br_ms_cnes__servico_especializado.sql @@ -11,7 +11,7 @@ pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", post_hook=[ 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', ], ) }}