From f9e29cffde67df0fd5205ef4811035205e91b98b Mon Sep 17 00:00:00 2001 From: folhesgabriel Date: Tue, 9 Apr 2024 12:12:02 -0300 Subject: [PATCH 1/3] feat: and new incremental logic and pre_hook post_hook --- .../br_ms_sia__producao_ambulatorial.sql | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql b/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql index 65ef26b6..bf9d26fe 100644 --- a/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql +++ b/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql @@ -2,13 +2,18 @@ config( alias="producao_ambulatorial", schema="br_ms_sia", - materialized="incremental", + materialized="table", partition_by={ "field": "ano", "data_type": "int64", "range": {"start": 2005, "end": 2024, "interval": 1}, }, cluster_by=["mes", "sigla_uf"], + pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=[ + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', + ], ) }} @@ -227,8 +232,9 @@ from sia {% if is_incremental() %} where - date(cast(ano as int64), cast(mes as int64), 1) - > (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) - or date(cast(ano as int64), cast(mes as int64), 1) - < (select min(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) + date(cast(ano as int64), cast(mes as int64), 1) not in ( + select distinct (date(cast(ano as int64), cast(mes as int64), 1)) + from {{ this }} + ) + {% endif %} From 405a2346f4c344c6ab0a8959ca41b349d46f5c1a Mon Sep 17 00:00:00 2001 From: folhesgabriel Date: Tue, 9 Apr 2024 12:20:48 -0300 Subject: [PATCH 2/3] set materilized param to table --- models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql b/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql index bf9d26fe..006eff6d 100644 --- a/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql +++ b/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql @@ -2,7 +2,7 @@ config( alias="producao_ambulatorial", schema="br_ms_sia", - materialized="table", + materialized="incremental", partition_by={ "field": "ano", "data_type": "int64", From 8681112a82dd67ff46557345b8b1c58cb3e349dd Mon Sep 17 00:00:00 2001 From: Pedro Castro Date: Tue, 9 Apr 2024 14:10:25 -0300 Subject: [PATCH 3/3] [dbt]: add tests br_inep_censo_escolar.turma (#530) * fix col rede br_inep_censo_escolar.turma * add exceptation tests * add test accepted_values --- .../br_inep_censo_escolar__turma.sql | 1 + models/br_inep_censo_escolar/code/main.py | 8 ++++++++ models/br_inep_censo_escolar/schema.yml | 2 ++ 3 files changed, 11 insertions(+) diff --git a/models/br_inep_censo_escolar/br_inep_censo_escolar__turma.sql b/models/br_inep_censo_escolar/br_inep_censo_escolar__turma.sql index 07e29452..26151518 100644 --- a/models/br_inep_censo_escolar/br_inep_censo_escolar__turma.sql +++ b/models/br_inep_censo_escolar/br_inep_censo_escolar__turma.sql @@ -10,6 +10,7 @@ cluster_by="sigla_uf", ) }} + select safe_cast(ano as int64) ano, safe_cast(sigla_uf as string) sigla_uf, diff --git a/models/br_inep_censo_escolar/code/main.py b/models/br_inep_censo_escolar/code/main.py index 4c248ffe..e8be1168 100644 --- a/models/br_inep_censo_escolar/code/main.py +++ b/models/br_inep_censo_escolar/code/main.py @@ -102,6 +102,14 @@ bq_storage_cols_order = [i["name"] for i in bq_cols["columns"]] +df["rede"].unique() # type: ignore + +df["rede"] = df["rede"].replace( # type: ignore + {"1": "federal", "2": "estadual", "3": "municipal", "4": "privada"} +) + +df["rede"].unique() # type: ignore + for keys, df_split in df.groupby(partitions): ano, sigla_uf = keys # type: ignore path = os.path.join(OUTPUT, f"ano={ano}", f"sigla_uf={sigla_uf}") diff --git a/models/br_inep_censo_escolar/schema.yml b/models/br_inep_censo_escolar/schema.yml index 3e599c66..e3e23636 100644 --- a/models/br_inep_censo_escolar/schema.yml +++ b/models/br_inep_censo_escolar/schema.yml @@ -1186,6 +1186,8 @@ models: - relationships: to: ref('br_bd_diretorios_brasil__escola') field: rede + - accepted_values: + values: [municipal, estadual, federal, privada] - name: id_escola description: Código da Escola - name: id_turma