From f9e29cffde67df0fd5205ef4811035205e91b98b Mon Sep 17 00:00:00 2001 From: folhesgabriel Date: Tue, 9 Apr 2024 12:12:02 -0300 Subject: [PATCH 1/2] feat: and new incremental logic and pre_hook post_hook --- .../br_ms_sia__producao_ambulatorial.sql | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql b/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql index 65ef26b6..bf9d26fe 100644 --- a/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql +++ b/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql @@ -2,13 +2,18 @@ config( alias="producao_ambulatorial", schema="br_ms_sia", - materialized="incremental", + materialized="table", partition_by={ "field": "ano", "data_type": "int64", "range": {"start": 2005, "end": 2024, "interval": 1}, }, cluster_by=["mes", "sigla_uf"], + pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=[ + 'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(CURRENT_DATE(),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") FILTER USING (True)', + ], ) }} @@ -227,8 +232,9 @@ from sia {% if is_incremental() %} where - date(cast(ano as int64), cast(mes as int64), 1) - > (select max(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) - or date(cast(ano as int64), cast(mes as int64), 1) - < (select min(date(cast(ano as int64), cast(mes as int64), 1)) from {{ this }}) + date(cast(ano as int64), cast(mes as int64), 1) not in ( + select distinct (date(cast(ano as int64), cast(mes as int64), 1)) + from {{ this }} + ) + {% endif %} From 405a2346f4c344c6ab0a8959ca41b349d46f5c1a Mon Sep 17 00:00:00 2001 From: folhesgabriel Date: Tue, 9 Apr 2024 12:20:48 -0300 Subject: [PATCH 2/2] set materilized param to table --- models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql b/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql index bf9d26fe..006eff6d 100644 --- a/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql +++ b/models/br_ms_sia/br_ms_sia__producao_ambulatorial.sql @@ -2,7 +2,7 @@ config( alias="producao_ambulatorial", schema="br_ms_sia", - materialized="table", + materialized="incremental", partition_by={ "field": "ano", "data_type": "int64",