diff --git a/models/br_fgv_igp/igp_10_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql similarity index 83% rename from models/br_fgv_igp/igp_10_mes.sql rename to models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql index 66e405a4..0e9e5216 100644 --- a/models/br_fgv_igp/igp_10_mes.sql +++ b/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql @@ -1,3 +1,9 @@ +{{ + config( + alias = 'igp_10_mes', + schema = 'br_fgv_igp' + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, diff --git a/models/br_fgv_igp/igp_di_ano.sql b/models/br_fgv_igp/br_fgv_igp__igp_di_ano.sql similarity index 76% rename from models/br_fgv_igp/igp_di_ano.sql rename to models/br_fgv_igp/br_fgv_igp__igp_di_ano.sql index 628e304f..c9916cef 100644 --- a/models/br_fgv_igp/igp_di_ano.sql +++ b/models/br_fgv_igp/br_fgv_igp__igp_di_ano.sql @@ -1,3 +1,9 @@ +{{ + config( + alias = "igp_di_ano", + schema = 'br_fgv_igp' + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(indice_medio AS FLOAT64) indice_medio, diff --git a/models/br_fgv_igp/igp_di_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql similarity index 71% rename from models/br_fgv_igp/igp_di_mes.sql rename to models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql index 83b4c104..0e9e5216 100644 --- a/models/br_fgv_igp/igp_di_mes.sql +++ b/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql @@ -1,3 +1,9 @@ +{{ + config( + alias = 'igp_10_mes', + schema = 'br_fgv_igp' + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -6,4 +12,4 @@ SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal -FROM basedosdados-dev.br_fgv_igp_staging.igp_di_mes AS t \ No newline at end of file +FROM basedosdados-dev.br_fgv_igp_staging.igp_10_mes AS t \ No newline at end of file diff --git a/models/br_fgv_igp/igp_m_ano.sql b/models/br_fgv_igp/br_fgv_igp__igp_m_ano.sql similarity index 76% rename from models/br_fgv_igp/igp_m_ano.sql rename to models/br_fgv_igp/br_fgv_igp__igp_m_ano.sql index c4ddc3fa..90b18161 100644 --- a/models/br_fgv_igp/igp_m_ano.sql +++ b/models/br_fgv_igp/br_fgv_igp__igp_m_ano.sql @@ -1,3 +1,9 @@ +{{ + config( + alias = "igp_m_ano", + schema = 'br_fgv_igp' + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(indice_medio AS FLOAT64) indice_medio, diff --git a/models/br_fgv_igp/igp_m_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql similarity index 87% rename from models/br_fgv_igp/igp_m_mes.sql rename to models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql index d7ee02d5..9f2ca06f 100644 --- a/models/br_fgv_igp/igp_m_mes.sql +++ b/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql @@ -1,3 +1,10 @@ +{{ + config( + alias = 'igp_m_mes', + schema = 'br_fgv_igp' + ) +}} + SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, diff --git a/models/br_fgv_igp/igp_og_ano.sql b/models/br_fgv_igp/br_fgv_igp__igp_og_ano.sql similarity index 76% rename from models/br_fgv_igp/igp_og_ano.sql rename to models/br_fgv_igp/br_fgv_igp__igp_og_ano.sql index e64acc9f..a0a07415 100644 --- a/models/br_fgv_igp/igp_og_ano.sql +++ b/models/br_fgv_igp/br_fgv_igp__igp_og_ano.sql @@ -1,3 +1,9 @@ +{{ + config( + alias = "igp_og_ano", + schema = 'br_fgv_igp' + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(indice_medio AS FLOAT64) indice_medio, diff --git a/models/br_fgv_igp/igp_og_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql similarity index 82% rename from models/br_fgv_igp/igp_og_mes.sql rename to models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql index 3aaaa800..a2b71d20 100644 --- a/models/br_fgv_igp/igp_og_mes.sql +++ b/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql @@ -1,3 +1,10 @@ +{{ + config( + alias = 'igp_og_mes', + schema = 'br_fgv_igp' + ) +}} + SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql index 3f1e7028..e6ce0f6d 100644 --- a/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql +++ b/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_brasil', schema='br_ibge_inpc') }} +{{ + config( + alias='mes_brasil', + schema='br_ibge_inpc', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 1979, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -9,3 +32,6 @@ SAFE_CAST(variacao_semestral AS FLOAT64) variacao_semestral, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses FROM basedosdados-dev.br_ibge_inpc_staging.mes_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql index 2ea22e4c..35af70be 100644 --- a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql +++ b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_brasil', schema='br_ibge_inpc') }} +{{ + config( + alias='mes_categoria_brasil', + schema='br_ibge_inpc', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2000, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,3 +33,6 @@ SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql index b79833b7..4eb67149 100644 --- a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql +++ b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_municipio', schema='br_ibge_inpc') }} +{{ + config( + alias='mes_categoria_municipio', + schema='br_ibge_inpc', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2000, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_municipio AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_municipio AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql index a5583723..ebc9715a 100644 --- a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql +++ b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_rm', schema='br_ibge_inpc') }} +{{ + config( + alias='mes_categoria_rm', + schema='br_ibge_inpc', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2000, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_rm AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_rm AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_brasil.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_brasil.sql index 98830f6b..3f99dc85 100644 --- a/models/br_ibge_ipca/br_ibge_ipca__mes_brasil.sql +++ b/models/br_ibge_ipca/br_ibge_ipca__mes_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_brasil', schema='br_ibge_ipca') }} +{{ + config( + alias='mes_brasil', + schema='br_ibge_ipca', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 1979, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -9,3 +32,7 @@ SAFE_CAST(variacao_semestral AS FLOAT64) variacao_semestral, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses FROM basedosdados-dev.br_ibge_ipca_staging.mes_brasil AS t + +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_brasil.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_brasil.sql index d06ef972..ea23f06a 100644 --- a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_brasil.sql +++ b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_brasil', schema='br_ibge_ipca') }} +{{ + config( + alias='mes_categoria_brasil', + schema='br_ibge_ipca', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -9,4 +32,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_brasil AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_municipio.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_municipio.sql index 802c4003..d00c180c 100644 --- a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_municipio.sql +++ b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_municipio.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_municipio', schema='br_ibge_ipca') }} +{{ + config( + alias='mes_categoria_municipio', + schema='br_ibge_ipca', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_municipio AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_municipio AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql index b19cfc90..bea0a9ee 100644 --- a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql +++ b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql @@ -1,4 +1,28 @@ -{{ config(alias='mes_categoria_rm', schema='br_ibge_ipca') }} +{{ + config( + alias='mes_categoria_rm', + schema='br_ibge_ipca', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} + SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +34,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_rm AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_rm AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca15/br_ibge_ipca15__mes_brasil.sql b/models/br_ibge_ipca15/br_ibge_ipca15__mes_brasil.sql index db925caa..bc3dbc55 100644 --- a/models/br_ibge_ipca15/br_ibge_ipca15__mes_brasil.sql +++ b/models/br_ibge_ipca15/br_ibge_ipca15__mes_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_brasil', schema='br_ibge_ipca15') }} +{{ + config( + alias='mes_brasil', + schema='br_ibge_ipca15', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2000, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -9,3 +32,6 @@ SAFE_CAST(variacao_semestral AS FLOAT64) variacao_semestral, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses FROM basedosdados-dev.br_ibge_ipca15_staging.mes_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_brasil.sql b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_brasil.sql index 98993787..b4f51da9 100644 --- a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_brasil.sql +++ b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_brasil', schema='br_ibge_ipca15') }} +{{ + config( + alias='mes_categoria_brasil', + schema='br_ibge_ipca15', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -9,4 +32,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_brasil AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_municipio.sql b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_municipio.sql index 2484f716..dc6b5a4d 100644 --- a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_municipio.sql +++ b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_municipio.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_municipio', schema='br_ibge_ipca15') }} +{{ + config( + alias='mes_categoria_municipio', + schema='br_ibge_ipca15', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_municipio AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_municipio AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_rm.sql b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_rm.sql index 75de6795..dd5b63b9 100644 --- a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_rm.sql +++ b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_rm.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_rm', schema='br_ibge_ipca15') }} +{{ + config( + alias='mes_categoria_rm', + schema='br_ibge_ipca15', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_rm AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_rm AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file