From e03257de576cfc2401a07404b17035f4b89a4ee1 Mon Sep 17 00:00:00 2001 From: folhesgabriel Date: Thu, 21 Sep 2023 19:51:49 -0300 Subject: [PATCH] adiciona ibge_ipca15, ibge_inpc e fgv_igp --- models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql | 36 ++++++++++++++++ models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql | 35 ++++++++++++++++ models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql | 37 ++++++++++++++++ models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql | 35 ++++++++++++++++ models/br_fgv_igp/igp_10_mes.sql | 9 ---- models/br_fgv_igp/igp_di_mes.sql | 9 ---- models/br_fgv_igp/igp_m_mes.sql | 11 ----- models/br_fgv_igp/igp_og_mes.sql | 9 ---- .../br_ibge_inpc/br_ibge_inpc__mes_brasil.sql | 28 ++++++++++++- .../br_ibge_inpc__mes_categoria_brasil.sql | 28 ++++++++++++- .../br_ibge_inpc__mes_categoria_municipio.sql | 30 ++++++++++++- .../br_ibge_inpc__mes_categoria_rm.sql | 30 ++++++++++++- .../br_ibge_ipca__mes_categoria_rm.sql | 42 +++++++++---------- .../br_ibge_ipca15__mes_brasil.sql | 28 ++++++++++++- .../br_ibge_ipca15__mes_categoria_brasil.sql | 30 ++++++++++++- ...r_ibge_ipca15__mes_categoria_municipio.sql | 30 ++++++++++++- .../br_ibge_ipca15__mes_categoria_rm.sql | 30 ++++++++++++- 17 files changed, 385 insertions(+), 72 deletions(-) create mode 100644 models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql create mode 100644 models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql create mode 100644 models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql create mode 100644 models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql delete mode 100644 models/br_fgv_igp/igp_10_mes.sql delete mode 100644 models/br_fgv_igp/igp_di_mes.sql delete mode 100644 models/br_fgv_igp/igp_m_mes.sql delete mode 100644 models/br_fgv_igp/igp_og_mes.sql diff --git a/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql new file mode 100644 index 00000000..1a664921 --- /dev/null +++ b/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql @@ -0,0 +1,36 @@ +{{ + config( + alias='igp_10_mes', + schema='br_fgv_igp', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 1993, + "end": 2024, + "interval": 1} + }, + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} +SELECT +SAFE_CAST(ano AS INT64) ano, +SAFE_CAST(mes AS INT64) mes, +SAFE_CAST(indice AS FLOAT64) indice, +SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, +SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, +SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, +SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal +FROM basedosdados-dev.br_fgv_igp_staging.igp_10_mes AS t +FROM basedosdados-dev.br_ibge_inpc_staging.mes_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql new file mode 100644 index 00000000..ed780e88 --- /dev/null +++ b/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql @@ -0,0 +1,35 @@ +{{ + config( + alias='igp_di_mes', + schema='br_fgv_igp', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 1944, + "end": 2024, + "interval": 1} + }, + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} +SELECT +SAFE_CAST(ano AS INT64) ano, +SAFE_CAST(mes AS INT64) mes, +SAFE_CAST(indice AS FLOAT64) indice, +SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, +SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, +SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, +SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal +FROM basedosdados-dev.br_fgv_igp_staging.igp_di_mes AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql new file mode 100644 index 00000000..ecbbc7c2 --- /dev/null +++ b/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql @@ -0,0 +1,37 @@ +{{ + config( + alias='igp_m_mes', + schema='br_fgv_igp', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 1989, + "end": 2024, + "interval": 1} + }, + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} +SELECT +SAFE_CAST(ano AS INT64) ano, +SAFE_CAST(mes AS INT64) mes, +SAFE_CAST(indice AS FLOAT64) indice, +SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, +SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, +SAFE_CAST(variacao_primeiro_decendio AS FLOAT64) variacao_primeiro_decendio, +SAFE_CAST(variacao_segundo_decendio AS FLOAT64) variacao_segundo_decendio, +SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, +SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal +FROM basedosdados-dev.br_fgv_igp_staging.igp_m_mes AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql new file mode 100644 index 00000000..3a9cc70d --- /dev/null +++ b/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql @@ -0,0 +1,35 @@ +{{ + config( + alias='igp_og_mes', + schema='br_fgv_igp', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 1969, + "end": 2024, + "interval": 1} + }, + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} +SELECT +SAFE_CAST(ano AS INT64) ano, +SAFE_CAST(mes AS INT64) mes, +SAFE_CAST(indice AS FLOAT64) indice, +SAFE_CAST(variacao_mensal AS FLOAT64) var_mensal, +SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, +SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, +SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal +FROM basedosdados-dev.br_fgv_igp_staging.igp_og_mes AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_fgv_igp/igp_10_mes.sql b/models/br_fgv_igp/igp_10_mes.sql deleted file mode 100644 index 66e405a4..00000000 --- a/models/br_fgv_igp/igp_10_mes.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT -SAFE_CAST(ano AS INT64) ano, -SAFE_CAST(mes AS INT64) mes, -SAFE_CAST(indice AS FLOAT64) indice, -SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, -SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, -SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, -SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal -FROM basedosdados-dev.br_fgv_igp_staging.igp_10_mes AS t \ No newline at end of file diff --git a/models/br_fgv_igp/igp_di_mes.sql b/models/br_fgv_igp/igp_di_mes.sql deleted file mode 100644 index 83b4c104..00000000 --- a/models/br_fgv_igp/igp_di_mes.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT -SAFE_CAST(ano AS INT64) ano, -SAFE_CAST(mes AS INT64) mes, -SAFE_CAST(indice AS FLOAT64) indice, -SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, -SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, -SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, -SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal -FROM basedosdados-dev.br_fgv_igp_staging.igp_di_mes AS t \ No newline at end of file diff --git a/models/br_fgv_igp/igp_m_mes.sql b/models/br_fgv_igp/igp_m_mes.sql deleted file mode 100644 index d7ee02d5..00000000 --- a/models/br_fgv_igp/igp_m_mes.sql +++ /dev/null @@ -1,11 +0,0 @@ -SELECT -SAFE_CAST(ano AS INT64) ano, -SAFE_CAST(mes AS INT64) mes, -SAFE_CAST(indice AS FLOAT64) indice, -SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, -SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, -SAFE_CAST(variacao_primeiro_decendio AS FLOAT64) variacao_primeiro_decendio, -SAFE_CAST(variacao_segundo_decendio AS FLOAT64) variacao_segundo_decendio, -SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, -SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal -FROM basedosdados-dev.br_fgv_igp_staging.igp_m_mes AS t \ No newline at end of file diff --git a/models/br_fgv_igp/igp_og_mes.sql b/models/br_fgv_igp/igp_og_mes.sql deleted file mode 100644 index 3aaaa800..00000000 --- a/models/br_fgv_igp/igp_og_mes.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT -SAFE_CAST(ano AS INT64) ano, -SAFE_CAST(mes AS INT64) mes, -SAFE_CAST(indice AS FLOAT64) indice, -SAFE_CAST(variacao_mensal AS FLOAT64) var_mensal, -SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, -SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, -SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal -FROM basedosdados-dev.br_fgv_igp_staging.igp_og_mes AS t \ No newline at end of file diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql index 3f1e7028..e6ce0f6d 100644 --- a/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql +++ b/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_brasil', schema='br_ibge_inpc') }} +{{ + config( + alias='mes_brasil', + schema='br_ibge_inpc', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 1979, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -9,3 +32,6 @@ SAFE_CAST(variacao_semestral AS FLOAT64) variacao_semestral, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses FROM basedosdados-dev.br_ibge_inpc_staging.mes_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql index 2ea22e4c..35af70be 100644 --- a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql +++ b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_brasil', schema='br_ibge_inpc') }} +{{ + config( + alias='mes_categoria_brasil', + schema='br_ibge_inpc', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2000, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,3 +33,6 @@ SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql index b79833b7..4eb67149 100644 --- a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql +++ b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_municipio', schema='br_ibge_inpc') }} +{{ + config( + alias='mes_categoria_municipio', + schema='br_ibge_inpc', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2000, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_municipio AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_municipio AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql index a5583723..ebc9715a 100644 --- a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql +++ b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_rm', schema='br_ibge_inpc') }} +{{ + config( + alias='mes_categoria_rm', + schema='br_ibge_inpc', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2000, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_rm AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_rm AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql index e6b2b35a..bea0a9ee 100644 --- a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql +++ b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql @@ -1,25 +1,25 @@ {{ - config( - alias='mes_categoria_rm', - schema='br_ibge_ipca', - materialized='incremental', - partition_by = { - "field": "ano", - "data_type": "int64", - "range": { - "start": 2020, - "end": 2024, - "interval": 1} - }, - pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", - post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter - ON {{this}} - GRANT TO ("allUsers") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter - ON {{this}} - GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + config( + alias='mes_categoria_rm', + schema='br_ibge_ipca', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] ) }} diff --git a/models/br_ibge_ipca15/br_ibge_ipca15__mes_brasil.sql b/models/br_ibge_ipca15/br_ibge_ipca15__mes_brasil.sql index db925caa..bc3dbc55 100644 --- a/models/br_ibge_ipca15/br_ibge_ipca15__mes_brasil.sql +++ b/models/br_ibge_ipca15/br_ibge_ipca15__mes_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_brasil', schema='br_ibge_ipca15') }} +{{ + config( + alias='mes_brasil', + schema='br_ibge_ipca15', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2000, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -9,3 +32,6 @@ SAFE_CAST(variacao_semestral AS FLOAT64) variacao_semestral, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses FROM basedosdados-dev.br_ibge_ipca15_staging.mes_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_brasil.sql b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_brasil.sql index 98993787..b4f51da9 100644 --- a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_brasil.sql +++ b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_brasil', schema='br_ibge_ipca15') }} +{{ + config( + alias='mes_categoria_brasil', + schema='br_ibge_ipca15', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -9,4 +32,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_brasil AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_municipio.sql b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_municipio.sql index 2484f716..dc6b5a4d 100644 --- a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_municipio.sql +++ b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_municipio.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_municipio', schema='br_ibge_ipca15') }} +{{ + config( + alias='mes_categoria_municipio', + schema='br_ibge_ipca15', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_municipio AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_municipio AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_rm.sql b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_rm.sql index 75de6795..dd5b63b9 100644 --- a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_rm.sql +++ b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_rm.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_rm', schema='br_ibge_ipca15') }} +{{ + config( + alias='mes_categoria_rm', + schema='br_ibge_ipca15', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_rm AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_rm AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file