From 59faec6ccbce3357ea6fee4d204e903f283be7e5 Mon Sep 17 00:00:00 2001 From: folhesgabriel Date: Thu, 21 Sep 2023 13:49:32 -0300 Subject: [PATCH 1/7] insere modelos br_ibge_ipca --- .../br_ibge_ipca/br_ibge_ipca__mes_brasil.sql | 29 ++++++++++++++++- .../br_ibge_ipca__mes_categoria_brasil.sql | 30 ++++++++++++++++-- .../br_ibge_ipca__mes_categoria_municipio.sql | 30 ++++++++++++++++-- .../br_ibge_ipca__mes_categoria_rm.sql | 31 +++++++++++++++++-- 4 files changed, 113 insertions(+), 7 deletions(-) diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_brasil.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_brasil.sql index 98830f6b..3f99dc85 100644 --- a/models/br_ibge_ipca/br_ibge_ipca__mes_brasil.sql +++ b/models/br_ibge_ipca/br_ibge_ipca__mes_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_brasil', schema='br_ibge_ipca') }} +{{ + config( + alias='mes_brasil', + schema='br_ibge_ipca', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 1979, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -9,3 +32,7 @@ SAFE_CAST(variacao_semestral AS FLOAT64) variacao_semestral, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses FROM basedosdados-dev.br_ibge_ipca_staging.mes_brasil AS t + +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_brasil.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_brasil.sql index d06ef972..ea23f06a 100644 --- a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_brasil.sql +++ b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_brasil', schema='br_ibge_ipca') }} +{{ + config( + alias='mes_categoria_brasil', + schema='br_ibge_ipca', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -9,4 +32,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_brasil AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_municipio.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_municipio.sql index 802c4003..d00c180c 100644 --- a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_municipio.sql +++ b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_municipio.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_municipio', schema='br_ibge_ipca') }} +{{ + config( + alias='mes_categoria_municipio', + schema='br_ibge_ipca', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_municipio AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_municipio AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql index b19cfc90..e6b2b35a 100644 --- a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql +++ b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql @@ -1,4 +1,28 @@ -{{ config(alias='mes_categoria_rm', schema='br_ibge_ipca') }} +{{ + config( + alias='mes_categoria_rm', + schema='br_ibge_ipca', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} + SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +34,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_rm AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_rm AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file From e03257de576cfc2401a07404b17035f4b89a4ee1 Mon Sep 17 00:00:00 2001 From: folhesgabriel Date: Thu, 21 Sep 2023 19:51:49 -0300 Subject: [PATCH 2/7] adiciona ibge_ipca15, ibge_inpc e fgv_igp --- models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql | 36 ++++++++++++++++ models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql | 35 ++++++++++++++++ models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql | 37 ++++++++++++++++ models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql | 35 ++++++++++++++++ models/br_fgv_igp/igp_10_mes.sql | 9 ---- models/br_fgv_igp/igp_di_mes.sql | 9 ---- models/br_fgv_igp/igp_m_mes.sql | 11 ----- models/br_fgv_igp/igp_og_mes.sql | 9 ---- .../br_ibge_inpc/br_ibge_inpc__mes_brasil.sql | 28 ++++++++++++- .../br_ibge_inpc__mes_categoria_brasil.sql | 28 ++++++++++++- .../br_ibge_inpc__mes_categoria_municipio.sql | 30 ++++++++++++- .../br_ibge_inpc__mes_categoria_rm.sql | 30 ++++++++++++- .../br_ibge_ipca__mes_categoria_rm.sql | 42 +++++++++---------- .../br_ibge_ipca15__mes_brasil.sql | 28 ++++++++++++- .../br_ibge_ipca15__mes_categoria_brasil.sql | 30 ++++++++++++- ...r_ibge_ipca15__mes_categoria_municipio.sql | 30 ++++++++++++- .../br_ibge_ipca15__mes_categoria_rm.sql | 30 ++++++++++++- 17 files changed, 385 insertions(+), 72 deletions(-) create mode 100644 models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql create mode 100644 models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql create mode 100644 models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql create mode 100644 models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql delete mode 100644 models/br_fgv_igp/igp_10_mes.sql delete mode 100644 models/br_fgv_igp/igp_di_mes.sql delete mode 100644 models/br_fgv_igp/igp_m_mes.sql delete mode 100644 models/br_fgv_igp/igp_og_mes.sql diff --git a/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql new file mode 100644 index 00000000..1a664921 --- /dev/null +++ b/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql @@ -0,0 +1,36 @@ +{{ + config( + alias='igp_10_mes', + schema='br_fgv_igp', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 1993, + "end": 2024, + "interval": 1} + }, + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} +SELECT +SAFE_CAST(ano AS INT64) ano, +SAFE_CAST(mes AS INT64) mes, +SAFE_CAST(indice AS FLOAT64) indice, +SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, +SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, +SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, +SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal +FROM basedosdados-dev.br_fgv_igp_staging.igp_10_mes AS t +FROM basedosdados-dev.br_ibge_inpc_staging.mes_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql new file mode 100644 index 00000000..ed780e88 --- /dev/null +++ b/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql @@ -0,0 +1,35 @@ +{{ + config( + alias='igp_di_mes', + schema='br_fgv_igp', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 1944, + "end": 2024, + "interval": 1} + }, + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} +SELECT +SAFE_CAST(ano AS INT64) ano, +SAFE_CAST(mes AS INT64) mes, +SAFE_CAST(indice AS FLOAT64) indice, +SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, +SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, +SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, +SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal +FROM basedosdados-dev.br_fgv_igp_staging.igp_di_mes AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql new file mode 100644 index 00000000..ecbbc7c2 --- /dev/null +++ b/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql @@ -0,0 +1,37 @@ +{{ + config( + alias='igp_m_mes', + schema='br_fgv_igp', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 1989, + "end": 2024, + "interval": 1} + }, + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} +SELECT +SAFE_CAST(ano AS INT64) ano, +SAFE_CAST(mes AS INT64) mes, +SAFE_CAST(indice AS FLOAT64) indice, +SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, +SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, +SAFE_CAST(variacao_primeiro_decendio AS FLOAT64) variacao_primeiro_decendio, +SAFE_CAST(variacao_segundo_decendio AS FLOAT64) variacao_segundo_decendio, +SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, +SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal +FROM basedosdados-dev.br_fgv_igp_staging.igp_m_mes AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql new file mode 100644 index 00000000..3a9cc70d --- /dev/null +++ b/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql @@ -0,0 +1,35 @@ +{{ + config( + alias='igp_og_mes', + schema='br_fgv_igp', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 1969, + "end": 2024, + "interval": 1} + }, + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} +SELECT +SAFE_CAST(ano AS INT64) ano, +SAFE_CAST(mes AS INT64) mes, +SAFE_CAST(indice AS FLOAT64) indice, +SAFE_CAST(variacao_mensal AS FLOAT64) var_mensal, +SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, +SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, +SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal +FROM basedosdados-dev.br_fgv_igp_staging.igp_og_mes AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_fgv_igp/igp_10_mes.sql b/models/br_fgv_igp/igp_10_mes.sql deleted file mode 100644 index 66e405a4..00000000 --- a/models/br_fgv_igp/igp_10_mes.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT -SAFE_CAST(ano AS INT64) ano, -SAFE_CAST(mes AS INT64) mes, -SAFE_CAST(indice AS FLOAT64) indice, -SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, -SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, -SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, -SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal -FROM basedosdados-dev.br_fgv_igp_staging.igp_10_mes AS t \ No newline at end of file diff --git a/models/br_fgv_igp/igp_di_mes.sql b/models/br_fgv_igp/igp_di_mes.sql deleted file mode 100644 index 83b4c104..00000000 --- a/models/br_fgv_igp/igp_di_mes.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT -SAFE_CAST(ano AS INT64) ano, -SAFE_CAST(mes AS INT64) mes, -SAFE_CAST(indice AS FLOAT64) indice, -SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, -SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, -SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, -SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal -FROM basedosdados-dev.br_fgv_igp_staging.igp_di_mes AS t \ No newline at end of file diff --git a/models/br_fgv_igp/igp_m_mes.sql b/models/br_fgv_igp/igp_m_mes.sql deleted file mode 100644 index d7ee02d5..00000000 --- a/models/br_fgv_igp/igp_m_mes.sql +++ /dev/null @@ -1,11 +0,0 @@ -SELECT -SAFE_CAST(ano AS INT64) ano, -SAFE_CAST(mes AS INT64) mes, -SAFE_CAST(indice AS FLOAT64) indice, -SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, -SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, -SAFE_CAST(variacao_primeiro_decendio AS FLOAT64) variacao_primeiro_decendio, -SAFE_CAST(variacao_segundo_decendio AS FLOAT64) variacao_segundo_decendio, -SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, -SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal -FROM basedosdados-dev.br_fgv_igp_staging.igp_m_mes AS t \ No newline at end of file diff --git a/models/br_fgv_igp/igp_og_mes.sql b/models/br_fgv_igp/igp_og_mes.sql deleted file mode 100644 index 3aaaa800..00000000 --- a/models/br_fgv_igp/igp_og_mes.sql +++ /dev/null @@ -1,9 +0,0 @@ -SELECT -SAFE_CAST(ano AS INT64) ano, -SAFE_CAST(mes AS INT64) mes, -SAFE_CAST(indice AS FLOAT64) indice, -SAFE_CAST(variacao_mensal AS FLOAT64) var_mensal, -SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, -SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, -SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal -FROM basedosdados-dev.br_fgv_igp_staging.igp_og_mes AS t \ No newline at end of file diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql index 3f1e7028..e6ce0f6d 100644 --- a/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql +++ b/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_brasil', schema='br_ibge_inpc') }} +{{ + config( + alias='mes_brasil', + schema='br_ibge_inpc', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 1979, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -9,3 +32,6 @@ SAFE_CAST(variacao_semestral AS FLOAT64) variacao_semestral, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses FROM basedosdados-dev.br_ibge_inpc_staging.mes_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql index 2ea22e4c..35af70be 100644 --- a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql +++ b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_brasil', schema='br_ibge_inpc') }} +{{ + config( + alias='mes_categoria_brasil', + schema='br_ibge_inpc', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2000, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,3 +33,6 @@ SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql index b79833b7..4eb67149 100644 --- a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql +++ b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_municipio', schema='br_ibge_inpc') }} +{{ + config( + alias='mes_categoria_municipio', + schema='br_ibge_inpc', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2000, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_municipio AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_municipio AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql index a5583723..ebc9715a 100644 --- a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql +++ b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_rm', schema='br_ibge_inpc') }} +{{ + config( + alias='mes_categoria_rm', + schema='br_ibge_inpc', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2000, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_rm AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_rm AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql index e6b2b35a..bea0a9ee 100644 --- a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql +++ b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql @@ -1,25 +1,25 @@ {{ - config( - alias='mes_categoria_rm', - schema='br_ibge_ipca', - materialized='incremental', - partition_by = { - "field": "ano", - "data_type": "int64", - "range": { - "start": 2020, - "end": 2024, - "interval": 1} - }, - pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", - post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter - ON {{this}} - GRANT TO ("allUsers") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter - ON {{this}} - GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + config( + alias='mes_categoria_rm', + schema='br_ibge_ipca', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] ) }} diff --git a/models/br_ibge_ipca15/br_ibge_ipca15__mes_brasil.sql b/models/br_ibge_ipca15/br_ibge_ipca15__mes_brasil.sql index db925caa..bc3dbc55 100644 --- a/models/br_ibge_ipca15/br_ibge_ipca15__mes_brasil.sql +++ b/models/br_ibge_ipca15/br_ibge_ipca15__mes_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_brasil', schema='br_ibge_ipca15') }} +{{ + config( + alias='mes_brasil', + schema='br_ibge_ipca15', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2000, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -9,3 +32,6 @@ SAFE_CAST(variacao_semestral AS FLOAT64) variacao_semestral, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses FROM basedosdados-dev.br_ibge_ipca15_staging.mes_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_brasil.sql b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_brasil.sql index 98993787..b4f51da9 100644 --- a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_brasil.sql +++ b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_brasil.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_brasil', schema='br_ibge_ipca15') }} +{{ + config( + alias='mes_categoria_brasil', + schema='br_ibge_ipca15', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -9,4 +32,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_brasil AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_brasil AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_municipio.sql b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_municipio.sql index 2484f716..dc6b5a4d 100644 --- a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_municipio.sql +++ b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_municipio.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_municipio', schema='br_ibge_ipca15') }} +{{ + config( + alias='mes_categoria_municipio', + schema='br_ibge_ipca15', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_municipio AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_municipio AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file diff --git a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_rm.sql b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_rm.sql index 75de6795..dd5b63b9 100644 --- a/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_rm.sql +++ b/models/br_ibge_ipca15/br_ibge_ipca15__mes_categoria_rm.sql @@ -1,4 +1,27 @@ -{{ config(alias='mes_categoria_rm', schema='br_ibge_ipca15') }} +{{ + config( + alias='mes_categoria_rm', + schema='br_ibge_ipca15', + materialized='incremental', + partition_by = { + "field": "ano", + "data_type": "int64", + "range": { + "start": 2020, + "end": 2024, + "interval": 1} + }, + pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter + ON {{this}} + GRANT TO ("allUsers") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', + 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter + ON {{this}} + GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") + FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(mes AS INT64) mes, @@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal, SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual, SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses -FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_rm AS t \ No newline at end of file +FROM basedosdados-dev.br_ibge_ipca15_staging.mes_categoria_rm AS t +{% if is_incremental() %} +WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) +{% endif %} \ No newline at end of file From 05cb47576eaa40ebc516dc96b884b6e0abc3d066 Mon Sep 17 00:00:00 2001 From: Gabriel Pisa <61624649+folhesgabriel@users.noreply.github.com> Date: Fri, 22 Sep 2023 08:36:55 -0300 Subject: [PATCH 3/7] Delete models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql --- models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql | 36 -------------------- 1 file changed, 36 deletions(-) delete mode 100644 models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql diff --git a/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql deleted file mode 100644 index 1a664921..00000000 --- a/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql +++ /dev/null @@ -1,36 +0,0 @@ -{{ - config( - alias='igp_10_mes', - schema='br_fgv_igp', - materialized='incremental', - partition_by = { - "field": "ano", - "data_type": "int64", - "range": { - "start": 1993, - "end": 2024, - "interval": 1} - }, - post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter - ON {{this}} - GRANT TO ("allUsers") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter - ON {{this}} - GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] - ) -}} -SELECT -SAFE_CAST(ano AS INT64) ano, -SAFE_CAST(mes AS INT64) mes, -SAFE_CAST(indice AS FLOAT64) indice, -SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, -SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, -SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, -SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal -FROM basedosdados-dev.br_fgv_igp_staging.igp_10_mes AS t -FROM basedosdados-dev.br_ibge_inpc_staging.mes_brasil AS t -{% if is_incremental() %} -WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) -{% endif %} \ No newline at end of file From 9896042306a62b22cd6421b9fa4d5f48fc38b4c8 Mon Sep 17 00:00:00 2001 From: Gabriel Pisa <61624649+folhesgabriel@users.noreply.github.com> Date: Fri, 22 Sep 2023 08:37:13 -0300 Subject: [PATCH 4/7] Delete models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql --- models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql | 35 -------------------- 1 file changed, 35 deletions(-) delete mode 100644 models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql diff --git a/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql deleted file mode 100644 index ed780e88..00000000 --- a/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql +++ /dev/null @@ -1,35 +0,0 @@ -{{ - config( - alias='igp_di_mes', - schema='br_fgv_igp', - materialized='incremental', - partition_by = { - "field": "ano", - "data_type": "int64", - "range": { - "start": 1944, - "end": 2024, - "interval": 1} - }, - post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter - ON {{this}} - GRANT TO ("allUsers") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter - ON {{this}} - GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] - ) -}} -SELECT -SAFE_CAST(ano AS INT64) ano, -SAFE_CAST(mes AS INT64) mes, -SAFE_CAST(indice AS FLOAT64) indice, -SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, -SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, -SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, -SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal -FROM basedosdados-dev.br_fgv_igp_staging.igp_di_mes AS t -{% if is_incremental() %} -WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) -{% endif %} \ No newline at end of file From cc9a901e24c049a2b5bc16bee8127acf1a10e6bd Mon Sep 17 00:00:00 2001 From: Gabriel Pisa <61624649+folhesgabriel@users.noreply.github.com> Date: Fri, 22 Sep 2023 08:37:22 -0300 Subject: [PATCH 5/7] Delete models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql --- models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql | 37 --------------------- 1 file changed, 37 deletions(-) delete mode 100644 models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql diff --git a/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql deleted file mode 100644 index ecbbc7c2..00000000 --- a/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql +++ /dev/null @@ -1,37 +0,0 @@ -{{ - config( - alias='igp_m_mes', - schema='br_fgv_igp', - materialized='incremental', - partition_by = { - "field": "ano", - "data_type": "int64", - "range": { - "start": 1989, - "end": 2024, - "interval": 1} - }, - post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter - ON {{this}} - GRANT TO ("allUsers") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter - ON {{this}} - GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] - ) -}} -SELECT -SAFE_CAST(ano AS INT64) ano, -SAFE_CAST(mes AS INT64) mes, -SAFE_CAST(indice AS FLOAT64) indice, -SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, -SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, -SAFE_CAST(variacao_primeiro_decendio AS FLOAT64) variacao_primeiro_decendio, -SAFE_CAST(variacao_segundo_decendio AS FLOAT64) variacao_segundo_decendio, -SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, -SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal -FROM basedosdados-dev.br_fgv_igp_staging.igp_m_mes AS t -{% if is_incremental() %} -WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) -{% endif %} \ No newline at end of file From 97d242df3c77de5d35ae1ae75ff754b799d641d9 Mon Sep 17 00:00:00 2001 From: Gabriel Pisa <61624649+folhesgabriel@users.noreply.github.com> Date: Fri, 22 Sep 2023 08:37:32 -0300 Subject: [PATCH 6/7] Delete models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql --- models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql | 35 -------------------- 1 file changed, 35 deletions(-) delete mode 100644 models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql diff --git a/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql deleted file mode 100644 index 3a9cc70d..00000000 --- a/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql +++ /dev/null @@ -1,35 +0,0 @@ -{{ - config( - alias='igp_og_mes', - schema='br_fgv_igp', - materialized='incremental', - partition_by = { - "field": "ano", - "data_type": "int64", - "range": { - "start": 1969, - "end": 2024, - "interval": 1} - }, - post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter - ON {{this}} - GRANT TO ("allUsers") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)', - 'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter - ON {{this}} - GRANT TO ("group:bd-pro@basedosdados.org", "group:sudo@basedosdados.org") - FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)'] - ) -}} -SELECT -SAFE_CAST(ano AS INT64) ano, -SAFE_CAST(mes AS INT64) mes, -SAFE_CAST(indice AS FLOAT64) indice, -SAFE_CAST(variacao_mensal AS FLOAT64) var_mensal, -SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, -SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, -SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal -FROM basedosdados-dev.br_fgv_igp_staging.igp_og_mes AS t -{% if is_incremental() %} -WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} ) -{% endif %} \ No newline at end of file From 94e14083d974e037b7c529306db1624cf1c79391 Mon Sep 17 00:00:00 2001 From: folhesgabriel Date: Fri, 22 Sep 2023 08:50:04 -0300 Subject: [PATCH 7/7] tchau tchau igp --- models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql | 15 +++++++++++++++ ...p_di_ano.sql => br_fgv_igp__igp_di_ano.sql} | 6 ++++++ models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql | 15 +++++++++++++++ ...igp_m_ano.sql => br_fgv_igp__igp_m_ano.sql} | 6 ++++++ models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql | 18 ++++++++++++++++++ ...p_og_ano.sql => br_fgv_igp__igp_og_ano.sql} | 6 ++++++ models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql | 16 ++++++++++++++++ 7 files changed, 82 insertions(+) create mode 100644 models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql rename models/br_fgv_igp/{igp_di_ano.sql => br_fgv_igp__igp_di_ano.sql} (76%) create mode 100644 models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql rename models/br_fgv_igp/{igp_m_ano.sql => br_fgv_igp__igp_m_ano.sql} (76%) create mode 100644 models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql rename models/br_fgv_igp/{igp_og_ano.sql => br_fgv_igp__igp_og_ano.sql} (76%) create mode 100644 models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql diff --git a/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql new file mode 100644 index 00000000..0e9e5216 --- /dev/null +++ b/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql @@ -0,0 +1,15 @@ +{{ + config( + alias = 'igp_10_mes', + schema = 'br_fgv_igp' + ) +}} +SELECT +SAFE_CAST(ano AS INT64) ano, +SAFE_CAST(mes AS INT64) mes, +SAFE_CAST(indice AS FLOAT64) indice, +SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, +SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, +SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, +SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal +FROM basedosdados-dev.br_fgv_igp_staging.igp_10_mes AS t \ No newline at end of file diff --git a/models/br_fgv_igp/igp_di_ano.sql b/models/br_fgv_igp/br_fgv_igp__igp_di_ano.sql similarity index 76% rename from models/br_fgv_igp/igp_di_ano.sql rename to models/br_fgv_igp/br_fgv_igp__igp_di_ano.sql index 628e304f..c9916cef 100644 --- a/models/br_fgv_igp/igp_di_ano.sql +++ b/models/br_fgv_igp/br_fgv_igp__igp_di_ano.sql @@ -1,3 +1,9 @@ +{{ + config( + alias = "igp_di_ano", + schema = 'br_fgv_igp' + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(indice_medio AS FLOAT64) indice_medio, diff --git a/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql new file mode 100644 index 00000000..0e9e5216 --- /dev/null +++ b/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql @@ -0,0 +1,15 @@ +{{ + config( + alias = 'igp_10_mes', + schema = 'br_fgv_igp' + ) +}} +SELECT +SAFE_CAST(ano AS INT64) ano, +SAFE_CAST(mes AS INT64) mes, +SAFE_CAST(indice AS FLOAT64) indice, +SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, +SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, +SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, +SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal +FROM basedosdados-dev.br_fgv_igp_staging.igp_10_mes AS t \ No newline at end of file diff --git a/models/br_fgv_igp/igp_m_ano.sql b/models/br_fgv_igp/br_fgv_igp__igp_m_ano.sql similarity index 76% rename from models/br_fgv_igp/igp_m_ano.sql rename to models/br_fgv_igp/br_fgv_igp__igp_m_ano.sql index c4ddc3fa..90b18161 100644 --- a/models/br_fgv_igp/igp_m_ano.sql +++ b/models/br_fgv_igp/br_fgv_igp__igp_m_ano.sql @@ -1,3 +1,9 @@ +{{ + config( + alias = "igp_m_ano", + schema = 'br_fgv_igp' + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(indice_medio AS FLOAT64) indice_medio, diff --git a/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql new file mode 100644 index 00000000..9f2ca06f --- /dev/null +++ b/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql @@ -0,0 +1,18 @@ +{{ + config( + alias = 'igp_m_mes', + schema = 'br_fgv_igp' + ) +}} + +SELECT +SAFE_CAST(ano AS INT64) ano, +SAFE_CAST(mes AS INT64) mes, +SAFE_CAST(indice AS FLOAT64) indice, +SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal, +SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, +SAFE_CAST(variacao_primeiro_decendio AS FLOAT64) variacao_primeiro_decendio, +SAFE_CAST(variacao_segundo_decendio AS FLOAT64) variacao_segundo_decendio, +SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, +SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal +FROM basedosdados-dev.br_fgv_igp_staging.igp_m_mes AS t \ No newline at end of file diff --git a/models/br_fgv_igp/igp_og_ano.sql b/models/br_fgv_igp/br_fgv_igp__igp_og_ano.sql similarity index 76% rename from models/br_fgv_igp/igp_og_ano.sql rename to models/br_fgv_igp/br_fgv_igp__igp_og_ano.sql index e64acc9f..a0a07415 100644 --- a/models/br_fgv_igp/igp_og_ano.sql +++ b/models/br_fgv_igp/br_fgv_igp__igp_og_ano.sql @@ -1,3 +1,9 @@ +{{ + config( + alias = "igp_og_ano", + schema = 'br_fgv_igp' + ) +}} SELECT SAFE_CAST(ano AS INT64) ano, SAFE_CAST(indice_medio AS FLOAT64) indice_medio, diff --git a/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql new file mode 100644 index 00000000..a2b71d20 --- /dev/null +++ b/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql @@ -0,0 +1,16 @@ +{{ + config( + alias = 'igp_og_mes', + schema = 'br_fgv_igp' + ) +}} + +SELECT +SAFE_CAST(ano AS INT64) ano, +SAFE_CAST(mes AS INT64) mes, +SAFE_CAST(indice AS FLOAT64) indice, +SAFE_CAST(variacao_mensal AS FLOAT64) var_mensal, +SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses, +SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano, +SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal +FROM basedosdados-dev.br_fgv_igp_staging.igp_og_mes AS t \ No newline at end of file