adiciona ibge_ipca15, ibge_inpc e fgv_igp

basedosdados · Sep 21, 2023 · e03257d · e03257d
1 parent 59faec6
commit e03257d
Show file tree

Hide file tree

Showing 17 changed files with 385 additions and 72 deletions.
diff --git a/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql
@@ -0,0 +1,36 @@
+{{ 
+    config(
+        alias='igp_10_mes', 
+        schema='br_fgv_igp',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 1993,
+        "end": 2024,
+        "interval": 1}
+     },
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
+SELECT
+SAFE_CAST(ano AS INT64) ano,
+SAFE_CAST(mes AS INT64) mes,
+SAFE_CAST(indice AS FLOAT64) indice,
+SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal,
+SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses,
+SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano,
+SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal
+FROM basedosdados-dev.br_fgv_igp_staging.igp_10_mes AS t
+FROM basedosdados-dev.br_ibge_inpc_staging.mes_brasil AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql
@@ -0,0 +1,35 @@
+{{ 
+    config(
+        alias='igp_di_mes', 
+        schema='br_fgv_igp',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 1944,
+        "end": 2024,
+        "interval": 1}
+     },
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
+SELECT
+SAFE_CAST(ano AS INT64) ano,
+SAFE_CAST(mes AS INT64) mes,
+SAFE_CAST(indice AS FLOAT64) indice,
+SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal,
+SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses,
+SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano,
+SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal
+FROM basedosdados-dev.br_fgv_igp_staging.igp_di_mes AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql
@@ -0,0 +1,37 @@
+{{ 
+    config(
+        alias='igp_m_mes', 
+        schema='br_fgv_igp',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 1989,
+        "end": 2024,
+        "interval": 1}
+     },
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
+SELECT
+SAFE_CAST(ano AS INT64) ano,
+SAFE_CAST(mes AS INT64) mes,
+SAFE_CAST(indice AS FLOAT64) indice,
+SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal,
+SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses,
+SAFE_CAST(variacao_primeiro_decendio AS FLOAT64) variacao_primeiro_decendio,
+SAFE_CAST(variacao_segundo_decendio AS FLOAT64) variacao_segundo_decendio,
+SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano,
+SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal
+FROM basedosdados-dev.br_fgv_igp_staging.igp_m_mes AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql b/models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql
@@ -0,0 +1,35 @@
+{{ 
+    config(
+        alias='igp_og_mes', 
+        schema='br_fgv_igp',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 1969,
+        "end": 2024,
+        "interval": 1}
+     },
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
+SELECT
+SAFE_CAST(ano AS INT64) ano,
+SAFE_CAST(mes AS INT64) mes,
+SAFE_CAST(indice AS FLOAT64) indice,
+SAFE_CAST(variacao_mensal AS FLOAT64) var_mensal,
+SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses,
+SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano,
+SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal
+FROM basedosdados-dev.br_fgv_igp_staging.igp_og_mes AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_fgv_igp/igp_10_mes.sql b/models/br_fgv_igp/igp_10_mes.sql
diff --git a/models/br_fgv_igp/igp_di_mes.sql b/models/br_fgv_igp/igp_di_mes.sql
diff --git a/models/br_fgv_igp/igp_m_mes.sql b/models/br_fgv_igp/igp_m_mes.sql
diff --git a/models/br_fgv_igp/igp_og_mes.sql b/models/br_fgv_igp/igp_og_mes.sql
diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql
@@ -1,4 +1,27 @@
-{{ config(alias='mes_brasil', schema='br_ibge_inpc') }}
+{{ 
+    config(
+        alias='mes_brasil', 
+        schema='br_ibge_inpc',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 1979,
+        "end": 2024,
+        "interval": 1}
+     },
+    pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
 SELECT 
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,
@@ -9,3 +32,6 @@ SAFE_CAST(variacao_semestral AS FLOAT64) variacao_semestral,
 SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual,
 SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses
 FROM basedosdados-dev.br_ibge_inpc_staging.mes_brasil AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql
@@ -1,4 +1,27 @@
-{{ config(alias='mes_categoria_brasil', schema='br_ibge_inpc') }}
+{{ 
+    config(
+        alias='mes_categoria_brasil', 
+        schema='br_ibge_inpc',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 2000,
+        "end": 2024,
+        "interval": 1}
+     },
+    pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
 SELECT 
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,
@@ -10,3 +33,6 @@ SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal,
 SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual,
 SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses
 FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_brasil AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql
@@ -1,4 +1,27 @@
-{{ config(alias='mes_categoria_municipio', schema='br_ibge_inpc') }}
+{{ 
+    config(
+        alias='mes_categoria_municipio', 
+        schema='br_ibge_inpc',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 2000,
+        "end": 2024,
+        "interval": 1}
+     },
+    pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
 SELECT 
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,
@@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal,
 SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal,
 SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual,
 SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses
-FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_municipio AS t
+FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_municipio AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql
@@ -1,4 +1,27 @@
-{{ config(alias='mes_categoria_rm', schema='br_ibge_inpc') }}
+{{ 
+    config(
+        alias='mes_categoria_rm', 
+        schema='br_ibge_inpc',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 2000,
+        "end": 2024,
+        "interval": 1}
+     },
+    pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
 SELECT 
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,
@@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal,
 SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal,
 SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual,
 SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses
-FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_rm AS t
+FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_rm AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql
@@ -1,25 +1,25 @@
 {{ 
-    config(
-        alias='mes_categoria_rm', 
-        schema='br_ibge_ipca',
-        materialized='incremental',
-    partition_by = {
-      "field": "ano",
-      "data_type": "int64",
-      "range": {
-        "start": 2020,
-        "end": 2024,
-        "interval": 1}
-     },
-    pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
-                    ON {{this}}
-                    GRANT TO ("allUsers")
-                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
-              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
-                    ON  {{this}}
-                    GRANT TO ("group:[email protected]", "group:[email protected]")
-                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  config(
+      alias='mes_categoria_rm', 
+      schema='br_ibge_ipca',
+      materialized='incremental',
+  partition_by = {
+    "field": "ano",
+    "data_type": "int64",
+    "range": {
+      "start": 2020,
+      "end": 2024,
+      "interval": 1}
+    },
+  pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+  post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                  ON {{this}}
+                  GRANT TO ("allUsers")
+              FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                  ON  {{this}}
+                  GRANT TO ("group:[email protected]", "group:[email protected]")
+                  FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
   )
 }}