Merge pull request #306 from basedosdados/fix_ibge

[dbt] br_ibge_
basedosdados · Sep 22, 2023 · cfea4e2 · cfea4e2
2 parents 92fa2d1 + 94e1408
commit cfea4e2
Show file tree

Hide file tree

Showing 19 changed files with 379 additions and 21 deletions.
diff --git a/models/br_fgv_igp/igp_10_mes.sql → models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql b/models/br_fgv_igp/igp_10_mes.sql → models/br_fgv_igp/br_fgv_igp__igp_10_mes.sql
@@ -1,3 +1,9 @@
+{{
+    config(
+        alias = 'igp_10_mes',
+        schema = 'br_fgv_igp'
+    )
+}}
 SELECT
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,

diff --git a/models/br_fgv_igp/igp_di_ano.sql → models/br_fgv_igp/br_fgv_igp__igp_di_ano.sql b/models/br_fgv_igp/igp_di_ano.sql → models/br_fgv_igp/br_fgv_igp__igp_di_ano.sql
@@ -1,3 +1,9 @@
+{{
+    config(
+        alias = "igp_di_ano",
+        schema  = 'br_fgv_igp'       
+    )
+}}
 SELECT
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(indice_medio AS FLOAT64) indice_medio,

diff --git a/models/br_fgv_igp/igp_di_mes.sql → models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql b/models/br_fgv_igp/igp_di_mes.sql → models/br_fgv_igp/br_fgv_igp__igp_di_mes.sql
@@ -1,3 +1,9 @@
+{{
+    config(
+        alias = 'igp_10_mes',
+        schema = 'br_fgv_igp'
+    )
+}}
 SELECT
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,
@@ -6,4 +12,4 @@ SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal,
 SAFE_CAST(variacao_12_meses AS FLOAT64) variacao_12_meses,
 SAFE_CAST(variacao_acumulada_ano AS FLOAT64) variacao_acumulada_ano,
 SAFE_CAST(indice_fechamento_mensal AS FLOAT64) indice_fechamento_mensal
-FROM basedosdados-dev.br_fgv_igp_staging.igp_di_mes AS t
+FROM basedosdados-dev.br_fgv_igp_staging.igp_10_mes AS t
diff --git a/models/br_fgv_igp/igp_m_ano.sql → models/br_fgv_igp/br_fgv_igp__igp_m_ano.sql b/models/br_fgv_igp/igp_m_ano.sql → models/br_fgv_igp/br_fgv_igp__igp_m_ano.sql
@@ -1,3 +1,9 @@
+{{
+    config(
+        alias = "igp_m_ano",
+        schema  = 'br_fgv_igp'       
+    )
+}}
 SELECT
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(indice_medio AS FLOAT64) indice_medio,

diff --git a/models/br_fgv_igp/igp_m_mes.sql → models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql b/models/br_fgv_igp/igp_m_mes.sql → models/br_fgv_igp/br_fgv_igp__igp_m_mes.sql
@@ -1,3 +1,10 @@
+{{
+    config(
+        alias = 'igp_m_mes',
+        schema = 'br_fgv_igp'
+    )
+}}
+
 SELECT
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,

diff --git a/models/br_fgv_igp/igp_og_ano.sql → models/br_fgv_igp/br_fgv_igp__igp_og_ano.sql b/models/br_fgv_igp/igp_og_ano.sql → models/br_fgv_igp/br_fgv_igp__igp_og_ano.sql
@@ -1,3 +1,9 @@
+{{
+    config(
+        alias = "igp_og_ano",
+        schema  = 'br_fgv_igp'       
+    )
+}}
 SELECT
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(indice_medio AS FLOAT64) indice_medio,

diff --git a/models/br_fgv_igp/igp_og_mes.sql → models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql b/models/br_fgv_igp/igp_og_mes.sql → models/br_fgv_igp/br_fgv_igp__igp_og_mes.sql
@@ -1,3 +1,10 @@
+{{
+    config(
+        alias = 'igp_og_mes',
+        schema = 'br_fgv_igp'
+    )
+}}
+
 SELECT
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,

diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_brasil.sql
@@ -1,4 +1,27 @@
-{{ config(alias='mes_brasil', schema='br_ibge_inpc') }}
+{{ 
+    config(
+        alias='mes_brasil', 
+        schema='br_ibge_inpc',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 1979,
+        "end": 2024,
+        "interval": 1}
+     },
+    pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
 SELECT 
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,
@@ -9,3 +32,6 @@ SAFE_CAST(variacao_semestral AS FLOAT64) variacao_semestral,
 SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual,
 SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses
 FROM basedosdados-dev.br_ibge_inpc_staging.mes_brasil AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_brasil.sql
@@ -1,4 +1,27 @@
-{{ config(alias='mes_categoria_brasil', schema='br_ibge_inpc') }}
+{{ 
+    config(
+        alias='mes_categoria_brasil', 
+        schema='br_ibge_inpc',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 2000,
+        "end": 2024,
+        "interval": 1}
+     },
+    pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
 SELECT 
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,
@@ -10,3 +33,6 @@ SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal,
 SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual,
 SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses
 FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_brasil AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_municipio.sql
@@ -1,4 +1,27 @@
-{{ config(alias='mes_categoria_municipio', schema='br_ibge_inpc') }}
+{{ 
+    config(
+        alias='mes_categoria_municipio', 
+        schema='br_ibge_inpc',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 2000,
+        "end": 2024,
+        "interval": 1}
+     },
+    pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
 SELECT 
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,
@@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal,
 SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal,
 SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual,
 SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses
-FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_municipio AS t
+FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_municipio AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql b/models/br_ibge_inpc/br_ibge_inpc__mes_categoria_rm.sql
@@ -1,4 +1,27 @@
-{{ config(alias='mes_categoria_rm', schema='br_ibge_inpc') }}
+{{ 
+    config(
+        alias='mes_categoria_rm', 
+        schema='br_ibge_inpc',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 2000,
+        "end": 2024,
+        "interval": 1}
+     },
+    pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
 SELECT 
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,
@@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal,
 SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal,
 SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual,
 SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses
-FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_rm AS t
+FROM basedosdados-dev.br_ibge_inpc_staging.mes_categoria_rm AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_brasil.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_brasil.sql
@@ -1,4 +1,27 @@
-{{ config(alias='mes_brasil', schema='br_ibge_ipca') }}
+{{ 
+    config(
+        alias='mes_brasil', 
+        schema='br_ibge_ipca',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 1979,
+        "end": 2024,
+        "interval": 1}
+     },
+    pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
 SELECT 
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,
@@ -9,3 +32,7 @@ SAFE_CAST(variacao_semestral AS FLOAT64) variacao_semestral,
 SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual,
 SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses
 FROM basedosdados-dev.br_ibge_ipca_staging.mes_brasil AS t
+
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_brasil.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_brasil.sql
@@ -1,4 +1,27 @@
-{{ config(alias='mes_categoria_brasil', schema='br_ibge_ipca') }}
+{{ 
+    config(
+        alias='mes_categoria_brasil', 
+        schema='br_ibge_ipca',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 2020,
+        "end": 2024,
+        "interval": 1}
+     },
+    pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
 SELECT 
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,
@@ -9,4 +32,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal,
 SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal,
 SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual,
 SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses
-FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_brasil AS t
+FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_brasil AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_municipio.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_municipio.sql
@@ -1,4 +1,27 @@
-{{ config(alias='mes_categoria_municipio', schema='br_ibge_ipca') }}
+{{ 
+    config(
+        alias='mes_categoria_municipio', 
+        schema='br_ibge_ipca',
+        materialized='incremental',
+    partition_by = {
+      "field": "ano",
+      "data_type": "int64",
+      "range": {
+        "start": 2020,
+        "end": 2024,
+        "interval": 1}
+     },
+    pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+    post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                    ON {{this}}
+                    GRANT TO ("allUsers")
+                FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+              'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                    ON  {{this}}
+                    GRANT TO ("group:[email protected]", "group:[email protected]")
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
 SELECT 
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,
@@ -10,4 +33,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal,
 SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal,
 SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual,
 SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses
-FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_municipio AS t
+FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_municipio AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}
diff --git a/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql b/models/br_ibge_ipca/br_ibge_ipca__mes_categoria_rm.sql
@@ -1,4 +1,28 @@
-{{ config(alias='mes_categoria_rm', schema='br_ibge_ipca') }}
+{{ 
+  config(
+      alias='mes_categoria_rm', 
+      schema='br_ibge_ipca',
+      materialized='incremental',
+  partition_by = {
+    "field": "ano",
+    "data_type": "int64",
+    "range": {
+      "start": 2020,
+      "end": 2024,
+      "interval": 1}
+    },
+  pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+  post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
+                  ON {{this}}
+                  GRANT TO ("allUsers")
+              FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) > 6)',
+            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
+                  ON  {{this}}
+                  GRANT TO ("group:[email protected]", "group:[email protected]")
+                  FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano AS INT64),CAST(mes AS INT64),1), MONTH) <= 6)']
+  )
+}}
+
 SELECT 
 SAFE_CAST(ano AS INT64) ano,
 SAFE_CAST(mes AS INT64) mes,
@@ -10,4 +34,7 @@ SAFE_CAST(peso_mensal AS FLOAT64) peso_mensal,
 SAFE_CAST(variacao_mensal AS FLOAT64) variacao_mensal,
 SAFE_CAST(variacao_anual AS FLOAT64) variacao_anual,
 SAFE_CAST(variacao_doze_meses AS FLOAT64) variacao_doze_meses
-FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_rm AS t
+FROM basedosdados-dev.br_ibge_ipca_staging.mes_categoria_rm AS t
+{% if is_incremental() %}
+WHERE DATE(CAST(ano AS INT64),CAST(mes AS INT64),1) > (SELECT MAX(DATE(CAST(ano AS INT64),CAST(mes AS INT64),1)) FROM {{ this }} )
+{% endif %}