Merge pull request #454 from basedosdados/br_cgu_beneficios_cidadao

[dbt] br_cgu_beneficios_cidadao
basedosdados · Mar 13, 2024 · 1badaa0 · 1badaa0
2 parents f460b57 + 0734278
commit 1badaa0
Show file tree

Hide file tree

Showing 6 changed files with 351 additions and 27 deletions.
diff --git a/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__auxilio_brasil.sql b/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__auxilio_brasil.sql
@@ -0,0 +1,30 @@
+{{
+    config(
+        alias="auxilio_brasil",
+        schema="br_cgu_beneficios_cidadao",
+        materialized="table",
+        partition_by={
+            "field": "ano_competencia",
+            "data_type": "int64",
+            "range": {
+                "start": 2020,
+                "end": 2023,
+                "interval": 1,
+            },
+        },
+        cluster_by=["sigla_uf", "id_municipio"],
+    )
+}}
+
+select distinct
+    safe_cast(split(mes_ref, '-')[offset(0)] as int64) as ano_competencia,
+    safe_cast(split(mes_ref, '-')[offset(1)] as int64) as mes_competencia,
+    safe_cast(left(mes, 4) as int64) ano_referencia,
+    safe_cast(right(mes, 2) as int64) mes_referencia,
+    safe_cast(id_municipio as string) id_municipio,
+    safe_cast(sigla_uf as string) sigla_uf,
+    case when cpf = '' then null else cpf end as cpf_favorecido,
+    safe_cast(nis as string) nis_favorecido,
+    safe_cast(nome as string) nome_favorecido,
+    safe_cast(valor_beneficio as float64) valor_parcela,
+from `basedosdados-dev.br_cgu_beneficios_cidadao_staging.auxilio_brasil` as t
diff --git a/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__bolsa_familia_pagamento.sql b/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__bolsa_familia_pagamento.sql
@@ -0,0 +1,30 @@
+{{
+    config(
+        alias="bolsa_familia_pagamento",
+        schema="br_cgu_beneficios_cidadao",
+        materialized="table",
+        partition_by={
+            "field": "ano_competencia",
+            "data_type": "int64",
+            "range": {
+                "start": 2020,
+                "end": 2023,
+                "interval": 1,
+            },
+        },
+        cluster_by=["sigla_uf", "id_municipio"],
+    )
+}}
+
+select distinct
+    safe_cast(split(mes_ref, '-')[offset(0)] as int64) as ano_competencia,
+    safe_cast(split(mes_ref, '-')[offset(1)] as int64) as mes_competencia,
+    safe_cast(left(mes, 4) as int64) ano_referencia,
+    safe_cast(right(mes, 2) as int64) mes_referencia,
+    safe_cast(id_municipio as string) id_municipio,
+    safe_cast(sigla_uf as string) sigla_uf,
+    case when cpf = '' then null else cpf end as cpf_favorecido,
+    safe_cast(nis as string) nis_favorecido,
+    safe_cast(nome as string) nome_favorecido,
+    safe_cast(valor_beneficio as float64) valor_parcela,
+from `basedosdados-dev.br_cgu_beneficios_cidadao_staging.bolsa_familia_pagamento` as t
diff --git a/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__bpc.sql b/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__bpc.sql
@@ -2,24 +2,29 @@
     config(
         alias="bpc",
         schema="br_cgu_beneficios_cidadao",
-        materialized="incremental",
+        materialized="table",
         partition_by={
-            "field": "ano",
+            "field": "ano_competencia",
             "data_type": "int64",
-            "range": {"start": 2023, "end": 2024, "interval": 1},
+            "range": {"start": 2019, "end": 2024, "interval": 1},
         },
-        cluster_by=["mes", "sigla_uf"],
+        cluster_by=["mes_competencia", "sigla_uf"],
+        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+        post_hook=[
+            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano_competencia AS INT64),CAST(mes_competencia AS INT64),1), MONTH) > 6)',
+            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano_competencia AS INT64),CAST(mes_competencia AS INT64),1), MONTH) <= 6)',
+        ],
     )
 }}
 with
     bpc as (
         select
-            safe_cast(substr(mes_competencia, 1, 4) as int64) ano,
-            safe_cast(substr(mes_competencia, 5, 2) as int64) mes,
-            safe_cast(mes_referencia as string) data_referencia,
+            safe_cast(substr(mes_competencia, 1, 4) as int64) ano_competencia,
+            safe_cast(substr(mes_competencia, 5, 2) as int64) mes_competencia,
+            safe_cast(substr(mes_referencia, 1, 4) as int64) ano_referencia,
+            safe_cast(substr(mes_referencia, 5, 2) as int64) mes_referencia,
             safe_cast(parse_date('%Y%m', mes_referencia) as date) data,
             t2.id_municipio,
-            t2.nome as nome_municipio,
             safe_cast(t1.sigla_uf as string) sigla_uf,
             safe_cast(nis as string) nis_favorecido,
             safe_cast(cpf as string) cpf_favorecido,
@@ -38,4 +43,3 @@ with
     )
 select * except (data)
 from bpc
-{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}
diff --git a/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__garantia_safra.sql b/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__garantia_safra.sql
@@ -2,24 +2,31 @@
     config(
         alias="garantia_safra",
         schema="br_cgu_beneficios_cidadao",
-        materialized="incremental",
+        materialized="table",
         partition_by={
-            "field": "ano",
+            "field": "ano_referencia",
             "data_type": "int64",
-            "range": {"start": 2023, "end": 2024, "interval": 1},
+            "range": {
+                "start": 2013,
+                "end": 2024,
+                "interval": 1,
+            },
         },
-        cluster_by=["mes", "sigla_uf"],
+        cluster_by=["mes_referencia", "sigla_uf"],
+        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+        post_hook=[
+            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano_referencia AS INT64),CAST(mes_referencia AS INT64),1), MONTH) > 6)',
+            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano_referencia AS INT64),CAST(mes_referencia AS INT64),1), MONTH) <= 6)',
+        ],
     )
 }}
 with
     garantia_safra as (
         select
-            safe_cast(substr(mes_referencia, 1, 4) as int64) ano,
-            safe_cast(substr(mes_referencia, 5, 2) as int64) mes,
-            safe_cast(mes_referencia as string) data_referencia,
+            safe_cast(substr(mes_referencia, 1, 4) as int64) ano_referencia,
+            safe_cast(substr(mes_referencia, 5, 2) as int64) mes_referencia,
             safe_cast(parse_date('%Y%m', mes_referencia) as date) data,
             t2.id_municipio,
-            t2.nome as nome_municipio,
             safe_cast(t1.sigla_uf as string) sigla_uf,
             safe_cast(nis as string) nis_favorecido,
             safe_cast(t1.nome as string) nome_favorecido,
@@ -32,4 +39,3 @@ with
     )
 select * except (data)
 from garantia_safra
-{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}
diff --git a/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__novo_bolsa_familia.sql b/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__novo_bolsa_familia.sql
@@ -2,24 +2,29 @@
     config(
         alias="novo_bolsa_familia",
         schema="br_cgu_beneficios_cidadao",
-        materialized="incremental",
+        materialized="table",
         partition_by={
-            "field": "ano",
+            "field": "ano_competencia",
             "data_type": "int64",
             "range": {"start": 2023, "end": 2024, "interval": 1},
         },
-        cluster_by=["mes", "sigla_uf"],
+        cluster_by=["mes_competencia", "sigla_uf"],
+        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+        post_hook=[
+            'CREATE OR REPLACE ROW ACCESS POLICY allusers_filter ON {{this}} GRANT TO ("allUsers") FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano_competencia AS INT64),CAST(mes_competencia AS INT64),1), MONTH) > 6)',
+            'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter ON {{this}} GRANT TO ("group:[email protected]", "group:[email protected]") FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(CAST(ano_competencia AS INT64),CAST(mes_competencia AS INT64),1), MONTH) <= 6)',
+        ],
     )
 }}
 with
     novo_bolsa_familia as (
-        select
-            safe_cast(substr(mes_competencia, 1, 4) as int64) ano,
-            safe_cast(substr(mes_competencia, 5, 2) as int64) mes,
-            safe_cast(mes_referencia as string) data_referencia,
+        select distinct
+            safe_cast(substr(mes_competencia, 1, 4) as int64) ano_competencia,
+            safe_cast(substr(mes_competencia, 5, 2) as int64) mes_competencia,
+            safe_cast(substr(mes_referencia, 1, 4) as int64) ano_referencia,
+            safe_cast(substr(mes_referencia, 5, 2) as int64) mes_referencia,
             safe_cast(parse_date('%Y%m', mes_referencia) as date) data,
             t2.id_municipio,
-            t2.nome as nome_municipio,
             safe_cast(t1.sigla_uf as string) sigla_uf,
             safe_cast(cpf as string) cpf_favorecido,
             safe_cast(nis as string) nis_favorecido,
@@ -33,4 +38,3 @@ with
     )
 select * except (data)
 from novo_bolsa_familia
-{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}