Merge pull request #544 from basedosdados/fix-cgu-models

[fix]: fix join operation on `br_cgu_beneficios_cidadao`
basedosdados · Apr 16, 2024 · f7d159b · f7d159b
2 parents 4c27602 + 7d97990
commit f7d159b
Show file tree

Hide file tree

Showing 3 changed files with 105 additions and 72 deletions.
diff --git a/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__bpc.sql b/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__bpc.sql
@@ -16,31 +16,42 @@
         ],
     )
 }}
-with
-    bpc as (
-        select
-            safe_cast(substr(mes_competencia, 1, 4) as int64) ano_competencia,
-            safe_cast(substr(mes_competencia, 5, 2) as int64) mes_competencia,
-            safe_cast(substr(mes_referencia, 1, 4) as int64) ano_referencia,
-            safe_cast(substr(mes_referencia, 5, 2) as int64) mes_referencia,
-            safe_cast(parse_date('%Y%m', mes_referencia) as date) data,
-            t2.id_municipio,
-            safe_cast(t1.sigla_uf as string) sigla_uf,
-            safe_cast(nis as string) nis_favorecido,
-            safe_cast(cpf as string) cpf_favorecido,
-            safe_cast(t1.nome as string) nome_favorecido,
-            safe_cast(nis_representante as string) nis_representante,
-            safe_cast(cpf_representante as string) cpf_representante,
-            safe_cast(t1.nome_representante as string) nome_representante,
-            safe_cast(numero as string) numero_beneficio,
-            safe_cast(concedido_judicialmente as string) concedido_judicialmente,
-            safe_cast(valor as float64) valor_parcela,
-        from `basedosdados-staging.br_cgu_beneficios_cidadao_staging.bpc` t1
-        left join
-            `basedosdados.br_bd_diretorios_brasil.municipio` t2
-            on safe_cast(t1.id_municipio_siafi as int64)
-            = safe_cast(t2.id_municipio_rf as int64)
-    )
-select * except (data)
-from bpc
-{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}
+select
+    safe_cast(substr(mes_competencia, 1, 4) as int64) ano_competencia,
+    safe_cast(substr(mes_competencia, 5, 2) as int64) mes_competencia,
+    safe_cast(substr(mes_referencia, 1, 4) as int64) ano_referencia,
+    safe_cast(substr(mes_referencia, 5, 2) as int64) mes_referencia,
+    t2.id_municipio,
+    safe_cast(t1.sigla_uf as string) sigla_uf,
+    safe_cast(nis as string) nis_favorecido,
+    safe_cast(cpf as string) cpf_favorecido,
+    safe_cast(t1.nome as string) nome_favorecido,
+    safe_cast(nis_representante as string) nis_representante,
+    safe_cast(cpf_representante as string) cpf_representante,
+    safe_cast(nome_representante as string) nome_representante,
+    safe_cast(numero as string) numero_beneficio,
+    safe_cast(concedido_judicialmente as string) concedido_judicialmente,
+    safe_cast(valor as float64) valor_parcela,
+from `basedosdados-staging.br_cgu_beneficios_cidadao_staging.bpc` t1
+left join
+    `basedosdados.br_bd_diretorios_brasil.municipio` t2
+    on safe_cast(t1.id_municipio_siafi as int64)
+    = safe_cast(t2.id_municipio_rf as int64)
+{% if is_incremental() %}
+    where
+        safe_cast(parse_date('%Y%m', mes_referencia) as date) > (
+            select
+                max(
+                    safe_cast(
+                        parse_date(
+                            '%Y%m',
+                            concat(
+                                cast(ano_referencia as string),
+                                lpad(cast(mes_referencia as string), 2, '0')
+                            )
+                        ) as date
+                    )
+                )
+            from {{ this }}
+        )
+{% endif %}
diff --git a/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__garantia_safra.sql b/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__garantia_safra.sql
@@ -20,23 +20,35 @@
         ],
     )
 }}
-with
-    garantia_safra as (
-        select
-            safe_cast(substr(mes_referencia, 1, 4) as int64) ano_referencia,
-            safe_cast(substr(mes_referencia, 5, 2) as int64) mes_referencia,
-            safe_cast(parse_date('%Y%m', mes_referencia) as date) data,
-            t2.id_municipio,
-            safe_cast(t1.sigla_uf as string) sigla_uf,
-            safe_cast(nis as string) nis_favorecido,
-            safe_cast(t1.nome as string) nome_favorecido,
-            safe_cast(valor as float64) valor_parcela,
-        from `basedosdados-staging.br_cgu_beneficios_cidadao_staging.garantia_safra` t1
-        left join
-            `basedosdados.br_bd_diretorios_brasil.municipio` t2
-            on safe_cast(t1.id_municipio_siafi as int64)
-            = safe_cast(t2.id_municipio_rf as int64)
-    )
-select * except (data)
-from garantia_safra
-{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}
+
+select
+    safe_cast(substr(mes_referencia, 1, 4) as int64) ano_referencia,
+    safe_cast(substr(mes_referencia, 5, 2) as int64) mes_referencia,
+    t2.id_municipio,
+    safe_cast(t1.sigla_uf as string) sigla_uf,
+    safe_cast(nis as string) nis_favorecido,
+    safe_cast(t1.nome as string) nome_favorecido,
+    safe_cast(valor as float64) valor_parcela,
+from `basedosdados-staging.br_cgu_beneficios_cidadao_staging.garantia_safra` t1
+left join
+    `basedosdados.br_bd_diretorios_brasil.municipio` t2
+    on safe_cast(t1.id_municipio_siafi as int64)
+    = safe_cast(t2.id_municipio_rf as int64)
+{% if is_incremental() %}
+    where
+        safe_cast(parse_date('%Y%m', mes_referencia) as date) > (
+            select
+                max(
+                    safe_cast(
+                        parse_date(
+                            '%Y%m',
+                            concat(
+                                cast(ano_referencia as string),
+                                lpad(cast(mes_referencia as string), 2, '0')
+                            )
+                        ) as date
+                    )
+                )
+            from {{ this }}
+        )
+{% endif %}
diff --git a/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__novo_bolsa_familia.sql b/models/br_cgu_beneficios_cidadao/br_cgu_beneficios_cidadao__novo_bolsa_familia.sql
@@ -16,27 +16,37 @@
         ],
     )
 }}
-with
-    novo_bolsa_familia as (
-        select
-            safe_cast(substr(mes_competencia, 1, 4) as int64) ano_competencia,
-            safe_cast(substr(mes_competencia, 5, 2) as int64) mes_competencia,
-            safe_cast(substr(mes_referencia, 1, 4) as int64) ano_referencia,
-            safe_cast(substr(mes_referencia, 5, 2) as int64) mes_referencia,
-            safe_cast(parse_date('%Y%m', mes_referencia) as date) data,
-            t2.id_municipio,
-            safe_cast(t1.sigla_uf as string) sigla_uf,
-            safe_cast(cpf as string) cpf_favorecido,
-            safe_cast(nis as string) nis_favorecido,
-            safe_cast(t1.nome as string) nome_favorecido,
-            safe_cast(valor as float64) valor_parcela,
-        from
-            `basedosdados-staging.br_cgu_beneficios_cidadao_staging.novo_bolsa_familia` t1
-        left join
-            `basedosdados.br_bd_diretorios_brasil.municipio` t2
-            on safe_cast(t1.id_municipio_siafi as int64)
-            = safe_cast(t2.id_municipio_rf as int64)
-    )
-select * except (data)
-from novo_bolsa_familia
-{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}
+select
+    safe_cast(substr(mes_competencia, 1, 4) as int64) ano_competencia,
+    safe_cast(substr(mes_competencia, 5, 2) as int64) mes_competencia,
+    safe_cast(substr(mes_referencia, 1, 4) as int64) ano_referencia,
+    safe_cast(substr(mes_referencia, 5, 2) as int64) mes_referencia,
+    t2.id_municipio,
+    safe_cast(t1.sigla_uf as string) sigla_uf,
+    safe_cast(cpf as string) cpf_favorecido,
+    safe_cast(nis as string) nis_favorecido,
+    safe_cast(t1.nome as string) nome_favorecido,
+    safe_cast(valor as float64) valor_parcela,
+from `basedosdados-dev.br_cgu_beneficios_cidadao_staging.novo_bolsa_familia` t1
+left join
+    `basedosdados.br_bd_diretorios_brasil.municipio` t2
+    on safe_cast(t1.id_municipio_siafi as int64)
+    = safe_cast(t2.id_municipio_rf as int64)
+{% if is_incremental() %}
+    where
+        safe_cast(parse_date('%Y%m', mes_referencia) as date) > (
+            select
+                max(
+                    safe_cast(
+                        parse_date(
+                            '%Y%m',
+                            concat(
+                                cast(ano_referencia as string),
+                                lpad(cast(mes_referencia as string), 2, '0')
+                            )
+                        ) as date
+                    )
+                )
+            from {{ this }}
+        )
+{% endif %}