Refactor: Modifica estratégia de materialização incremental

basedosdados · Dec 18, 2024 · 418e985 · 418e985
1 parent 27920ea
commit 418e985
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 16 deletions.
diff --git a/models/br_me_cnpj/br_me_cnpj__empresas.sql b/models/br_me_cnpj/br_me_cnpj__empresas.sql
@@ -1,18 +1,13 @@
 {{
     config(
         schema="br_me_cnpj",
-        materialized="incremental",
         alias="empresas",
-        unique_key="data",
+        materialized="incremental",
         partition_by={
             "field": "data",
             "data_type": "date",
         },
-        incremental_strategy="insert_overwrite",
-        pre_hook=[
-            "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-            "DELETE FROM {{ this }}         WHERE data > '2024-11-15'",
-        ],
+        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
     )
 }}
 
@@ -32,4 +27,4 @@ with
     )
 select *
 from cnpj_empresas
-{% if is_incremental() %} where data = '2024-11-15' {% endif %}
+{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}
diff --git a/models/br_me_cnpj/br_me_cnpj__estabelecimentos.sql b/models/br_me_cnpj/br_me_cnpj__estabelecimentos.sql
@@ -1,18 +1,13 @@
 {{
     config(
         schema="br_me_cnpj",
-        materialized="incremental",
         alias="estabelecimentos",
+        materialized="incremental",
         partition_by={
             "field": "data",
             "data_type": "date",
         },
-        cluster_by="sigla_uf",
-        incremental_strategy="insert_overwrite",
-        pre_hook=[
-            "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
-            "DELETE FROM {{ this }}         WHERE data > '2024-11-15'",
-        ],
+        pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}",
     )
 }}
 with
@@ -63,4 +58,4 @@ with
     )
 select *
 from cnpj_estabelecimentos
-{% if is_incremental() %} where data = '2024-11-15' {% endif %}
+{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %}