From 9637dacb4c68461354545b6f0133313f47075416 Mon Sep 17 00:00:00 2001 From: uiro-bi Date: Mon, 18 Nov 2024 22:32:55 -0300 Subject: [PATCH 1/2] fix: br_me_cnpj partitions --- models/br_me_cnpj/br_me_cnpj__empresas.sql | 8 ++++++-- models/br_me_cnpj/br_me_cnpj__estabelecimentos.sql | 8 ++++++-- models/br_me_cnpj/br_me_cnpj__socios.sql | 8 ++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/models/br_me_cnpj/br_me_cnpj__empresas.sql b/models/br_me_cnpj/br_me_cnpj__empresas.sql index 4b860e69..5741fc40 100644 --- a/models/br_me_cnpj/br_me_cnpj__empresas.sql +++ b/models/br_me_cnpj/br_me_cnpj__empresas.sql @@ -8,7 +8,11 @@ "field": "data", "data_type": "date", }, - pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", + incremental_strategy="insert_overwrite", + pre_hook=[ + "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + "DELETE FROM {{ this }} WHERE data > '2024-11-15'", + ], ) }} with @@ -27,4 +31,4 @@ with ) select * from cnpj_empresas -{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %} +{% if is_incremental() %} where data = '2024-11-15' {% endif %} diff --git a/models/br_me_cnpj/br_me_cnpj__estabelecimentos.sql b/models/br_me_cnpj/br_me_cnpj__estabelecimentos.sql index ec08e88a..5e62422a 100644 --- a/models/br_me_cnpj/br_me_cnpj__estabelecimentos.sql +++ b/models/br_me_cnpj/br_me_cnpj__estabelecimentos.sql @@ -8,7 +8,11 @@ "data_type": "date", }, cluster_by="sigla_uf", - pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", + incremental_strategy="insert_overwrite", + pre_hook=[ + "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + "DELETE FROM {{ this }} WHERE data > '2024-11-15'", + ], ) }} with @@ -59,4 +63,4 @@ with ) select * from cnpj_estabelecimentos -{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %} +{% if is_incremental() %} where data > '2024-11-15' {% endif %} diff --git a/models/br_me_cnpj/br_me_cnpj__socios.sql b/models/br_me_cnpj/br_me_cnpj__socios.sql index cc0f5d5d..9955b852 100644 --- a/models/br_me_cnpj/br_me_cnpj__socios.sql +++ b/models/br_me_cnpj/br_me_cnpj__socios.sql @@ -7,7 +7,11 @@ "field": "data", "data_type": "date", }, - pre_hook="DROP ALL ROW ACCESS POLICIES ON {{ this }}", + incremental_strategy="insert_overwrite", + pre_hook=[ + "DROP ALL ROW ACCESS POLICIES ON {{ this }}", + "DELETE FROM {{ this }} WHERE data > '2024-11-15'", + ], ) }} with @@ -32,4 +36,4 @@ with ) select * from cnpj_socios -{% if is_incremental() %} where data > (select max(data) from {{ this }}) {% endif %} +{% if is_incremental() %} where data > '2024-11-15' {% endif %} From ee5ff8fdcafe34d985d4db59621b49c713f966c9 Mon Sep 17 00:00:00 2001 From: uiro-bi Date: Mon, 18 Nov 2024 22:47:36 -0300 Subject: [PATCH 2/2] adjust incremental filters --- models/br_me_cnpj/br_me_cnpj__empresas.sql | 2 +- models/br_me_cnpj/br_me_cnpj__estabelecimentos.sql | 4 ++-- models/br_me_cnpj/br_me_cnpj__socios.sql | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/models/br_me_cnpj/br_me_cnpj__empresas.sql b/models/br_me_cnpj/br_me_cnpj__empresas.sql index 5741fc40..fa3edfcd 100644 --- a/models/br_me_cnpj/br_me_cnpj__empresas.sql +++ b/models/br_me_cnpj/br_me_cnpj__empresas.sql @@ -11,7 +11,7 @@ incremental_strategy="insert_overwrite", pre_hook=[ "DROP ALL ROW ACCESS POLICIES ON {{ this }}", - "DELETE FROM {{ this }} WHERE data > '2024-11-15'", + "DELETE FROM {{ this }} WHERE data > '2024-03-15'", ], ) }} diff --git a/models/br_me_cnpj/br_me_cnpj__estabelecimentos.sql b/models/br_me_cnpj/br_me_cnpj__estabelecimentos.sql index 5e62422a..049288d1 100644 --- a/models/br_me_cnpj/br_me_cnpj__estabelecimentos.sql +++ b/models/br_me_cnpj/br_me_cnpj__estabelecimentos.sql @@ -11,7 +11,7 @@ incremental_strategy="insert_overwrite", pre_hook=[ "DROP ALL ROW ACCESS POLICIES ON {{ this }}", - "DELETE FROM {{ this }} WHERE data > '2024-11-15'", + "DELETE FROM {{ this }} WHERE data > '2024-03-15'", ], ) }} @@ -63,4 +63,4 @@ with ) select * from cnpj_estabelecimentos -{% if is_incremental() %} where data > '2024-11-15' {% endif %} +{% if is_incremental() %} where data = '2024-11-15' {% endif %} diff --git a/models/br_me_cnpj/br_me_cnpj__socios.sql b/models/br_me_cnpj/br_me_cnpj__socios.sql index 9955b852..6c701c80 100644 --- a/models/br_me_cnpj/br_me_cnpj__socios.sql +++ b/models/br_me_cnpj/br_me_cnpj__socios.sql @@ -10,7 +10,7 @@ incremental_strategy="insert_overwrite", pre_hook=[ "DROP ALL ROW ACCESS POLICIES ON {{ this }}", - "DELETE FROM {{ this }} WHERE data > '2024-11-15'", + "DELETE FROM {{ this }} WHERE data > '2024-03-15'", ], ) }} @@ -36,4 +36,4 @@ with ) select * from cnpj_socios -{% if is_incremental() %} where data > '2024-11-15' {% endif %} +{% if is_incremental() %} where data = '2024-11-15' {% endif %}