[dbt] br_cvm

models/br_cvm_administradores_carteira/br_cvm_administradores_carteira__pessoa_fisica.sql

@@ -16,15 +16,24 @@
               'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
                     ON  {{this}}
                     GRANT TO ("group:[email protected]", "group:[email protected]")
-                    FILTER USING (EXTRACT(YEAR from data_registro) = EXTRACT(YEAR from  DATE("{{ run_started_at.strftime("%Y-%m-%d") }}")))' ] 
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(data_registro), MONTH) <= 6)'] 
     )
  }}
-SELECT 
+WITH tabela as( SELECT 
 SAFE_CAST(nome AS STRING) nome,
 SAFE_CAST(data_registro AS DATE) data_registro,
 SAFE_CAST(data_cancelamento AS DATE) data_cancelamento,
 SAFE_CAST(motivo_cancelamento AS STRING) motivo_cancelamento,
 SAFE_CAST(situacao AS STRING) situacao,
 SAFE_CAST(data_inicio_situacao AS DATE) data_inicio_situacao,
 SAFE_CAST(categoria_registro AS STRING) categoria_registro
-FROM basedosdados-dev.br_cvm_administradores_carteira_staging.pessoa_fisica AS t
+FROM basedosdados-dev.br_cvm_administradores_carteira_staging.pessoa_fisica AS t)
+select * 
+from tabela
+{% if is_incremental() %}
+
+  -- this filter will only be applied on an incremental run
+  -- (uses > to include records whose timestamp occurred since the last run of this model)
+  where data_registro > (select max(data_registro) from {{ this }})
+
+{% endif %}

models/br_cvm_administradores_carteira/br_cvm_administradores_carteira__pessoa_juridica.sql

@@ -16,11 +16,11 @@
               'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
                     ON  {{this}}
                     GRANT TO ("group:[email protected]", "group:[email protected]")
-                    FILTER USING (EXTRACT(YEAR from data_registro) = EXTRACT(YEAR from  DATE("{{ run_started_at.strftime("%Y-%m-%d") }}")))' ] 
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(data_registro), MONTH) <= 6)' ] 
     )
  }}
 
-SELECT 
+WITH tabela as(SELECT 
 SAFE_CAST(cnpj AS STRING) cnpj,
 SAFE_CAST(denominacao_social AS STRING) denominacao_social,
 SAFE_CAST(denominacao_comercial AS STRING) denominacao_comercial,
@@ -45,4 +45,13 @@ SAFE_CAST(valor_patrimonial_liquido AS STRING) valor_patrimonial_liquido,
 SAFE_CAST(data_patrimonio_liquido AS DATE) data_patrimonio_liquido,
 SAFE_CAST(email AS STRING) email,
 SAFE_CAST(website AS STRING) website
-FROM basedosdados-dev.br_cvm_administradores_carteira_staging.pessoa_juridica AS t
+FROM basedosdados-dev.br_cvm_administradores_carteira_staging.pessoa_juridica AS t)
+select * 
+from tabela
+{% if is_incremental() %}
+
+  -- this filter will only be applied on an incremental run
+  -- (uses > to include records whose timestamp occurred since the last run of this model)
+  where data_registro > (select max(data_registro) from {{ this }})
+
+{% endif %}

models/br_cvm_administradores_carteira/br_cvm_administradores_carteira__responsavel.sql

@@ -2,7 +2,7 @@
   config(
     alias = 'responsavel',
     schema='br_cvm_administradores_carteira',
-    materialized='incremental', 
+    materialized='table', 
     )
  }}
 SELECT 

models/br_cvm_oferta_publica_distribuicao/br_cvm_oferta_publica_distribuicao__dia.sql

@@ -8,19 +8,19 @@
       "data_type": "date",
       "granularity": "day"
      },
-     pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
+    pre_hook = "DROP ALL ROW ACCESS POLICIES ON {{ this }}",
     post_hook=['CREATE OR REPLACE ROW ACCESS POLICY allusers_filter 
                     ON {{this}}
                     GRANT TO ("allUsers")
                 FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(data_abertura_processo), MONTH) > 6)',
               'CREATE OR REPLACE ROW ACCESS POLICY bdpro_filter 
                     ON  {{this}}
                     GRANT TO ("group:[email protected]", "group:[email protected]")
-                    FILTER USING (EXTRACT(YEAR from data_abertura_processo) = EXTRACT(YEAR from  DATE("{{ run_started_at.strftime("%Y-%m-%d") }}")))' ] 
-    )
+                    FILTER USING (DATE_DIFF(DATE("{{ run_started_at.strftime("%Y-%m-%d") }}"),DATE(data_abertura_processo), MONTH) <= 6)']
+  )
  }}
 
-SELECT 
+WITH tabela as(SELECT 
 SAFE_CAST(numero_processo AS STRING) numero_processo,
 SAFE_CAST(numero_registro_oferta AS STRING) numero_registro_oferta,
 SAFE_CAST(tipo_oferta AS STRING) tipo_oferta,
@@ -65,4 +65,13 @@ SAFE_CAST(tipo_societario_emissor AS STRING) tipo_societario_emissor,
 SAFE_CAST(tipo_fundo_investimento AS STRING) tipo_fundo_investimento,
 SAFE_CAST(ultimo_comunicado AS STRING) ultimo_comunicado,
 SAFE_CAST(data_comunicado AS DATE) data_comunicado
-FROM basedosdados-dev.br_cvm_oferta_publica_distribuicao_staging.dia AS t
+FROM basedosdados-dev.br_cvm_oferta_publica_distribuicao_staging.dia AS t)
+SELECT *
+FROM tabela
+{% if is_incremental() %}
+
+  -- this filter will only be applied on an incremental run
+  -- (uses > to include records whose timestamp occurred since the last run of this model)
+  where data_abertura_processo > (select max(data_abertura_processo) from {{ this }})
+
+{% endif %}