From dd6e63f88e39bc5a5b85cdb1251af7f9b4e3d229 Mon Sep 17 00:00:00 2001 From: Oscar Date: Tue, 8 Oct 2024 16:41:06 +0100 Subject: [PATCH] Rework on return_limits_from_model macro ------------------ Co-authored-by: Agnes Kiss --- .../models/utils/cross_db/cross_db.yml | 2 -- macros/utils/return_limits_from_model.sql | 13 +++++++++---- macros/utils/schema.yml | 3 +++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/integration_tests/models/utils/cross_db/cross_db.yml b/integration_tests/models/utils/cross_db/cross_db.yml index baf03371..b94a5275 100644 --- a/integration_tests/models/utils/cross_db/cross_db.yml +++ b/integration_tests/models/utils/cross_db/cross_db.yml @@ -17,8 +17,6 @@ models: tests: - dbt_utils.equality: compare_model: ref('expected_get_field_bq') - config: - +enabled: "{{ target.type in ['bigquery'] | as_bool() }}" - name: test_indexed_unnest tests: - dbt_utils.equality: diff --git a/macros/utils/return_limits_from_model.sql b/macros/utils/return_limits_from_model.sql index 09e06bf0..7de61939 100644 --- a/macros/utils/return_limits_from_model.sql +++ b/macros/utils/return_limits_from_model.sql @@ -4,7 +4,7 @@ This program is licensed to you under the Snowplow Personal and Academic License and you may not use this file except in compliance with the Snowplow Personal and Academic License Version 1.0. You may obtain a copy of the Snowplow Personal and Academic License Version 1.0 at https://docs.snowplow.io/personal-and-academic-license-1.0/ #} -{% macro return_limits_from_model(model, lower_limit_col, upper_limit_col) -%} +{% macro return_limits_from_model(model, lower_limit_col, upper_limit_col, lower_output=False) -%} {# In case of not execute just return empty strings to avoid hitting database #} {% if not execute %} @@ -35,9 +35,14 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0 {{ snowplow_utils.log_message("Snowplow Warning: *************") }} {% do exceptions.warn("Snowplow Warning: No data in "~this~" for date range from variables, please modify your run variables to include data if this is not expected.") %} {{ snowplow_utils.log_message("Snowplow Warning: *************") }} - {# This allows for bigquery to still run the same way the other warehouses do, but also ensures no data is processed #} - {% set lower_limit = snowplow_utils.cast_to_tstamp('9999-01-01 00:00:00') %} - {% set upper_limit = snowplow_utils.cast_to_tstamp('9999-01-02 00:00:00') %} + {% if lower_output %} + {% set lower_limit = snowplow_utils.cast_to_tstamp('0000-01-01 00:00:00') %} + {% set upper_limit = snowplow_utils.cast_to_tstamp('0000-01-02 00:00:00') %} + {%- else -%} + {# Default behaviour for incrementalization. This allows for bigquery to still run the same way the other warehouses do, but also ensures no data is processed #} + {% set lower_limit = snowplow_utils.cast_to_tstamp('9999-01-01 00:00:00') %} + {% set upper_limit = snowplow_utils.cast_to_tstamp('9999-01-02 00:00:00') %} + {% endif %} {%- else -%} {% set lower_limit = snowplow_utils.cast_to_tstamp(results.columns[0].values()[0]) %} {% set upper_limit = snowplow_utils.cast_to_tstamp(results.columns[1].values()[0]) %} diff --git a/macros/utils/schema.yml b/macros/utils/schema.yml index b28a7604..80c17a1e 100644 --- a/macros/utils/schema.yml +++ b/macros/utils/schema.yml @@ -92,6 +92,9 @@ macros: - name: upper_limit_col type: string description: The column to take the `max` of to get the upper limit + - name: lower_output + type: boolean + description: This changes the output to a low set date if true and high set date if false, in the event the function cannot find value but needs to set a value - name: set_query_tag description: '{{ doc("macro_set_query_tag") }}' arguments: