diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index d10228c8..48c0913c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -31,7 +31,6 @@ https://docs.github.com/en/free-pro-team@latest/github/managing-your-work-on-git --> ## Checklist -- [ ] 🎉 I have verified that these changes work in Redshift - [ ] 💣 Is your change a breaking change? - [ ] 📖 I have updated the CHANGELOG.md diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index 7709606f..5d276d25 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -61,7 +61,7 @@ jobs: strategy: matrix: dbt_version: ["1.*"] - warehouse: ["postgres", "bigquery", "snowflake", "databricks"] # TODO: Add RS self-hosted runner + warehouse: ["postgres", "bigquery", "snowflake", "databricks", "redshift"] # TODO: Add RS self-hosted runner services: postgres: image: postgres:latest diff --git a/CHANGELOG b/CHANGELOG index 5aa8061c..8afef319 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,14 @@ +snowplow-utils 0.16.6 (2024-05-24) +--------------------------------------- +## Summary +This release adds support for a list of unique keys when using the snowplow optimized incremental approach with the delete+insert strategy. + +## Features +- Support list of unique keys when using delete+insert strategy with snowplow optimize + +## Upgrading +To upgrade, bump the package version in your `packages.yml` file. + snowplow-utils 0.16.5 (2024-05-09) --------------------------------------- ## Summary diff --git a/dbt_project.yml b/dbt_project.yml index a03e5190..16bfc99d 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'snowplow_utils' -version: '0.16.5' +version: '0.16.6' config-version: 2 require-dbt-version: [">=1.4.0", "<2.0.0"] diff --git a/integration_tests/data/materializations/bigquery/data_incremental_expected.csv b/integration_tests/data/materializations/bigquery/data_incremental_expected.csv index e90c4830..5bad3085 100644 --- a/integration_tests/data/materializations/bigquery/data_incremental_expected.csv +++ b/integration_tests/data/materializations/bigquery/data_incremental_expected.csv @@ -1,8 +1,8 @@ -id,start_tstamp -1,2021-01-01 00:00:00 -1,2021-03-05 00:00:00 -2,2021-03-02 00:00:00 -2,2021-03-02 00:00:00 -3,2021-03-07 00:00:00 -4,2021-03-04 00:00:00 -5,2021-03-08 00:00:00 +id,id2,start_tstamp +1,1,2021-01-01 00:00:00 +1,1,2021-03-05 00:00:00 +2,2,2021-03-02 00:00:00 +2,2,2021-03-02 00:00:00 +3,3,2021-03-07 00:00:00 +4,4,2021-03-04 00:00:00 +5,5,2021-03-08 00:00:00 diff --git a/integration_tests/data/materializations/data_incremental.csv b/integration_tests/data/materializations/data_incremental.csv index d11ba2ba..851501be 100644 --- a/integration_tests/data/materializations/data_incremental.csv +++ b/integration_tests/data/materializations/data_incremental.csv @@ -1,10 +1,10 @@ -run,id,start_tstamp -1,1,2021-01-01 00:00:00 -1,2,2021-03-01 00:00:00 -1,2,2021-03-03 00:00:00 -1,3,2021-03-03 00:00:00 -1,4,2021-03-04 00:00:00 -2,1,2021-03-05 00:00:00 -2,2,2021-03-02 00:00:00 -2,3,2021-03-07 00:00:00 -2,5,2021-03-08 00:00:00 +run,id,id2,start_tstamp +1,1,1,2021-01-01 00:00:00 +1,2,2,2021-03-01 00:00:00 +1,2,2,2021-03-03 00:00:00 +1,3,3,2021-03-03 00:00:00 +1,4,4,2021-03-04 00:00:00 +2,1,1,2021-03-05 00:00:00 +2,2,2,2021-03-02 00:00:00 +2,3,3,2021-03-07 00:00:00 +2,5,5,2021-03-08 00:00:00 diff --git a/integration_tests/data/materializations/databricks/data_incremental_expected.csv b/integration_tests/data/materializations/databricks/data_incremental_expected.csv index e90c4830..5bad3085 100644 --- a/integration_tests/data/materializations/databricks/data_incremental_expected.csv +++ b/integration_tests/data/materializations/databricks/data_incremental_expected.csv @@ -1,8 +1,8 @@ -id,start_tstamp -1,2021-01-01 00:00:00 -1,2021-03-05 00:00:00 -2,2021-03-02 00:00:00 -2,2021-03-02 00:00:00 -3,2021-03-07 00:00:00 -4,2021-03-04 00:00:00 -5,2021-03-08 00:00:00 +id,id2,start_tstamp +1,1,2021-01-01 00:00:00 +1,1,2021-03-05 00:00:00 +2,2,2021-03-02 00:00:00 +2,2,2021-03-02 00:00:00 +3,3,2021-03-07 00:00:00 +4,4,2021-03-04 00:00:00 +5,5,2021-03-08 00:00:00 diff --git a/integration_tests/data/materializations/default/data_incremental_expected.csv b/integration_tests/data/materializations/default/data_incremental_expected.csv index d350319d..2c1eb458 100644 --- a/integration_tests/data/materializations/default/data_incremental_expected.csv +++ b/integration_tests/data/materializations/default/data_incremental_expected.csv @@ -1,7 +1,7 @@ -id,start_tstamp -1,2021-01-01 00:00:00 -4,2021-03-04 00:00:00 -5,2021-03-08 00:00:00 -1,2021-03-05 00:00:00 -2,2021-03-02 00:00:00 -3,2021-03-07 00:00:00 +id,id2,start_tstamp +1,1,2021-01-01 00:00:00 +4,4,2021-03-04 00:00:00 +5,5,2021-03-08 00:00:00 +1,1,2021-03-05 00:00:00 +2,2,2021-03-02 00:00:00 +3,3,2021-03-07 00:00:00 diff --git a/integration_tests/data/materializations/snowflake/data_incremental_delete_insert_expected.csv b/integration_tests/data/materializations/snowflake/data_incremental_delete_insert_expected.csv index d350319d..2c1eb458 100644 --- a/integration_tests/data/materializations/snowflake/data_incremental_delete_insert_expected.csv +++ b/integration_tests/data/materializations/snowflake/data_incremental_delete_insert_expected.csv @@ -1,7 +1,7 @@ -id,start_tstamp -1,2021-01-01 00:00:00 -4,2021-03-04 00:00:00 -5,2021-03-08 00:00:00 -1,2021-03-05 00:00:00 -2,2021-03-02 00:00:00 -3,2021-03-07 00:00:00 +id,id2,start_tstamp +1,1,2021-01-01 00:00:00 +4,4,2021-03-04 00:00:00 +5,5,2021-03-08 00:00:00 +1,1,2021-03-05 00:00:00 +2,2,2021-03-02 00:00:00 +3,3,2021-03-07 00:00:00 diff --git a/integration_tests/data/materializations/snowflake/data_incremental_expected.csv b/integration_tests/data/materializations/snowflake/data_incremental_expected.csv index 1fa3b843..3173b9b6 100644 --- a/integration_tests/data/materializations/snowflake/data_incremental_expected.csv +++ b/integration_tests/data/materializations/snowflake/data_incremental_expected.csv @@ -1,7 +1,7 @@ -id,start_tstamp -1,2021-01-01 00:00:00 -1,2021-03-05 00:00:00 -2,2021-03-02 00:00:00 -3,2021-03-07 00:00:00 -4,2021-03-04 00:00:00 -5,2021-03-08 00:00:00 +id,id2,start_tstamp +1,1,2021-01-01 00:00:00 +1,1,2021-03-05 00:00:00 +2,2,2021-03-02 00:00:00 +3,3,2021-03-07 00:00:00 +4,4,2021-03-04 00:00:00 +5,5,2021-03-08 00:00:00 diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 6b7307a9..e320d892 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'snowplow_utils_integration_tests' -version: '0.16.5' +version: '0.16.6' config-version: 2 profile: 'integration_tests' diff --git a/integration_tests/models/materializations/default_strategy/test_incremental.sql b/integration_tests/models/materializations/default_strategy/test_incremental.sql index 01ff137f..9b08b3dc 100644 --- a/integration_tests/models/materializations/default_strategy/test_incremental.sql +++ b/integration_tests/models/materializations/default_strategy/test_incremental.sql @@ -12,7 +12,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0 {{ config( materialized='incremental', - unique_key='id', + unique_key=['id', 'id2'], upsert_date_key='start_tstamp', partition_by = snowplow_utils.get_value_by_target_type(bigquery_val={ "field": "start_tstamp", @@ -36,6 +36,7 @@ with data as ( select id, + id2, start_tstamp from data @@ -45,6 +46,7 @@ with data as ( select id, + id2, start_tstamp from data diff --git a/integration_tests/models/materializations/snowflake_delete_insert/test_incremental_delete_insert.sql b/integration_tests/models/materializations/snowflake_delete_insert/test_incremental_delete_insert.sql index 3c55c04c..d405d09b 100644 --- a/integration_tests/models/materializations/snowflake_delete_insert/test_incremental_delete_insert.sql +++ b/integration_tests/models/materializations/snowflake_delete_insert/test_incremental_delete_insert.sql @@ -13,7 +13,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0 config( materialized='incremental', incremental_strategy='delete+insert', - unique_key='id', + unique_key=['id', 'id2'], upsert_date_key='start_tstamp', tags=["requires_script"], snowplow_optimize=true @@ -28,6 +28,7 @@ with data as ( select id, + id2, start_tstamp from data @@ -37,6 +38,7 @@ with data as ( select id, + id2, start_tstamp from data diff --git a/macros/materializations/base_incremental/common/get_merge_sql.sql b/macros/materializations/base_incremental/common/get_merge_sql.sql index 33689e93..23145616 100644 --- a/macros/materializations/base_incremental/common/get_merge_sql.sql +++ b/macros/materializations/base_incremental/common/get_merge_sql.sql @@ -99,7 +99,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0 {% set lower_limit, upper_limit = limits[0], limits[1] %} -- use those calculated min + max values to limit 'target' scan, to only the days with new data {% set predicate_override %} - {{ date_column }} between '{{ lower_limit }}' and '{{ upper_limit }}' + {{target_tb}}.{{ date_column }} between '{{ lower_limit }}' and '{{ upper_limit }}' {% endset %} {% endif %} {# Combine predicates with user provided ones #}