Release/snowplow utils/0.16.6

.github/pull_request_template.md

@@ -31,7 +31,6 @@ https://docs.github.com/en/free-pro-team@latest/github/managing-your-work-on-git
 -->
 
 ## Checklist
-- [ ] 🎉 I have verified that these changes work in Redshift
 - [ ] 💣 Is your change a breaking change?
 - [ ] 📖 I have updated the CHANGELOG.md
 

.github/workflows/pr_tests.yml

@@ -61,7 +61,7 @@ jobs:
     strategy:
       matrix:
         dbt_version: ["1.*"]
-        warehouse: ["postgres", "bigquery", "snowflake", "databricks"] # TODO: Add RS self-hosted runner
+        warehouse: ["postgres", "bigquery", "snowflake", "databricks", "redshift"] # TODO: Add RS self-hosted runner
     services:
       postgres:
         image: postgres:latest

CHANGELOG

@@ -1,3 +1,14 @@
+snowplow-utils 0.16.6 (2024-05-24)
+---------------------------------------
+## Summary
+This release adds support for a list of unique keys when using the snowplow optimized incremental approach with the delete+insert strategy.
+
+## Features
+- Support list of unique keys when using delete+insert strategy with snowplow optimize
+
+## Upgrading
+To upgrade, bump the package version in your `packages.yml` file.
+
 snowplow-utils 0.16.5 (2024-05-09)
 ---------------------------------------
 ## Summary

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'snowplow_utils'
-version: '0.16.5'
+version: '0.16.6'
 config-version: 2
 
 require-dbt-version: [">=1.4.0", "<2.0.0"]

integration_tests/data/materializations/bigquery/data_incremental_expected.csv

@@ -1,8 +1,8 @@
-id,start_tstamp
-1,2021-01-01 00:00:00
-1,2021-03-05 00:00:00
-2,2021-03-02 00:00:00
-2,2021-03-02 00:00:00
-3,2021-03-07 00:00:00
-4,2021-03-04 00:00:00
-5,2021-03-08 00:00:00
+id,id2,start_tstamp
+1,1,2021-01-01 00:00:00
+1,1,2021-03-05 00:00:00
+2,2,2021-03-02 00:00:00
+2,2,2021-03-02 00:00:00
+3,3,2021-03-07 00:00:00
+4,4,2021-03-04 00:00:00
+5,5,2021-03-08 00:00:00

integration_tests/data/materializations/data_incremental.csv

@@ -1,10 +1,10 @@
-run,id,start_tstamp
-1,1,2021-01-01 00:00:00
-1,2,2021-03-01 00:00:00
-1,2,2021-03-03 00:00:00
-1,3,2021-03-03 00:00:00
-1,4,2021-03-04 00:00:00
-2,1,2021-03-05 00:00:00
-2,2,2021-03-02 00:00:00
-2,3,2021-03-07 00:00:00
-2,5,2021-03-08 00:00:00
+run,id,id2,start_tstamp
+1,1,1,2021-01-01 00:00:00
+1,2,2,2021-03-01 00:00:00
+1,2,2,2021-03-03 00:00:00
+1,3,3,2021-03-03 00:00:00
+1,4,4,2021-03-04 00:00:00
+2,1,1,2021-03-05 00:00:00
+2,2,2,2021-03-02 00:00:00
+2,3,3,2021-03-07 00:00:00
+2,5,5,2021-03-08 00:00:00

integration_tests/data/materializations/databricks/data_incremental_expected.csv

@@ -1,8 +1,8 @@
-id,start_tstamp
-1,2021-01-01 00:00:00
-1,2021-03-05 00:00:00
-2,2021-03-02 00:00:00
-2,2021-03-02 00:00:00
-3,2021-03-07 00:00:00
-4,2021-03-04 00:00:00
-5,2021-03-08 00:00:00
+id,id2,start_tstamp
+1,1,2021-01-01 00:00:00
+1,1,2021-03-05 00:00:00
+2,2,2021-03-02 00:00:00
+2,2,2021-03-02 00:00:00
+3,3,2021-03-07 00:00:00
+4,4,2021-03-04 00:00:00
+5,5,2021-03-08 00:00:00

integration_tests/data/materializations/default/data_incremental_expected.csv

@@ -1,7 +1,7 @@
-id,start_tstamp
-1,2021-01-01 00:00:00
-4,2021-03-04 00:00:00
-5,2021-03-08 00:00:00
-1,2021-03-05 00:00:00
-2,2021-03-02 00:00:00
-3,2021-03-07 00:00:00
+id,id2,start_tstamp
+1,1,2021-01-01 00:00:00
+4,4,2021-03-04 00:00:00
+5,5,2021-03-08 00:00:00
+1,1,2021-03-05 00:00:00
+2,2,2021-03-02 00:00:00
+3,3,2021-03-07 00:00:00

integration_tests/data/materializations/snowflake/data_incremental_delete_insert_expected.csv

@@ -1,7 +1,7 @@
-id,start_tstamp
-1,2021-01-01 00:00:00
-4,2021-03-04 00:00:00
-5,2021-03-08 00:00:00
-1,2021-03-05 00:00:00
-2,2021-03-02 00:00:00
-3,2021-03-07 00:00:00
+id,id2,start_tstamp
+1,1,2021-01-01 00:00:00
+4,4,2021-03-04 00:00:00
+5,5,2021-03-08 00:00:00
+1,1,2021-03-05 00:00:00
+2,2,2021-03-02 00:00:00
+3,3,2021-03-07 00:00:00

integration_tests/data/materializations/snowflake/data_incremental_expected.csv

@@ -1,7 +1,7 @@
-id,start_tstamp
-1,2021-01-01 00:00:00
-1,2021-03-05 00:00:00
-2,2021-03-02 00:00:00
-3,2021-03-07 00:00:00
-4,2021-03-04 00:00:00
-5,2021-03-08 00:00:00
+id,id2,start_tstamp
+1,1,2021-01-01 00:00:00
+1,1,2021-03-05 00:00:00
+2,2,2021-03-02 00:00:00
+3,3,2021-03-07 00:00:00
+4,4,2021-03-04 00:00:00
+5,5,2021-03-08 00:00:00

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'snowplow_utils_integration_tests'
-version: '0.16.5'
+version: '0.16.6'
 config-version: 2
 
 profile: 'integration_tests'

integration_tests/models/materializations/default_strategy/test_incremental.sql

@@ -12,7 +12,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
 {{
   config(
     materialized='incremental',
-    unique_key='id',
+    unique_key=['id', 'id2'],
     upsert_date_key='start_tstamp',
     partition_by = snowplow_utils.get_value_by_target_type(bigquery_val={
       "field": "start_tstamp",
@@ -36,6 +36,7 @@ with data as (
 
   select
     id,
+    id2,
     start_tstamp
 
   from data
@@ -45,6 +46,7 @@ with data as (
 
   select
     id,
+    id2,
     start_tstamp
 
   from data

integration_tests/models/materializations/snowflake_delete_insert/test_incremental_delete_insert.sql

@@ -13,7 +13,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
   config(
     materialized='incremental',
     incremental_strategy='delete+insert',
-    unique_key='id',
+    unique_key=['id', 'id2'],
     upsert_date_key='start_tstamp',
     tags=["requires_script"],
     snowplow_optimize=true
@@ -28,6 +28,7 @@ with data as (
 
   select
     id,
+    id2,
     start_tstamp
 
   from data
@@ -37,6 +38,7 @@ with data as (
 
   select
     id,
+    id2,
     start_tstamp
 
   from data

macros/materializations/base_incremental/common/get_merge_sql.sql

@@ -99,7 +99,7 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
         {% set lower_limit, upper_limit = limits[0], limits[1] %}
         -- use those calculated min + max values to limit 'target' scan, to only the days with new data
         {% set predicate_override %}
-            {{ date_column }} between '{{ lower_limit }}' and '{{ upper_limit }}'
+            {{target_tb}}.{{ date_column }} between '{{ lower_limit }}' and '{{ upper_limit }}'
         {% endset %}
     {% endif %}
     {# Combine predicates with user provided ones #}