Release/snowplow utils/0.16.7

CHANGELOG

@@ -1,3 +1,17 @@
+snowplow-utils 0.16.7 (2024-06-10)
+---------------------------------------
+## Summary
+This release extends support for the combine_column_versions macro to support column prefixes not ending in a major version number (when used in combination with exclude column versions).
+
+## Features
+- Extend functionality of combine_column_versions
+
+## Under the hood
+- Update some of our test to pass with newer versions of dbt utils
+
+## Upgrading
+To upgrade, bump the package version in your `packages.yml` file.
+
 snowplow-utils 0.16.6 (2024-05-24)
 ---------------------------------------
 ## Summary

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'snowplow_utils'
-version: '0.16.6'
+version: '0.16.7'
 config-version: 2
 
 require-dbt-version: [">=1.4.0", "<2.0.0"]

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'snowplow_utils_integration_tests'
-version: '0.16.6'
+version: '0.16.7'
 config-version: 2
 
 profile: 'integration_tests'

integration_tests/models/utils/bigquery/data_combine_column_versions.sql

@@ -44,7 +44,11 @@ with data as (
     struct('Charlie' as f_name, 31 as numeric_col) as person_1_0_2,
     struct('Dog the bounty' as f_name, 50 as numeric_col) as person_1_1_0,
     struct('Dog the bounty' as f_name, 50 as numeric_col) as person_1_1_30,
-    struct('Dog the bounty' as f_name, 50 as numeric_col) as person_1_1_31
+    struct('Dog the bounty' as f_name, 50 as numeric_col) as person_1_1_31,
+    struct('a' as field_1, 2 as field_2) as test_1_0_0,
+    struct('b' as field_1, 3 as field_2) as test_1_1_0,
+    struct('b' as field_1, 3 as field_2) as test_1_1_15,
+    struct('c' as field_1, 4 as field_2) as test_new_1_0_0
 
 )
 

integration_tests/models/utils/bigquery/expected_combine_column_versions.sql

@@ -24,6 +24,9 @@ with prep as (
     -- Test 5
     coalesce(person_1_1_31.f_name, person_1_1_0.f_name, person_1_0_0.f_name) as f_name,
     coalesce(person_1_1_31.numeric_col, person_1_1_0.numeric_col, person_1_0_0.numeric_col) as numeric_col,
+    -- Test 6
+    coalesce(test_1_1_15.field_1, test_1_0_0.field_1) as field_1,
+    coalesce(test_1_1_15.field_2, test_1_0_0.field_2) as field_2
 
   from {{ ref('data_combine_column_versions') }}
 )

integration_tests/models/utils/bigquery/test_combine_column_versions.sql

@@ -39,6 +39,10 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
                                                               column_prefix='person_1',
                                                               exclude_versions=['1_0_1', '1_0_2', '1_1_30']) %}
 
+{% set test_6_actual = snowplow_utils.combine_column_versions(relation=ref('data_combine_column_versions'),
+                                                              column_prefix='test',
+                                                              exclude_versions=['new_1_0_0', '1_1_0']) %}
+
 with prep as (
    select
       -- Test 1
@@ -50,7 +54,9 @@ with prep as (
       -- Test 4
       {{ test_4_actual|join(',') }} as product_volume,
       -- Test 5
-      {{ test_5_actual|join(',') }}
+      {{ test_5_actual|join(',') }},
+      -- Test 6
+      {{ test_6_actual|join(',') }}
 
    from {{ ref('data_combine_column_versions') }} a
 )

integration_tests/models/utils/cross_db/cross_db.yml

@@ -23,3 +23,4 @@ models:
     tests:
       - dbt_utils.equality:
           compare_model: ref('expected_indexed_unnest')
+          exclude_columns: ['source_index']

macros/utils/bigquery/combine_column_versions/exclude_column_versions.sql

@@ -5,20 +5,30 @@ and you may not use this file except in compliance with the Snowplow Personal an
 You may obtain a copy of the Snowplow Personal and Academic License Version 1.0 at https://docs.snowplow.io/personal-and-academic-license-1.0/
 #}
 {% macro exclude_column_versions(columns, exclude_versions) %}
+  {# 
+  columns: an array of column names
+  exclude_versions: an array of versions (more generically a suffix) to exclude columns that end with this value
+  returns: filtered array of columns that do not end with any of the exclude versions.
+  #}
   {% if not exclude_versions %}
     {{ return(columns) }}
   {% endif %}
 
   {%- set filtered_columns_by_version = [] -%}
+
+  {%- set columns_to_exclude = [] -%}
+
   {% for column in columns %}
   {# Remove columns that end with the version we want to exclude #}
     {% for version in exclude_versions %}
-      {% if not column.name.endswith(version) %}
-        {% do filtered_columns_by_version.append(column) %}
+      {% if column.name.endswith(version) %}
+        {% do columns_to_exclude.append(column) %}
       {% endif %}
     {% endfor %}
+    {% if not column in columns_to_exclude %}
+        {% do filtered_columns_by_version.append(column) %}
+    {% endif %}
   {% endfor %}
-
   {{ return(filtered_columns_by_version) }}
 
 {% endmacro %}