From 0312abbc7d66618dcd5c2d1d30ed5b660a2f337e Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Mon, 10 Jun 2024 12:36:57 +0100 Subject: [PATCH 1/3] Extend functionality of combine column versions --- .../bigquery/data_combine_column_versions.sql | 6 +++++- .../expected_combine_column_versions.sql | 3 +++ .../bigquery/test_combine_column_versions.sql | 8 +++++++- .../exclude_column_versions.sql | 16 +++++++++++++--- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/integration_tests/models/utils/bigquery/data_combine_column_versions.sql b/integration_tests/models/utils/bigquery/data_combine_column_versions.sql index c142993f..39085b1c 100644 --- a/integration_tests/models/utils/bigquery/data_combine_column_versions.sql +++ b/integration_tests/models/utils/bigquery/data_combine_column_versions.sql @@ -44,7 +44,11 @@ with data as ( struct('Charlie' as f_name, 31 as numeric_col) as person_1_0_2, struct('Dog the bounty' as f_name, 50 as numeric_col) as person_1_1_0, struct('Dog the bounty' as f_name, 50 as numeric_col) as person_1_1_30, - struct('Dog the bounty' as f_name, 50 as numeric_col) as person_1_1_31 + struct('Dog the bounty' as f_name, 50 as numeric_col) as person_1_1_31, + struct('a' as field_1, 2 as field_2) as test_1_0_0, + struct('b' as field_1, 3 as field_2) as test_1_1_0, + struct('b' as field_1, 3 as field_2) as test_1_1_15, + struct('c' as field_1, 4 as field_2) as test_new_1_0_0 ) diff --git a/integration_tests/models/utils/bigquery/expected_combine_column_versions.sql b/integration_tests/models/utils/bigquery/expected_combine_column_versions.sql index 53376eb8..d0939e85 100644 --- a/integration_tests/models/utils/bigquery/expected_combine_column_versions.sql +++ b/integration_tests/models/utils/bigquery/expected_combine_column_versions.sql @@ -24,6 +24,9 @@ with prep as ( -- Test 5 coalesce(person_1_1_31.f_name, person_1_1_0.f_name, person_1_0_0.f_name) as f_name, coalesce(person_1_1_31.numeric_col, person_1_1_0.numeric_col, person_1_0_0.numeric_col) as numeric_col, + -- Test 6 + coalesce(test_1_1_15.field_1, test_1_0_0.field_1) as field_1, + coalesce(test_1_1_15.field_2, test_1_0_0.field_2) as field_2 from {{ ref('data_combine_column_versions') }} ) diff --git a/integration_tests/models/utils/bigquery/test_combine_column_versions.sql b/integration_tests/models/utils/bigquery/test_combine_column_versions.sql index c5dd95f3..949d5ceb 100644 --- a/integration_tests/models/utils/bigquery/test_combine_column_versions.sql +++ b/integration_tests/models/utils/bigquery/test_combine_column_versions.sql @@ -39,6 +39,10 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0 column_prefix='person_1', exclude_versions=['1_0_1', '1_0_2', '1_1_30']) %} +{% set test_6_actual = snowplow_utils.combine_column_versions(relation=ref('data_combine_column_versions'), + column_prefix='test', + exclude_versions=['new_1_0_0', '1_1_0']) %} + with prep as ( select -- Test 1 @@ -50,7 +54,9 @@ with prep as ( -- Test 4 {{ test_4_actual|join(',') }} as product_volume, -- Test 5 - {{ test_5_actual|join(',') }} + {{ test_5_actual|join(',') }}, + -- Test 6 + {{ test_6_actual|join(',') }} from {{ ref('data_combine_column_versions') }} a ) diff --git a/macros/utils/bigquery/combine_column_versions/exclude_column_versions.sql b/macros/utils/bigquery/combine_column_versions/exclude_column_versions.sql index 99f7a147..d76955ca 100644 --- a/macros/utils/bigquery/combine_column_versions/exclude_column_versions.sql +++ b/macros/utils/bigquery/combine_column_versions/exclude_column_versions.sql @@ -5,20 +5,30 @@ and you may not use this file except in compliance with the Snowplow Personal an You may obtain a copy of the Snowplow Personal and Academic License Version 1.0 at https://docs.snowplow.io/personal-and-academic-license-1.0/ #} {% macro exclude_column_versions(columns, exclude_versions) %} + {# + columns: an array of column names + exclude_versions: an array of versions (more generically a suffix) to exclude columns that end with this value + returns: filtered array of columns that do not end with any of the exclude versions. + #} {% if not exclude_versions %} {{ return(columns) }} {% endif %} {%- set filtered_columns_by_version = [] -%} + + {%- set columns_to_exclude = [] -%} + {% for column in columns %} {# Remove columns that end with the version we want to exclude #} {% for version in exclude_versions %} - {% if not column.name.endswith(version) %} - {% do filtered_columns_by_version.append(column) %} + {% if column.name.endswith(version) %} + {% do columns_to_exclude.append(column) %} {% endif %} {% endfor %} + {% if not column in columns_to_exclude %} + {% do filtered_columns_by_version.append(column) %} + {% endif %} {% endfor %} - {{ return(filtered_columns_by_version) }} {% endmacro %} From b7d4849f99504a10540b8f23aad824e6107c3464 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Mon, 10 Jun 2024 12:47:43 +0100 Subject: [PATCH 2/3] Fix comparison tests --- integration_tests/models/utils/cross_db/cross_db.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/models/utils/cross_db/cross_db.yml b/integration_tests/models/utils/cross_db/cross_db.yml index a8b93f87..baf03371 100644 --- a/integration_tests/models/utils/cross_db/cross_db.yml +++ b/integration_tests/models/utils/cross_db/cross_db.yml @@ -23,3 +23,4 @@ models: tests: - dbt_utils.equality: compare_model: ref('expected_indexed_unnest') + exclude_columns: ['source_index'] From e2a88ed493a74cb4fafcdefbeba4544e5d700684 Mon Sep 17 00:00:00 2001 From: Ryan Hill Date: Mon, 10 Jun 2024 12:38:30 +0100 Subject: [PATCH 3/3] Prepare for release --- CHANGELOG | 14 ++++++++++++++ dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 8afef319..71c94d51 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,17 @@ +snowplow-utils 0.16.7 (2024-06-10) +--------------------------------------- +## Summary +This release extends support for the combine_column_versions macro to support column prefixes not ending in a major version number (when used in combination with exclude column versions). + +## Features +- Extend functionality of combine_column_versions + +## Under the hood +- Update some of our test to pass with newer versions of dbt utils + +## Upgrading +To upgrade, bump the package version in your `packages.yml` file. + snowplow-utils 0.16.6 (2024-05-24) --------------------------------------- ## Summary diff --git a/dbt_project.yml b/dbt_project.yml index 16bfc99d..a22aa82b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'snowplow_utils' -version: '0.16.6' +version: '0.16.7' config-version: 2 require-dbt-version: [">=1.4.0", "<2.0.0"] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index e320d892..e9a2e850 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'snowplow_utils_integration_tests' -version: '0.16.6' +version: '0.16.7' config-version: 2 profile: 'integration_tests'