Skip to content

Commit

Permalink
Add support for browser context 2
Browse files Browse the repository at this point in the history
  • Loading branch information
@ilias1111
ilias1111 authored Oct 11, 2024
1 parent 4896f1a commit 606a8d2
Show file tree
Hide file tree
Showing 11 changed files with 294 additions and 15 deletions.
3 changes: 3 additions & 0 deletions dbt_project.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ vars:
snowplow__enable_iab: false
snowplow__enable_ua: false
snowplow__enable_browser_context: false
snowplow__enable_browser_context_2 : false
# add extra mobile fields:
snowplow__enable_mobile_context: false
snowplow__enable_geolocation_context: false
Expand Down Expand Up @@ -135,6 +136,7 @@ vars:
snowplow__yauaa_context: nl_basjes_yauaa_context_1
snowplow__cwv_events: com_snowplowanalytics_snowplow_web_vitals_1
snowplow__browser_context: com_snowplowanalytics_snowplow_browser_context_1
snowplow__browser_context_2: com_snowplowanalytics_snowplow_browser_context_2
snowplow__session_context: com_snowplowanalytics_snowplow_client_session_1
snowplow__mobile_context: com_snowplowanalytics_snowplow_mobile_context_1
snowplow__geolocation_context: com_snowplowanalytics_snowplow_geolocation_context_1
Expand All @@ -145,6 +147,7 @@ vars:
snowplow__deep_link_context: com_snowplowanalytics_mobile_deep_link_1
snowplow__screen_summary_context: com_snowplowanalytics_mobile_screen_summary_1


# Completely or partially remove models from the manifest during run start.
on-run-start:
- "{{ snowplow_utils.snowplow_delete_from_manifest(var('models_to_remove',[]), ref('snowplow_unified_incremental_manifest')) }}"
Expand Down
15 changes: 10 additions & 5 deletions integration_tests/.scripts/integration_test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ for db in ${DATABASES[@]}; do
echo "Snowplow unified integration tests: Try run without data"
eval "dbt run --full-refresh --vars '{snowplow__allow_refresh: true, snowplow__backfill_limit_days: 1, snowplow__enable_cwv: false, snowplow__start_date: 2010-01-01}' --target $db" || exit 1;

echo "Snowplow unified integration tests: Try run without data for browser_context_2 only enabled"
eval "dbt run --full-refresh --vars '{snowplow__allow_refresh: true, snowplow__enable_browser_context: false, snowplow__enable_browser_context_2:true, snowplow__backfill_limit_days: 1, snowplow__enable_cwv: false, snowplow__start_date: 2010-01-01}' --target $db" || exit 1;

echo "Snowplow unified integration tests: Try run without data for browser_context_2 and browser_context enabled"
eval "dbt run --full-refresh --vars '{snowplow__allow_refresh: true, snowplow__enable_browser_context:true, snowplow__enable_browser_context_2:true, snowplow__backfill_limit_days: 1, snowplow__enable_cwv: false, snowplow__start_date: 2010-01-01}' --target $db" || exit 1;

echo "Snowplow unified integration tests: Conversions"
eval "dbt run --full-refresh --select +snowplow_unified_conversions snowplow_unified_integration_tests.source --vars '{snowplow__allow_refresh: true, snowplow__backfill_limit_days: 220, snowplow__enable_cwv: false, snowplow__enable_conversions: true}' --target $db" || exit 1;
Expand All @@ -37,7 +42,7 @@ for db in ${DATABASES[@]}; do
eval "dbt run --full-refresh --select +snowplow_unified_app_errors snowplow_unified_integration_tests.source --vars '{snowplow__allow_refresh: true, snowplow__backfill_limit_days: 220, snowplow__enable_cwv: false, snowplow__enable_app_errors: true}' --target $db" || exit 1;

echo "Snowplow unified integration tests: Late enabled contexts"
eval "dbt run --full-refresh --select +test_late_enabled_contexts snowplow_unified_integration_tests.source --vars '{snowplow__allow_refresh: true, snowplow__backfill_limit_days: 220, snowplow__enable_cwv: false, snowplow__enable_mobile_context: false, snowplow__enable_geolocation_context: false, snowplow__enable_application_context: false, snowplow__enable_screen_context: false, snowplow__enable_app_errors: false, snowplow__enable_deep_link_context: false, snowplow__enable_cwv: false, snowplow__enable_iab: false, snowplow__enable_ua: false, snowplow__enable_browser_context: false, snowplow__enable_consent: false}' --target $db" || exit 1;
eval "dbt run --full-refresh --select +test_late_enabled_contexts snowplow_unified_integration_tests.source --vars '{snowplow__allow_refresh: true, snowplow__backfill_limit_days: 220, snowplow__enable_cwv: false, snowplow__enable_mobile_context: false, snowplow__enable_geolocation_context: false, snowplow__enable_application_context: false, snowplow__enable_screen_context: false, snowplow__enable_app_errors: false, snowplow__enable_deep_link_context: false, snowplow__enable_cwv: false, snowplow__enable_iab: false, snowplow__enable_ua: false, snowplow__enable_browser_context: false, snowplow__enable_browser_context_2: false, snowplow__enable_consent: false}' --target $db" || exit 1;

eval "dbt run --select +test_late_enabled_contexts run --vars '{snowplow__allow_refresh: true, snowplow__backfill_limit_days: 250, snowplow__enable_cwv: false}' --target $db"
echo "Snowplow unified integration tests: Late enabled contexts test passed"
Expand All @@ -58,22 +63,22 @@ for db in ${DATABASES[@]}; do
echo "Snowplow unified integration tests: All non-CWV tests passed"

echo "Snowplow unified integration tests - Core Web Vitals: Execute models"
eval "dbt run --select +snowplow_unified_web_vital_measurements_actual snowplow_unified_web_vital_measurements_expected_stg source --full-refresh --vars '{snowplow__allow_refresh: true, snowplow__start_date: '2023-03-01', snowplow__backfill_limit_days: 50, snowplow__cwv_days_to_measure: 999, snowplow__enable_mobile: false, snowplow__enable_mobile_context: false, snowplow__enable_geolocation_context: false, snowplow__enable_application_context: false, snowplow__enable_screen_context: false, snowplow__enable_app_errors: false, snowplow__enable_deep_link_context: false, snowplow__enable_ua: false, snowplow__enable_browser_context: false, snowplow__enable_consent: false}' --target $db" || exit 1;
eval "dbt run --select +snowplow_unified_web_vital_measurements_actual snowplow_unified_web_vital_measurements_expected_stg source --full-refresh --vars '{snowplow__allow_refresh: true, snowplow__start_date: '2023-03-01', snowplow__backfill_limit_days: 50, snowplow__cwv_days_to_measure: 999, snowplow__enable_mobile: false, snowplow__enable_mobile_context: false, snowplow__enable_geolocation_context: false, snowplow__enable_application_context: false, snowplow__enable_screen_context: false, snowplow__enable_app_errors: false, snowplow__enable_deep_link_context: false, snowplow__enable_ua: false, snowplow__enable_browser_context: false, snowplow__enable_browser_context_2: false, snowplow__enable_consent: false}' --target $db" || exit 1;
eval "dbt test --select snowplow_unified_web_vital_measurements_actual --store-failures --target $db" || exit 1;

echo "Snowplow unified integration tests: Execute web (all web contexts except for cwv)"
eval "dbt run --full-refresh --vars '{snowplow__allow_refresh: true, snowplow__backfill_limit_days: 9999, snowplow__enable_mobile: false, snowplow__enable_mobile_context: false, snowplow__enable_geolocation_context: false, snowplow__enable_application_context: false, snowplow__enable_screen_context: false, snowplow__enable_app_errors: false, snowplow__enable_deep_link_context: false, snowplow__enable_cwv: false}' --select +snowplow_unified_users snowplow_unified_events_stg --target $db" || exit 1;

echo "Snowplow unified integration tests: Execute mobile (all mobile contexts)"
eval "dbt run --full-refresh --vars '{snowplow__allow_refresh: true, snowplow__backfill_limit_days: 9999, snowplow__enable_web: false, snowplow__enable_iab: false, snowplow__enable_ua: false, snowplow__enable_browser_context: false, snowplow__enable_consent: false, snowplow__enable_cwv: false}' --select +snowplow_unified_users snowplow_unified_events_stg --target $db" || exit 1;
eval "dbt run --full-refresh --vars '{snowplow__allow_refresh: true, snowplow__backfill_limit_days: 9999, snowplow__enable_web: false, snowplow__enable_iab: false, snowplow__enable_ua: false, snowplow__enable_browser_context: false, snowplow__enable_browser_context_2: false, snowplow__enable_consent: false, snowplow__enable_cwv: false}' --select +snowplow_unified_users snowplow_unified_events_stg --target $db" || exit 1;

echo "Snowplow unified integration tests: All CWV tests passed"

echo "Snowplow unified integration tests: Test mobile screen engagement"

eval "dbt run --select +snowplow_unified_views_mobile_screen_engagement_actual snowplow_unified_views_mobile_screen_engagement_expected_stg source --full-refresh --vars '{snowplow__allow_refresh: true, snowplow__start_date: '2023-12-19', snowplow__backfill_limit_days: 50, snowplow__enable_cwv: false, snowplow__enable_screen_summary_context: true, snowplow__enable_ua: false, snowplow__enable_iab: false, snowplow__enable_web: false, snowplow__enable_browser_context: false, snowplow__enable_consent: false, snowplow__enable_yauaa: false, snowplow__enable_geolocation_context: false, snowplow__enable_deep_link_context: false, snowplow__enable_app_errors: false}' --target $db" || exit 1;
eval "dbt run --select +snowplow_unified_views_mobile_screen_engagement_actual snowplow_unified_views_mobile_screen_engagement_expected_stg source --full-refresh --vars '{snowplow__allow_refresh: true, snowplow__start_date: '2023-12-19', snowplow__backfill_limit_days: 50, snowplow__enable_cwv: false, snowplow__enable_screen_summary_context: true, snowplow__enable_ua: false, snowplow__enable_iab: false, snowplow__enable_web: false, snowplow__enable_browser_context: false, snowplow__enable_browser_context_2: false, snowplow__enable_consent: false, snowplow__enable_yauaa: false, snowplow__enable_geolocation_context: false, snowplow__enable_deep_link_context: false, snowplow__enable_app_errors: false}' --target $db" || exit 1;

eval "dbt test --select snowplow_unified_views_mobile_screen_engagement_actual --vars '{snowplow__enable_screen_summary_context: true, snowplow__enable_web: false, snowplow__enable_cwv: false, snowplow__enable_ua: false, snowplow__enable_iab: false, snowplow__enable_web: false, snowplow__enable_browser_context: false, snowplow__enable_consent: false, snowplow__enable_yauaa: false, snowplow__enable_geolocation_context: false, snowplow__enable_deep_link_context: false, snowplow__enable_app_errors: false}' --store-failures --target $db" || exit 1;
eval "dbt test --select snowplow_unified_views_mobile_screen_engagement_actual --vars '{snowplow__enable_screen_summary_context: true, snowplow__enable_web: false, snowplow__enable_cwv: false, snowplow__enable_ua: false, snowplow__enable_iab: false, snowplow__enable_browser_context: false, snowplow__enable_browser_context_2: false, snowplow__enable_consent: false, snowplow__enable_yauaa: false, snowplow__enable_geolocation_context: false, snowplow__enable_deep_link_context: false, snowplow__enable_app_errors: false}' --store-failures --target $db" || exit 1;

echo "Snowplow unified integration tests: Mobile screen engagement tests passed"

Expand Down
2 changes: 2 additions & 0 deletions integration_tests/dbt_project.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ vars:
snowplow__iab_context: snowplow_unified_spider_context_stg
snowplow__ua_parser_context: snowplow_unified_ua_context_stg
snowplow__browser_context: snowplow_unified_browser_context_stg
snowplow__browser_context_2: snowplow_unified_browser_context_2_stg
# mobile only contexts
snowplow__screen_view_events: snowplow_unified_screen_view_event_stg
snowplow__session_context: snowplow_unified_session_context_stg
Expand Down Expand Up @@ -85,6 +86,7 @@ vars:
snowplow__user_aggregations: [{'type': 'sum', 'field': "agg_test", 'alias': 'agg_test'}]
# add extra web fields:
snowplow__enable_browser_context: true
snowplow__enable_browser_context_2: false
# add extra mobile fields:
snowplow__enable_mobile_context: true
snowplow__enable_geolocation_context: true
Expand Down
19 changes: 19 additions & 0 deletions integration_tests/models/source/bigquery/snowplow_unified_events_stg.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,25 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
from unnest(contexts_com_snowplowanalytics_snowplow_browser_context_1_0_0) as json_array
) as contexts_com_snowplowanalytics_snowplow_browser_context_1_0_0,


array(
select as struct
JSON_EXTRACT_scalar(json_array,'$.viewport') as viewport,
JSON_EXTRACT_scalar(json_array,'$.documentSize') as document_size,
JSON_EXTRACT_scalar(json_array,'$.resolution') as resolution,
cast(JSON_EXTRACT_scalar(json_array,'$.colorDepth') as integer) as color_depth,
cast(JSON_EXTRACT_scalar(json_array,'$.devicePixelRatio') as FLOAT64) as device_pixel_ratio,
cast(JSON_EXTRACT_scalar(json_array,'$.cookiesEnabled') as boolean) as cookies_enabled,
cast(JSON_EXTRACT_scalar(json_array,'$.online') as boolean) as online,
JSON_EXTRACT_scalar(json_array,'$.browserLanguage') as browser_language,
JSON_EXTRACT_scalar(json_array,'$.documentLanguage') as document_language,
cast(JSON_EXTRACT_scalar(json_array,'$.webdriver') as boolean) as webdriver,
cast(JSON_EXTRACT_scalar(json_array,'$.deviceMemory') as integer) as device_memory,
cast(JSON_EXTRACT_scalar(json_array,'$.hardwareConcurrency') as integer) as hardware_concurrency,
JSON_EXTRACT_scalar(json_array,'$.tab_id') as tab_id
from unnest(contexts_com_snowplowanalytics_snowplow_browser_context_1_0_0) as json_array
) as contexts_com_snowplowanalytics_snowplow_browser_context_2_0_0,

array(
select as struct JSON_EXTRACT_scalar(json_array,'$.version') as version,
JSON_EXTRACT_scalar(json_array,'$.build') as build
Expand Down
14 changes: 14 additions & 0 deletions integration_tests/models/source/databricks/snowplow_unified_events_stg.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,19 @@ You may obtain a copy of the Snowplow Personal and Academic License Version 1.0
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].deviceMemory::int AS device_memory,
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].hardwareConcurrency::int AS hardware_concurrency,
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].tabId::string AS tab_id)) as contexts_com_snowplowanalytics_snowplow_browser_context_1,
array(struct(contexts_com_snowplowanalytics_snowplow_browser_context_1[0].viewport::string AS viewport,
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].documentSize::string AS document_size,
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].resolution::string AS resolution,
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].colorDepth::int AS color_depth,
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].devicePixelRatio::double AS device_pixel_ratio,
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].cookiesEnabled::boolean AS cookies_enabled,
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].online::boolean AS online,
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].browserLanguage::string AS browser_language,
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].documentLanguage::string AS document_language,
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].webdriver::boolean AS webdriver,
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].deviceMemory::int AS device_memory,
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].hardwareConcurrency::int AS hardware_concurrency,
contexts_com_snowplowanalytics_snowplow_browser_context_1[0].tabId::string AS tab_id)) as contexts_com_snowplowanalytics_snowplow_browser_context_2,
array(struct(contexts_com_snowplowanalytics_snowplow_mobile_context_1[0].deviceManufacturer::string AS device_manufacturer,
contexts_com_snowplowanalytics_snowplow_mobile_context_1[0].deviceModel::string AS device_model,
contexts_com_snowplowanalytics_snowplow_mobile_context_1[0].osType::string AS os_type,
Expand Down Expand Up @@ -552,6 +565,7 @@ select
NULL as contexts_com_snowplowanalytics_mobile_application_1,
NULL as contexts_com_snowplowanalytics_mobile_deep_link_1,
NULL as contexts_com_snowplowanalytics_snowplow_browser_context_1,
NULL as contexts_com_snowplowanalytics_snowplow_browser_context_2,
NULL as contexts_com_snowplowanalytics_snowplow_mobile_context_1,
NULL as contexts_com_snowplowanalytics_mobile_screen_1,
NULL as unstruct_event_com_snowplowanalytics_snowplow_application_error_1
Expand Down
27 changes: 27 additions & 0 deletions integration_tests/models/source/default/snowplow_unified_browser_context_2_stg.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
{#
Copyright (c) 2023-present Snowplow Analytics Ltd. All rights reserved.
This program is licensed to you under the Snowplow Personal and Academic License Version 1.0,
and you may not use this file except in compliance with the Snowplow Personal and Academic License Version 1.0.
You may obtain a copy of the Snowplow Personal and Academic License Version 1.0 at https://docs.snowplow.io/personal-and-academic-license-1.0/
#}

Select

event_id as root_id,
collector_tstamp::timestamp as root_tstamp,
case when platform = 'web' then cast('na' as {{ type_string() }}) else null end as viewport,
case when platform = 'web' then cast('na' as {{ type_string() }}) else null end as document_size,
case when platform = 'web' then cast('na' as {{ type_string() }}) else null end as resolution,
case when platform = 'web' then cast(1 as {{ type_int() }}) else null end as color_depth,
case when platform = 'web' then cast(1 as {{ type_int() }}) else null end as device_pixel_ratio,
case when platform = 'web' then cast(1 as {{ type_boolean() }}) else null end as cookies_enabled,
case when platform = 'web' then cast(1 as {{ type_boolean() }}) else null end as online,
case when platform = 'web' then cast('na' as {{ type_string() }}) else null end as browser_language,
case when platform = 'web' then cast('na' as {{ type_string() }}) else null end as document_language,
case when platform = 'web' then cast(1 as {{ type_boolean() }}) else null end as webdriver,
case when platform = 'web' then cast(1 as {{ type_int() }}) else null end as device_memory,
case when platform = 'web' then cast(1 as {{ type_int() }}) else null end as hardware_concurrency,
case when platform = 'web' then cast('na' as {{ type_string() }}) else null end as tab_id,
'browser_context' as schema_name

from {{ ref('snowplow_unified_events') }}
Loading

0 comments on commit 606a8d2

Please sign in to comment.