Merge pull request #101 from snowplow/release/mobile/bigquery/1.1.0

Release/mobile/bigquery/1.1.0
snowplow · Jun 15, 2021 · ead84ed · ead84ed
2 parents 847a2e6 + 1ec4098
commit ead84ed
Show file tree

Hide file tree

Showing 32 changed files with 57 additions and 38 deletions.
diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_base.json b/.test/great_expectations/expectations/mobile/v1/mobile_base.json
@@ -137,8 +137,8 @@
   ],
   "meta": {
     "versions": {
-      "test_suite_version": "1.0.2",
-      "bigquery_model_version": "1.0.0",
+      "test_suite_version": "1.0.3",
+      "bigquery_model_version": "1.1.0",
       "snowflake_model_version": "1.0.0"
     },
     "great_expectations.__version__": "0.12.0"

diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_base_redshift.json b/.test/great_expectations/expectations/mobile/v1/mobile_base_redshift.json
@@ -248,7 +248,7 @@
   ],
   "meta": {
     "versions": {
-      "test_suite_version": "1.0.2",
+      "test_suite_version": "1.0.3",
       "redshift_model_version": "1.1.0"
     },
     "great_expectations.__version__": "0.12.0"

diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_metadata.json b/.test/great_expectations/expectations/mobile/v1/mobile_metadata.json
@@ -102,9 +102,9 @@
   ],
   "meta": {
     "versions": {
-      "test_suite_version": "1.0.2",
+      "test_suite_version": "1.0.3",
       "redshift_model_version": "1.1.0",
-      "bigquery_model_version": "1.0.0",
+      "bigquery_model_version": "1.1.0",
       "snowflake_model_version": "1.0.0"
     },
     "great_expectations.__version__": "0.12.0"

diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_screen_view_in_session_values.json b/.test/great_expectations/expectations/mobile/v1/mobile_screen_view_in_session_values.json
@@ -26,9 +26,9 @@
   ],
   "meta": {
     "versions": {
-      "test_suite_version": "1.0.2",
+      "test_suite_version": "1.0.3",
       "redshift_model_version": "1.1.0",
-      "bigquery_model_version": "1.0.0",
+      "bigquery_model_version": "1.1.0",
       "snowflake_model_version": "1.0.0"
     },
     "great_expectations.__version__": "0.12.0"

diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_screen_views.json b/.test/great_expectations/expectations/mobile/v1/mobile_screen_views.json
@@ -166,9 +166,9 @@
   ],
   "meta": {
     "versions": {
-      "test_suite_version": "1.0.2",
+      "test_suite_version": "1.0.3",
       "redshift_model_version": "1.1.0",
-      "bigquery_model_version": "1.0.0",
+      "bigquery_model_version": "1.1.0",
       "snowflake_model_version": "1.0.0"
     },
     "great_expectations.__version__": "0.12.0"

diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_sessions.json b/.test/great_expectations/expectations/mobile/v1/mobile_sessions.json
@@ -154,9 +154,9 @@
   ],
   "meta": {
     "versions": {
-      "test_suite_version": "1.0.2",
+      "test_suite_version": "1.0.3",
       "redshift_model_version": "1.1.0",
-      "bigquery_model_version": "1.0.0",
+      "bigquery_model_version": "1.1.0",
       "snowflake_model_version": "1.0.0"
     },
     "great_expectations.__version__": "0.12.0"

diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_staging_reconciliation.json b/.test/great_expectations/expectations/mobile/v1/mobile_staging_reconciliation.json
@@ -87,9 +87,9 @@
   "meta": {
     "great_expectations.__version__": "0.12.0",
     "versions": {
-      "test_suite_version": "1.0.2",
+      "test_suite_version": "1.0.3",
       "redshift_model_version": "1.1.0",
-      "bigquery_model_version": "1.0.0",
+      "bigquery_model_version": "1.1.0",
       "snowflake_model_version": "1.0.0"
     }
   }

diff --git a/.test/great_expectations/expectations/mobile/v1/mobile_users.json b/.test/great_expectations/expectations/mobile/v1/mobile_users.json
@@ -113,9 +113,9 @@
   ],
   "meta": {
     "versions": {
-      "test_suite_version": "1.0.2",
+      "test_suite_version": "1.0.3",
       "redshift_model_version": "1.1.0",
-      "bigquery_model_version": "1.0.0",
+      "bigquery_model_version": "1.1.0",
       "snowflake_model_version": "1.0.0"
     },
     "great_expectations.__version__": "0.12.0"

diff --git a/.test/great_expectations/validation_configs/mobile/v1/bigquery/temp_tables.json b/.test/great_expectations/validation_configs/mobile/v1/bigquery/temp_tables.json
@@ -54,6 +54,15 @@
         "bigquery_temp_table": "ge_test_scratch_mobile_usr_this_run"
       },
       "expectation_suite_names": ["mobile.v1.mobile_users"]
+    },
+    {
+      "batch_kwargs": {
+
+        "table": "scratch.mobile_staging_reconciliation",
+        "datasource": "bigquery",
+        "bigquery_temp_table": "ge_test_scratch_mobile_stg_recon"
+      },
+      "expectation_suite_names": ["mobile.v1.mobile_staging_reconciliation"]
     }
   ]
 }
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,8 @@
+BigQuery Mobile Version 1.1.0 (2021-06-15)
+---------------------------------------
+BigQuery Mobile: Add mobile_staging_reconcilition to temp tables validation config (Close #100)
+BigQuery Mobile: Fix column order in events_this_run to allow for events_staged table migration (Close #99)
+
 Redshift Mobile Version 1.1.0 (2021-05-17)
 ---------------------------------------
 Redshift Mobile: Increase session_id character limit in manifest (Close #93)

diff --git a/mobile/v1/bigquery/CHANGELOG b/mobile/v1/bigquery/CHANGELOG
@@ -1,3 +1,8 @@
+Version 1.1.0 (2021-06-15)
+--------------------------
+BigQuery Mobile: Add mobile_staging_reconcilition to temp tables validation config (Close #100)
+BigQuery Mobile: Fix column order in events_this_run to allow for events_staged table migration (Close #99)
+
 Version 1.0.0 (2021-04-26)
 --------------------------
 Mobile: Add BigQuery mobile model v1 (close #73)
diff --git a/mobile/v1/bigquery/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl b/mobile/v1/bigquery/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl
@@ -4,7 +4,7 @@
   :project: 
   :region:
 :variables:
-  :model_version:      bigquery/mobile/1.0.0
+  :model_version:      bigquery/mobile/1.1.0
   :model:              mobile
   :scratch_schema:     scratch
   :output_schema:      derived

diff --git a/mobile/v1/bigquery/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl b/mobile/v1/bigquery/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl
@@ -4,7 +4,7 @@
   :project:
   :region:
 :variables:
-  :model_version:      bigquery/mobile/1.0.0
+  :model_version:      bigquery/mobile/1.1.0
   :model:              mobile
   :scratch_schema:     scratch
   :output_schema:      derived

diff --git a/mobile/v1/bigquery/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl b/mobile/v1/bigquery/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl
@@ -4,7 +4,7 @@
   :project:
   :region:
 :variables:
-  :model_version:      bigquery/mobile/1.0.0
+  :model_version:      bigquery/mobile/1.1.0
   :model:              mobile
   :scratch_schema:     scratch
   :output_schema:      derived

diff --git a/mobile/v1/bigquery/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl b/mobile/v1/bigquery/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl
@@ -4,7 +4,7 @@
   :project:
   :region:
 :variables:
-  :model_version:              bigquery/mobile/1.0.0
+  :model_version:              bigquery/mobile/1.1.0
   :model:                      mobile
   :input_schema:               atomic
   :scratch_schema:             scratch

diff --git a/mobile/v1/bigquery/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl b/mobile/v1/bigquery/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl
@@ -4,7 +4,7 @@
   :project:
   :region:
 :variables:
-  :model_version:      bigquery/mobile/1.0.0
+  :model_version:      bigquery/mobile/1.1.0
   :model:              mobile
   :scratch_schema:     scratch
   :output_schema:      derived

diff --git a/mobile/v1/bigquery/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl b/mobile/v1/bigquery/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl
@@ -4,7 +4,7 @@
   :project:
   :region:
 :variables:
-  :model_version:      bigquery/mobile/1.0.0
+  :model_version:      bigquery/mobile/1.1.0
   :model:              mobile
   :scratch_schema:     scratch
   :output_schema:      derived

diff --git a/...e/v1/bigquery/sql-runner/playbooks/standard/02-screen-views/01-screen-views-main.yml.tmpl b/...e/v1/bigquery/sql-runner/playbooks/standard/02-screen-views/01-screen-views-main.yml.tmpl
@@ -4,9 +4,9 @@
   :project:
   :region:
 :variables:
-  :model_version:         bigquery/mobile/1.0.0
   :scratch_schema:        scratch
   :output_schema:         derived
+  :model_version:         bigquery/mobile/1.1.0
   :entropy:               ""
   :upsert_lookback_days:
   :skip_derived:

diff --git a/.../bigquery/sql-runner/playbooks/standard/02-screen-views/99-screen-views-complete.yml.tmpl b/.../bigquery/sql-runner/playbooks/standard/02-screen-views/99-screen-views-complete.yml.tmpl
@@ -4,9 +4,9 @@
   :project:
   :region:
 :variables:
-  :model_version:      bigquery/mobile/1.0.0
   :scratch_schema:     scratch
   :output_schema:      derived
+  :model_version:      bigquery/mobile/1.1.0
   :entropy:            ""
   :cleanup_mode:       all
   :ends_run:           false

diff --git a/...1/bigquery/sql-runner/playbooks/standard/02-screen-views/XX-destroy-screen-views.yml.tmpl b/...1/bigquery/sql-runner/playbooks/standard/02-screen-views/XX-destroy-screen-views.yml.tmpl
@@ -4,9 +4,9 @@
   :project:
   :region:
 :variables:
-  :model_version:      bigquery/mobile/1.0.0
   :scratch_schema:     scratch
   :output_schema:      derived
+  :model_version:      bigquery/mobile/1.1.0
   :entropy:            ""
   :cleanup_mode:       all
   :ends_run:           true

diff --git a/...l-runner/playbooks/standard/03-optional-modules/01-app-errors/01-app-errors-main.yml.tmpl b/...l-runner/playbooks/standard/03-optional-modules/01-app-errors/01-app-errors-main.yml.tmpl
@@ -4,9 +4,9 @@
   :project:
   :region:
 :variables:
-  :model_version:         bigquery/mobile/1.0.0
   :scratch_schema:        scratch
   :output_schema:         derived
+  :model_version:         bigquery/mobile/1.1.0
   :enabled:               false
   :entropy:               ""
   :upsert_lookback_days:

diff --git a/...nner/playbooks/standard/03-optional-modules/01-app-errors/99-app-errors-complete.yml.tmpl b/...nner/playbooks/standard/03-optional-modules/01-app-errors/99-app-errors-complete.yml.tmpl
@@ -4,9 +4,9 @@
   :project:
   :region:
 :variables:
-  :model_version:      bigquery/mobile/1.0.0
   :scratch_schema:     scratch
   :output_schema:      derived
+  :model_version:      bigquery/mobile/1.1.0
   :entropy:            ""
   :cleanup_mode:       all
   :ends_run:           false

diff --git a/...unner/playbooks/standard/03-optional-modules/01-app-errors/XX-destroy-app-errors.yml.tmpl b/...unner/playbooks/standard/03-optional-modules/01-app-errors/XX-destroy-app-errors.yml.tmpl
@@ -4,9 +4,9 @@
   :project:
   :region:
 :variables:
-  :model_version:      bigquery/mobile/1.0.0
   :scratch_schema:     scratch
   :output_schema:      derived
+  :model_version:      bigquery/mobile/1.1.0
   :entropy:            ""
   :cleanup_mode:       all
   :ends_run:           true

diff --git a/mobile/v1/bigquery/sql-runner/playbooks/standard/04-sessions/01-sessions-main.yml.tmpl b/mobile/v1/bigquery/sql-runner/playbooks/standard/04-sessions/01-sessions-main.yml.tmpl
@@ -4,9 +4,9 @@
   :project:
   :region:
 :variables:
-  :model_version:         bigquery/mobile/1.0.0
   :scratch_schema:        scratch
   :output_schema:         derived
+  :model_version:         bigquery/mobile/1.1.0
   :entropy:               ""
   :upsert_lookback_days:
   :stage_next:            true

diff --git a/mobile/v1/bigquery/sql-runner/playbooks/standard/04-sessions/99-sessions-complete.yml.tmpl b/mobile/v1/bigquery/sql-runner/playbooks/standard/04-sessions/99-sessions-complete.yml.tmpl
@@ -4,9 +4,9 @@
   :project:
   :region:
 :variables:
-  :model_version:      bigquery/mobile/1.0.0
   :scratch_schema:     scratch
   :output_schema:      derived
+  :model_version:      bigquery/mobile/1.1.0
   :entropy:            ""
   :cleanup_mode:       all
   :ends_run:           false

diff --git a/mobile/v1/bigquery/sql-runner/playbooks/standard/04-sessions/XX-destroy-sessions.yml.tmpl b/mobile/v1/bigquery/sql-runner/playbooks/standard/04-sessions/XX-destroy-sessions.yml.tmpl
@@ -4,9 +4,9 @@
   :project:
   :region:
 :variables:
-  :model_version:      bigquery/mobile/1.0.0
   :scratch_schema:     scratch
   :output_schema:      derived
+  :model_version:      bigquery/mobile/1.1.0
   :entropy:            ""
   :cleanup_mode:       all
   :ends_run:           true

diff --git a/mobile/v1/bigquery/sql-runner/playbooks/standard/05-users/01-users-main.yml.tmpl b/mobile/v1/bigquery/sql-runner/playbooks/standard/05-users/01-users-main.yml.tmpl
@@ -4,9 +4,9 @@
   :project:
   :region:
 :variables:
-  :model_version:         bigquery/mobile/1.0.0
   :scratch_schema:        scratch
   :output_schema:         derived
+  :model_version:         bigquery/mobile/1.1.0
   :entropy:               ""
   :skip_derived:
   :upsert_lookback_days:

diff --git a/mobile/v1/bigquery/sql-runner/playbooks/standard/05-users/99-users-complete.yml.tmpl b/mobile/v1/bigquery/sql-runner/playbooks/standard/05-users/99-users-complete.yml.tmpl
@@ -4,9 +4,9 @@
   :project:
   :region:
 :variables:
-  :model_version:      bigquery/mobile/1.0.0
   :scratch_schema:     scratch
   :output_schema:      derived
+  :model_version:      bigquery/mobile/1.1.0
   :entropy:            ""
   :cleanup_mode:       all
   :ends_run:           true

diff --git a/mobile/v1/bigquery/sql-runner/playbooks/standard/05-users/XX-destroy-users.yml.tmpl b/mobile/v1/bigquery/sql-runner/playbooks/standard/05-users/XX-destroy-users.yml.tmpl
@@ -4,9 +4,9 @@
   :project:
   :region:
 :variables:
-  :model_version:      bigquery/mobile/1.0.0
   :scratch_schema:     scratch
   :output_schema:      derived
+  :model_version:      bigquery/mobile/1.1.0
   :entropy:            ""
   :cleanup_mode:       all
   :ends_run:           true

diff --git a/...-runner/playbooks/tests/00-staging-reconciliation/01-staging-reconciliation-main.yml.tmpl b/...-runner/playbooks/tests/00-staging-reconciliation/01-staging-reconciliation-main.yml.tmpl
@@ -4,7 +4,7 @@
   :project:
   :region:
 :variables:
-  :model_version:           bigquery/mobile/1.0.0
+  :model_version:           bigquery/mobile/1.1.0
   :model:                   mobile
   :scratch_schema:          scratch
   :entropy:                 ""

diff --git a/...ner/playbooks/tests/00-staging-reconciliation/99-staging-reconciliation-complete.yml.tmpl b/...ner/playbooks/tests/00-staging-reconciliation/99-staging-reconciliation-complete.yml.tmpl
@@ -4,7 +4,7 @@
   :project:
   :region:
 :variables:
-  :model_version:           bigquery/mobile/1.0.0
+  :model_version:           bigquery/mobile/1.1.0
   :model:                   mobile
   :scratch_schema:          scratch
   :entropy:                 ""

diff --git a/mobile/v1/bigquery/sql-runner/sql/standard/01-base/01-main/06-events-this-run.sql b/mobile/v1/bigquery/sql-runner/sql/standard/01-base/01-main/06-events-this-run.sql
@@ -170,9 +170,9 @@ SET (LOWER_LIMIT, UPPER_LIMIT) = (SELECT AS STRUCT lower_limit, upper_limit FROM
         e.event_id
       )
 
-    SELECT 
-      *,
-      ROW_NUMBER() OVER(PARTITION BY d.session_id ORDER BY d.derived_tstamp) AS event_index_in_session
+    SELECT
+      ROW_NUMBER() OVER(PARTITION BY d.session_id ORDER BY d.derived_tstamp) AS event_index_in_session, 
+      d.*
     
     FROM
       deduped_events AS d