multiple fixes when running the extension

bqbooster · Jan 4, 2025 · 3c5dfb6 · 3c5dfb6
1 parent 89f14d4
commit 3c5dfb6
Show file tree

Hide file tree

Showing 6 changed files with 51 additions and 33 deletions.
diff --git a/.changes/unreleased/Fixes-20250105-002625.yaml b/.changes/unreleased/Fixes-20250105-002625.yaml
@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix bi engine statistics structure not consistent between audit logs & information schema
+time: 2025-01-05T00:26:25.959179+01:00
+custom:
+    Author: Kayrnt
+    Issue: ""
diff --git a/.changes/unreleased/Fixes-20250105-002713.yaml b/.changes/unreleased/Fixes-20250105-002713.yaml
@@ -0,0 +1,6 @@
+kind: Fixes
+body: Fix materialized_as_view_if_explicit_projects macro as not working as intended in region mode
+time: 2025-01-05T00:27:13.276091+01:00
+custom:
+    Author: Kayrnt
+    Issue: ""
diff --git a/.changes/unreleased/Fixes-20250105-003040.yaml b/.changes/unreleased/Fixes-20250105-003040.yaml
@@ -0,0 +1,6 @@
+kind: Fixes
+body: Protect get_query_comment against potential undefined
+time: 2025-01-05T00:30:40.27033+01:00
+custom:
+    Author: Kayrnt
+    Issue: ""
diff --git a/macros/materialization_view_if_explicit_projects.sql b/macros/materialization_view_if_explicit_projects.sql
@@ -5,7 +5,7 @@
     but these references are not project-qualified: "region-XXX.INFORMATION_SCHEMA.XXX"
  -#}
 {% macro materialized_as_view_if_explicit_projects() -%}
-  {% if var('input_gcp_projects', []) | length == 0 %}
+  {% if project_list() | length == 0 %}
     {{ return('ephemeral') }}
   {% else %}
     {{ return('view') }}

diff --git a/macros/set_query_comment.sql b/macros/set_query_comment.sql
@@ -7,44 +7,44 @@
     {%- do comment_dict.update(
         app='dbt',
         dbt_bigquery_monitoring_version="0.17.0",
-        dbt_version=dbt_version,
-        project_name=project_name,
-        target_name=target.name,
-        target_database=target.database,
-        target_schema=target.schema,
-        invocation_id=invocation_id
+        dbt_version=dbt_version | default(''),
+        project_name=project_name | default(''),
+        target_name=target.name | default(''),
+        target_database=target.database | default(''),
+        target_schema=target.schema | default(''),
+        invocation_id=invocation_id | default('')
     ) -%}
 
     {%- if node is not none -%}
         {%- do comment_dict.update(
-            node_name=node.name,
-            node_alias=node.alias,
-            node_package_name=node.package_name,
-            node_original_file_path=node.original_file_path,
-            node_database=node.database,
-            node_schema=node.schema,
-            node_id=node.unique_id,
-            node_resource_type=node.resource_type,
-            node_meta=node.config.meta,
-            node_tags=node.tags,
-            full_refresh=flags.FULL_REFRESH,
-            which=flags.WHICH,
+            node_name=node.name | default(''),
+            node_alias=node.alias | default(''),
+            node_package_name=node.package_name | default(''),
+            node_original_file_path=node.original_file_path | default(''),
+            node_database=node.database | default(''),
+            node_schema=node.schema | default(''),
+            node_id=node.unique_id | default(''),
+            node_resource_type=node.resource_type | default(''),
+            node_meta=node.config.meta | default({}),
+            node_tags=node.tags | default([]),
+            full_refresh=flags.FULL_REFRESH | default(''),
+            which=flags.WHICH | default('')
         ) -%}
 
         {%- if flags.INVOCATION_COMMAND -%}
             {%- do comment_dict.update(
-                invocation_command=flags.INVOCATION_COMMAND
+                invocation_command=flags.INVOCATION_COMMAND | default('')
             ) -%}
         {%- endif -%}
 
-        {%- if node.resource_type != ('seed') -%} {# Otherwise this throws an error saying 'Seeds cannot depend on other nodes.' #}
+        {%- if node.resource_type != ('seed') -%}
             {%- if node.refs is defined -%}
                 {% set refs = [] %}
                 {% for ref in node.refs %}
                     {%- if dbt_version >= '1.5.0' -%}
-                        {%- do refs.append(ref.name) -%}
+                        {%- do refs.append(ref.name | default('')) -%}
                     {%- else -%}
-                        {%- do refs.append(ref[0]) -%}
+                        {%- do refs.append(ref[0] | default('')) -%}
                     {%- endif -%}
                 {% endfor %}
                 {%- do comment_dict.update(
@@ -54,26 +54,26 @@
         {%- endif -%}
         {%- if node.resource_type == 'model' -%}
             {%- do comment_dict.update(
-                materialized=node.config.materialized,
+                materialized=node.config.materialized | default('')
             ) -%}
 
             {%- if node.config.materialized == 'incremental' and node.config.incremental_strategy is defined -%}
                 {%- do comment_dict.update(
-                    incremental_strategy=node.config.incremental_strategy,
+                    incremental_strategy=node.config.incremental_strategy | default('')
                 ) -%}
             {%- endif -%}
 
             {%- if node.config.partition_by is defined -%}
 
                 {%- if node.config.partition_by.copy_partitions is defined -%}
                 {%- do comment_dict.update(
-                    copy_partitions=node.config.partition_by.copy_partitions,
+                    copy_partitions=node.config.partition_by.copy_partitions | default('')
                 ) -%}
                 {%- endif -%}
 
                 {%- if node.config.partition_by.time_ingestion_partitioning is defined -%}
                 {%- do comment_dict.update(
-                    time_ingestion_partitioning=node.config.partition_by.time_ingestion_partitioning,
+                    time_ingestion_partitioning=node.config.partition_by.time_ingestion_partitioning | default('')
                 ) -%}
                 {%- endif -%}
 
@@ -84,31 +84,31 @@
 
     {%- if env_var('DBT_CLOUD_PROJECT_ID', False) -%}
     {%- do comment_dict.update(
-        dbt_cloud_project_id=env_var('DBT_CLOUD_PROJECT_ID')
+        dbt_cloud_project_id=env_var('DBT_CLOUD_PROJECT_ID') | default('')
     ) -%}
     {%- endif -%}
 
     {%- if env_var('DBT_CLOUD_JOB_ID', False) -%}
     {%- do comment_dict.update(
-        dbt_cloud_job_id=env_var('DBT_CLOUD_JOB_ID')
+        dbt_cloud_job_id=env_var('DBT_CLOUD_JOB_ID') | default('')
     ) -%}
     {%- endif -%}
 
     {%- if env_var('DBT_CLOUD_RUN_ID', False) -%}
     {%- do comment_dict.update(
-        dbt_cloud_run_id=env_var('DBT_CLOUD_RUN_ID')
+        dbt_cloud_run_id=env_var('DBT_CLOUD_RUN_ID') | default('')
     ) -%}
     {%- endif -%}
 
     {%- if env_var('DBT_CLOUD_RUN_REASON_CATEGORY', False) -%}
     {%- do comment_dict.update(
-        dbt_cloud_run_reason_category=env_var('DBT_CLOUD_RUN_REASON_CATEGORY')
+        dbt_cloud_run_reason_category=env_var('DBT_CLOUD_RUN_REASON_CATEGORY') | default('')
     ) -%}
     {%- endif -%}
 
     {%- if env_var('DBT_CLOUD_RUN_REASON', False) -%}
     {%- do comment_dict.update(
-        dbt_cloud_run_reason=env_var('DBT_CLOUD_RUN_REASON')
+        dbt_cloud_run_reason=env_var('DBT_CLOUD_RUN_REASON') | default('')
     ) -%}
     {%- endif -%}
 

diff --git a/models/base/jobs_from_audit_logs.sql b/models/base/jobs_from_audit_logs.sql
@@ -6,7 +6,7 @@
 }}
 SELECT
     -- bi_engine_statistics is not available in the audit logs, so we default to NULL
-    CAST(NULL AS STRUCT<bi_engine_mode STRING, acceleration_mode STRING, bi_engine_reasons ARRAY<STRUCT<code STRING, message STRING>>>) AS bi_engine_statistics,
+    CAST(NULL AS STRUCT<bi_engine_mode STRING, bi_engine_reasons ARRAY<STRUCT<code STRING, message STRING>>, acceleration_mode STRING>) AS bi_engine_statistics,
     CAST(JSON_VALUE(protopayload_auditlog.metadataJson,
             '$.jobChange.job.jobStats.queryStats.cacheHit') AS BOOL) AS cache_hit,
     TIMESTAMP(JSON_VALUE(protopayload_auditlog.metadataJson,