diff --git a/.changes/unreleased/Fixes-20250105-002625.yaml b/.changes/unreleased/Fixes-20250105-002625.yaml new file mode 100644 index 0000000..8d0c31f --- /dev/null +++ b/.changes/unreleased/Fixes-20250105-002625.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix bi engine statistics structure not consistent between audit logs & information schema +time: 2025-01-05T00:26:25.959179+01:00 +custom: + Author: Kayrnt + Issue: "" diff --git a/.changes/unreleased/Fixes-20250105-002713.yaml b/.changes/unreleased/Fixes-20250105-002713.yaml new file mode 100644 index 0000000..913fc76 --- /dev/null +++ b/.changes/unreleased/Fixes-20250105-002713.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Fix materialized_as_view_if_explicit_projects macro as not working as intended in region mode +time: 2025-01-05T00:27:13.276091+01:00 +custom: + Author: Kayrnt + Issue: "" diff --git a/.changes/unreleased/Fixes-20250105-003040.yaml b/.changes/unreleased/Fixes-20250105-003040.yaml new file mode 100644 index 0000000..3b03fe7 --- /dev/null +++ b/.changes/unreleased/Fixes-20250105-003040.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Protect get_query_comment against potential undefined +time: 2025-01-05T00:30:40.27033+01:00 +custom: + Author: Kayrnt + Issue: "" diff --git a/macros/materialization_view_if_explicit_projects.sql b/macros/materialization_view_if_explicit_projects.sql index e17c706..45890e0 100644 --- a/macros/materialization_view_if_explicit_projects.sql +++ b/macros/materialization_view_if_explicit_projects.sql @@ -5,7 +5,7 @@ but these references are not project-qualified: "region-XXX.INFORMATION_SCHEMA.XXX" -#} {% macro materialized_as_view_if_explicit_projects() -%} - {% if var('input_gcp_projects', []) | length == 0 %} + {% if project_list() | length == 0 %} {{ return('ephemeral') }} {% else %} {{ return('view') }} diff --git a/macros/set_query_comment.sql b/macros/set_query_comment.sql index b935486..8a8c291 100644 --- a/macros/set_query_comment.sql +++ b/macros/set_query_comment.sql @@ -7,44 +7,44 @@ {%- do comment_dict.update( app='dbt', dbt_bigquery_monitoring_version="0.17.0", - dbt_version=dbt_version, - project_name=project_name, - target_name=target.name, - target_database=target.database, - target_schema=target.schema, - invocation_id=invocation_id + dbt_version=dbt_version | default(''), + project_name=project_name | default(''), + target_name=target.name | default(''), + target_database=target.database | default(''), + target_schema=target.schema | default(''), + invocation_id=invocation_id | default('') ) -%} {%- if node is not none -%} {%- do comment_dict.update( - node_name=node.name, - node_alias=node.alias, - node_package_name=node.package_name, - node_original_file_path=node.original_file_path, - node_database=node.database, - node_schema=node.schema, - node_id=node.unique_id, - node_resource_type=node.resource_type, - node_meta=node.config.meta, - node_tags=node.tags, - full_refresh=flags.FULL_REFRESH, - which=flags.WHICH, + node_name=node.name | default(''), + node_alias=node.alias | default(''), + node_package_name=node.package_name | default(''), + node_original_file_path=node.original_file_path | default(''), + node_database=node.database | default(''), + node_schema=node.schema | default(''), + node_id=node.unique_id | default(''), + node_resource_type=node.resource_type | default(''), + node_meta=node.config.meta | default({}), + node_tags=node.tags | default([]), + full_refresh=flags.FULL_REFRESH | default(''), + which=flags.WHICH | default('') ) -%} {%- if flags.INVOCATION_COMMAND -%} {%- do comment_dict.update( - invocation_command=flags.INVOCATION_COMMAND + invocation_command=flags.INVOCATION_COMMAND | default('') ) -%} {%- endif -%} - {%- if node.resource_type != ('seed') -%} {# Otherwise this throws an error saying 'Seeds cannot depend on other nodes.' #} + {%- if node.resource_type != ('seed') -%} {%- if node.refs is defined -%} {% set refs = [] %} {% for ref in node.refs %} {%- if dbt_version >= '1.5.0' -%} - {%- do refs.append(ref.name) -%} + {%- do refs.append(ref.name | default('')) -%} {%- else -%} - {%- do refs.append(ref[0]) -%} + {%- do refs.append(ref[0] | default('')) -%} {%- endif -%} {% endfor %} {%- do comment_dict.update( @@ -54,12 +54,12 @@ {%- endif -%} {%- if node.resource_type == 'model' -%} {%- do comment_dict.update( - materialized=node.config.materialized, + materialized=node.config.materialized | default('') ) -%} {%- if node.config.materialized == 'incremental' and node.config.incremental_strategy is defined -%} {%- do comment_dict.update( - incremental_strategy=node.config.incremental_strategy, + incremental_strategy=node.config.incremental_strategy | default('') ) -%} {%- endif -%} @@ -67,13 +67,13 @@ {%- if node.config.partition_by.copy_partitions is defined -%} {%- do comment_dict.update( - copy_partitions=node.config.partition_by.copy_partitions, + copy_partitions=node.config.partition_by.copy_partitions | default('') ) -%} {%- endif -%} {%- if node.config.partition_by.time_ingestion_partitioning is defined -%} {%- do comment_dict.update( - time_ingestion_partitioning=node.config.partition_by.time_ingestion_partitioning, + time_ingestion_partitioning=node.config.partition_by.time_ingestion_partitioning | default('') ) -%} {%- endif -%} @@ -84,31 +84,31 @@ {%- if env_var('DBT_CLOUD_PROJECT_ID', False) -%} {%- do comment_dict.update( - dbt_cloud_project_id=env_var('DBT_CLOUD_PROJECT_ID') + dbt_cloud_project_id=env_var('DBT_CLOUD_PROJECT_ID') | default('') ) -%} {%- endif -%} {%- if env_var('DBT_CLOUD_JOB_ID', False) -%} {%- do comment_dict.update( - dbt_cloud_job_id=env_var('DBT_CLOUD_JOB_ID') + dbt_cloud_job_id=env_var('DBT_CLOUD_JOB_ID') | default('') ) -%} {%- endif -%} {%- if env_var('DBT_CLOUD_RUN_ID', False) -%} {%- do comment_dict.update( - dbt_cloud_run_id=env_var('DBT_CLOUD_RUN_ID') + dbt_cloud_run_id=env_var('DBT_CLOUD_RUN_ID') | default('') ) -%} {%- endif -%} {%- if env_var('DBT_CLOUD_RUN_REASON_CATEGORY', False) -%} {%- do comment_dict.update( - dbt_cloud_run_reason_category=env_var('DBT_CLOUD_RUN_REASON_CATEGORY') + dbt_cloud_run_reason_category=env_var('DBT_CLOUD_RUN_REASON_CATEGORY') | default('') ) -%} {%- endif -%} {%- if env_var('DBT_CLOUD_RUN_REASON', False) -%} {%- do comment_dict.update( - dbt_cloud_run_reason=env_var('DBT_CLOUD_RUN_REASON') + dbt_cloud_run_reason=env_var('DBT_CLOUD_RUN_REASON') | default('') ) -%} {%- endif -%} diff --git a/models/base/jobs_from_audit_logs.sql b/models/base/jobs_from_audit_logs.sql index a13dc93..c0b7021 100644 --- a/models/base/jobs_from_audit_logs.sql +++ b/models/base/jobs_from_audit_logs.sql @@ -6,7 +6,7 @@ }} SELECT -- bi_engine_statistics is not available in the audit logs, so we default to NULL - CAST(NULL AS STRUCT>>) AS bi_engine_statistics, + CAST(NULL AS STRUCT>, acceleration_mode STRING>) AS bi_engine_statistics, CAST(JSON_VALUE(protopayload_auditlog.metadataJson, '$.jobChange.job.jobStats.queryStats.cacheHit') AS BOOL) AS cache_hit, TIMESTAMP(JSON_VALUE(protopayload_auditlog.metadataJson,