From 383ee3cedefd15102dfb7427fb5f1e061ad41e4f Mon Sep 17 00:00:00 2001 From: Marc Beillevaire Date: Mon, 11 Mar 2024 19:23:20 +0100 Subject: [PATCH] Handle BigQuery non-string option 'max_staleness' (#237) * Handle BigQuery non-string option 'max_staleness' * Integration test for max_staleness BigQuery option --- .../models/plugins/bigquery/bigquery_external.yml | 10 ++++++++++ macros/plugins/bigquery/create_external_table.sql | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/integration_tests/models/plugins/bigquery/bigquery_external.yml b/integration_tests/models/plugins/bigquery/bigquery_external.yml index 3b81230b..3d6a0062 100644 --- a/integration_tests/models/plugins/bigquery/bigquery_external.yml +++ b/integration_tests/models/plugins/bigquery/bigquery_external.yml @@ -67,6 +67,16 @@ sources: columns: *cols-of-the-people tests: *equal-to-the-people + - name: people_csv_with_max_staleness + external: + location: 'gs://dbt-external-tables-testing/csv/*' + options: + format: csv + skip_leading_rows: 1 + max_staleness: INTERVAL 1 HOUR + columns: *cols-of-the-people + tests: *equal-to-the-people + # - name: people_json_unpartitioned # external: &json-people # location: 'gs://dbt-external-tables-testing/json/*' diff --git a/macros/plugins/bigquery/create_external_table.sql b/macros/plugins/bigquery/create_external_table.sql index 5e098894..f578782a 100644 --- a/macros/plugins/bigquery/create_external_table.sql +++ b/macros/plugins/bigquery/create_external_table.sql @@ -4,6 +4,7 @@ {%- set external = source_node.external -%} {%- set partitions = external.partitions -%} {%- set options = external.options -%} + {%- set non_string_options = ['max_staleness'] %} {% if options is mapping and options.get('connection_name', none) %} {% set connection_name = options.pop('connection_name') %} @@ -38,7 +39,7 @@ uris = [{%- for uri in uris -%} '{{uri}}' {{- "," if not loop.last}} {%- endfor -%}] {%- if options is mapping -%} {%- for key, value in options.items() if key != 'uris' %} - {%- if value is string -%} + {%- if value is string and key not in non_string_options -%} , {{key}} = '{{value}}' {%- else -%} , {{key}} = {{value}}