Merge branch 'main' into bigquery-table-expiry

integration_tests/models/plugins/bigquery/bigquery_external.yml

@@ -16,6 +16,7 @@ sources:
         columns: &cols-of-the-people
           - name: id
             data_type: int64
+            description: id_of_the_person
           - name: first_name
             data_type: string
           - name: last_name

macros/common/stage_external_sources.sql

@@ -66,6 +66,9 @@
 
         {% endfor %}
 
+        {% set update_columns = dbt_external_tables.update_external_table_columns(node) %}
+        {{ update_columns }}
+
     {% endfor %}
 
 {% endmacro %}

macros/common/update_external_table_columns.sql

@@ -0,0 +1,7 @@
+{% macro update_external_table_columns(source_node) %}
+    {{ return(adapter.dispatch('update_external_table_columns', 'dbt_external_tables')(source_node)) }}
+{% endmacro %}
+
+{% macro default__update_external_table_columns(source_node) %}
+
+{% endmacro %}

macros/plugins/bigquery/create_external_table.sql

@@ -1,5 +1,4 @@
 {% macro bigquery__create_external_table(source_node) %}
-
     {%- set columns = source_node.columns.values() -%}
     {%- set external = source_node.external -%}
     {%- set partitions = external.partitions -%}

macros/plugins/bigquery/update_external_table_columns.sql

@@ -0,0 +1,5 @@
+{% macro bigquery__update_external_table_columns(source_node) %}
+    {%- set columns = source_node.columns -%}
+    {%- set relation = source(source_node.source_name, source_node.name) -%}
+    {%- do adapter.update_columns(relation, columns) -%}
+{% endmacro %}

macros/plugins/snowflake/create_external_table.sql

@@ -51,6 +51,9 @@
     {% if external.auto_refresh in (true, false) -%}
       auto_refresh = {{external.auto_refresh}}
     {%- endif %}
+    {% if external.table_format | lower == "delta" %}
+      refresh_on_create = false
+    {% endif %}
     {% if external.pattern -%} pattern = '{{external.pattern}}' {%- endif %}
     {% if external.integration -%} integration = '{{external.integration}}' {%- endif %}
     file_format = {{external.file_format}}

macros/plugins/snowflake/refresh_external_table.sql

@@ -5,8 +5,9 @@
 
     {% set auto_refresh = external.get('auto_refresh', false) %}
     {% set partitions = external.get('partitions', none) %}
+    {% set delta_format = (external.table_format | lower == "delta") %}
 
-    {% set manual_refresh = (partitions and not auto_refresh) %}
+    {% set manual_refresh = not auto_refresh %}
 
     {% if manual_refresh %}
 

macros/plugins/snowflake/snowpipe/create_empty_table.sql

@@ -12,6 +12,7 @@
         {% endif %}
             metadata_filename varchar,
             metadata_file_row_number bigint,
+            metadata_file_last_modified timestamp,
             _dbt_copied_at timestamp
     );
 

macros/plugins/snowflake/snowpipe/get_copy_sql.sql

@@ -5,11 +5,11 @@
     {%- set external = source_node.external -%}
     {%- set is_csv = dbt_external_tables.is_csv(external.file_format) %}
     {%- set copy_options = external.snowpipe.get('copy_options', none) -%}
-   
+
     {%- if explicit_transaction -%} begin; {%- endif %}
-    
+
     copy into {{source(source_node.source_name, source_node.name)}}
-    from ( 
+    from (
         select
         {% if columns|length == 0 %}
             $1::variant as value,
@@ -25,13 +25,14 @@
         {% endif %}
             metadata$filename::varchar as metadata_filename,
             metadata$file_row_number::bigint as metadata_file_row_number,
+            metadata$file_last_modified::timestamp as metadata_file_last_modified,
             current_timestamp::timestamp as _dbt_copied_at
         from {{external.location}} {# stage #}
     )
     file_format = {{external.file_format}}
     {% if external.pattern -%} pattern = '{{external.pattern}}' {%- endif %}
     {% if copy_options %} {{copy_options}} {% endif %};
-    
+
     {% if explicit_transaction -%} commit; {%- endif -%}
 
 {% endmacro %}