From d32dbc764961f13e8b4ce0de09e736c2aee74a9a Mon Sep 17 00:00:00 2001 From: Robby Robinson Date: Thu, 11 Apr 2024 13:21:28 -0700 Subject: [PATCH 1/7] added ignore case option --- macros/plugins/snowflake/create_external_table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/plugins/snowflake/create_external_table.sql b/macros/plugins/snowflake/create_external_table.sql index 7db5749..76a63f1 100644 --- a/macros/plugins/snowflake/create_external_table.sql +++ b/macros/plugins/snowflake/create_external_table.sql @@ -7,7 +7,7 @@ {% if infer_schema %} {% set query_infer_schema %} - select * from table( infer_schema( location=>'{{external.location}}', file_format=>'{{external.file_format}}') ) + select * from table( infer_schema( location=>'{{external.location}}', file_format=>'{{external.file_format}}', ignore_case=>'{{external.ignore_case}}') ) {% endset %} {% if execute %} {% set columns_infer = run_query(query_infer_schema) %} From 033ff52a40e891a858c3517ab62298854fead17e Mon Sep 17 00:00:00 2001 From: Robby Robinson Date: Thu, 11 Apr 2024 16:03:42 -0700 Subject: [PATCH 2/7] fixed data type --- macros/plugins/snowflake/create_external_table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/plugins/snowflake/create_external_table.sql b/macros/plugins/snowflake/create_external_table.sql index 76a63f1..ca32cf7 100644 --- a/macros/plugins/snowflake/create_external_table.sql +++ b/macros/plugins/snowflake/create_external_table.sql @@ -7,7 +7,7 @@ {% if infer_schema %} {% set query_infer_schema %} - select * from table( infer_schema( location=>'{{external.location}}', file_format=>'{{external.file_format}}', ignore_case=>'{{external.ignore_case}}') ) + select * from table( infer_schema( location=>'{{external.location}}', file_format=>'{{external.file_format}}', ignore_case=>{{external.ignore_case}} ) ) {% endset %} {% if execute %} {% set columns_infer = run_query(query_infer_schema) %} From 52b4a7c4ab4919155b47468330171a963a0417bc Mon Sep 17 00:00:00 2001 From: Robby Robinson Date: Thu, 11 Apr 2024 16:14:04 -0700 Subject: [PATCH 3/7] made optional --- macros/plugins/snowflake/create_external_table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/plugins/snowflake/create_external_table.sql b/macros/plugins/snowflake/create_external_table.sql index ca32cf7..2a9b372 100644 --- a/macros/plugins/snowflake/create_external_table.sql +++ b/macros/plugins/snowflake/create_external_table.sql @@ -7,7 +7,7 @@ {% if infer_schema %} {% set query_infer_schema %} - select * from table( infer_schema( location=>'{{external.location}}', file_format=>'{{external.file_format}}', ignore_case=>{{external.ignore_case}} ) ) + select * from table( infer_schema( location=>'{{external.location}}', file_format=>'{{external.file_format}}'{%- if external.ignore_case -%}, ignore_case=>{{external.ignore_case}} {%- endif %} ) ) {% endset %} {% if execute %} {% set columns_infer = run_query(query_infer_schema) %} From 47cf91ab3ff7a378e27e1fb55cf7263e4afd86c1 Mon Sep 17 00:00:00 2001 From: Robby Robinson Date: Mon, 15 Apr 2024 10:14:32 -0700 Subject: [PATCH 4/7] added test source --- sample_sources/snowflake.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sample_sources/snowflake.yml b/sample_sources/snowflake.yml index 59ecde2..e0ee7ed 100644 --- a/sample_sources/snowflake.yml +++ b/sample_sources/snowflake.yml @@ -86,6 +86,23 @@ sources: - name: name description: and this is a name + - name: parquet_with_inferred_schema_and_mixed_column_case + description: "External table using Parquet and inferring the schema with mixed column case" + external: + location: "@stage" # reference an existing external stage + file_format: "my_file_format" # we need a named file format for infer to work + infer_schema: true # parameter to tell Snowflake we want to infer the table schema + ignore_case: true # parameter to tell Snowflake we want to ignore column case differences + partitions: + - name: section # we can define partitions on top of the schema columns + data_type: varchar(64) + expression: "substr(split_part(metadata$filename, 'section=', 2), 1, 1)" + columns: # columns can still be listed for documentation/testing purpose + - name: id + description: this is an id + - name: name + description: and this is a name + - name: aws_sns_refresh_tbl description: "External table using AWS SNS for auto-refresh" external: From def72195c29550ea950ff03cafcc3cc439ffb9b3 Mon Sep 17 00:00:00 2001 From: Robby Robinson Date: Thu, 18 Apr 2024 13:56:38 -0700 Subject: [PATCH 5/7] set ignore_case var --- macros/plugins/snowflake/create_external_table.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/macros/plugins/snowflake/create_external_table.sql b/macros/plugins/snowflake/create_external_table.sql index 2a9b372..98d1202 100644 --- a/macros/plugins/snowflake/create_external_table.sql +++ b/macros/plugins/snowflake/create_external_table.sql @@ -4,10 +4,11 @@ {%- set external = source_node.external -%} {%- set partitions = external.partitions -%} {%- set infer_schema = external.infer_schema -%} + {%- set ignore_case = external.ignore_case -%} {% if infer_schema %} {% set query_infer_schema %} - select * from table( infer_schema( location=>'{{external.location}}', file_format=>'{{external.file_format}}'{%- if external.ignore_case -%}, ignore_case=>{{external.ignore_case}} {%- endif %} ) ) + select * from table( infer_schema( location=>'{{external.location}}', file_format=>'{{external.file_format}}'{%- if ignore_case -%}, ignore_case=>{{ignore_case}} {%- endif %} ) ) {% endset %} {% if execute %} {% set columns_infer = run_query(query_infer_schema) %} From ea6ba33cc5fe8e35ba9ccd59c745ea741bf1a2f9 Mon Sep 17 00:00:00 2001 From: Robby Robinson Date: Thu, 18 Apr 2024 14:43:25 -0700 Subject: [PATCH 6/7] updated external table creation to use get_ignore_case --- macros/plugins/snowflake/create_external_table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/plugins/snowflake/create_external_table.sql b/macros/plugins/snowflake/create_external_table.sql index 98d1202..ee02394 100644 --- a/macros/plugins/snowflake/create_external_table.sql +++ b/macros/plugins/snowflake/create_external_table.sql @@ -41,7 +41,7 @@ {%- for column in columns_infer %} {%- set col_expression -%} {%- set col_id = 'value:' ~ column[0] -%} - (case when is_null_value({{col_id}}) or lower({{col_id}}) = 'null' then null else {{col_id}} end) + (case when is_null_value({{col_id}}) or lower({{col_id}}) = 'null' then null else {% if ignore_case -%} GET_IGNORE_CASE(value, '{{column[0]}}') {%- else -%} {{col_id}} {%- endif %} end) {%- endset %} {{column[0]}} {{column[1]}} as ({{col_expression}}::{{column[1]}}) {{- ',' if not loop.last -}} From 7e4acfe7d1a6ac0d5906849fec91bc7eeb85414c Mon Sep 17 00:00:00 2001 From: Robby Robinson Date: Wed, 1 May 2024 09:39:13 -0700 Subject: [PATCH 7/7] fixed ignore_case col_id for nulls --- macros/plugins/snowflake/create_external_table.sql | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/macros/plugins/snowflake/create_external_table.sql b/macros/plugins/snowflake/create_external_table.sql index be74554..480f858 100644 --- a/macros/plugins/snowflake/create_external_table.sql +++ b/macros/plugins/snowflake/create_external_table.sql @@ -51,8 +51,12 @@ {% else %} {%- for column in columns_infer %} {%- set col_expression -%} - {%- set col_id = 'value:' ~ column[0] -%} - (case when is_null_value({{col_id}}) or lower({{col_id}}) = 'null' then null else {% if ignore_case -%} GET_IGNORE_CASE(value, '{{column[0]}}') {%- else -%} {{col_id}} {%- endif %} end) + {%- if not ignore_case -%} + {%- set col_id = 'value:' ~ column[0] -%} + {%- else -%} + {%- set col_id = "get_ignore_case(value, '{}')".format(column[0]) -%} + {%- endif -%} + (case when is_null_value({{col_id}}) or lower({{col_id}}) = 'null' then null else {{col_id}} end) {%- endset %} {{column[0]}} {{column[1]}} as ({{col_expression}}::{{column[1]}}) {{- ',' if not loop.last -}}