diff --git a/.changes/unreleased/Fixes-20241104-172349.yaml b/.changes/unreleased/Fixes-20241104-172349.yaml new file mode 100644 index 000000000..07c90d93c --- /dev/null +++ b/.changes/unreleased/Fixes-20241104-172349.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Iceberg quoting ignore fix. +time: 2024-11-04T17:23:49.706297-08:00 +custom: + Author: versusfacit + Issue: "1227" diff --git a/dbt/adapters/snowflake/impl.py b/dbt/adapters/snowflake/impl.py index 89c21f531..dc0926d93 100644 --- a/dbt/adapters/snowflake/impl.py +++ b/dbt/adapters/snowflake/impl.py @@ -263,6 +263,11 @@ def list_relations_without_caching( # this can be collapsed once Snowflake adds is_iceberg to show objects columns = ["database_name", "schema_name", "name", "kind", "is_dynamic"] if self.behavior.enable_iceberg_materializations.no_warn: + # The QUOTED_IDENTIFIERS_IGNORE_CASE setting impacts column names like + # is_iceberg which is created by dbt, but it does not affect the case + # of column values in Snowflake's SHOW OBJECTS query! This + # normalization step ensures metadata queries are handled consistently. + schema_objects = schema_objects.rename(column_names={"IS_ICEBERG": "is_iceberg"}) columns.append("is_iceberg") return [self._parse_list_relations_result(obj) for obj in schema_objects.select(columns)] diff --git a/dbt/include/snowflake/macros/adapters.sql b/dbt/include/snowflake/macros/adapters.sql index b60cea0b0..0ca756c6c 100644 --- a/dbt/include/snowflake/macros/adapters.sql +++ b/dbt/include/snowflake/macros/adapters.sql @@ -135,7 +135,6 @@ {% endmacro %} {% macro snowflake__list_relations_without_caching(schema_relation, max_iter=10, max_results_per_iter=10000) %} - {%- set max_total_results = max_results_per_iter * max_iter -%} {%- set sql -%} {% if schema_relation is string %} @@ -147,7 +146,7 @@ {# -- Gated for performance reason. If you don't want Iceberg, you shouldn't pay the -- latency penalty. #} {% if adapter.behavior.enable_iceberg_materializations.no_warn %} - select all_objects.*, is_iceberg as "is_iceberg" + select all_objects.*, is_iceberg from table(result_scan(last_query_id(-1))) all_objects left join INFORMATION_SCHEMA.tables as all_tables on all_tables.table_name = all_objects."name" diff --git a/tests/functional/adapter/list_relations_tests/test_show_objects.py b/tests/functional/adapter/list_relations_tests/test_show_objects.py index e5eee39d9..91fb94f79 100644 --- a/tests/functional/adapter/list_relations_tests/test_show_objects.py +++ b/tests/functional/adapter/list_relations_tests/test_show_objects.py @@ -3,6 +3,8 @@ import pytest +from pathlib import Path + from dbt.adapters.factory import get_adapter_by_type from dbt.adapters.snowflake import SnowflakeRelation @@ -41,8 +43,32 @@ """ ) +_MODEL_ICEBERG = """ +{{ + config( + materialized = "table", + table_format="iceberg", + external_volume="s3_iceberg_snow", + ) +}} + +select 1 +""" + + +class ShowObjectsBase: + @staticmethod + def list_relations_without_caching(project) -> List[SnowflakeRelation]: + my_adapter = get_adapter_by_type("snowflake") + schema = my_adapter.Relation.create( + database=project.database, schema=project.test_schema, identifier="" + ) + with get_connection(my_adapter): + relations = my_adapter.list_relations_without_caching(schema) + return relations + -class TestShowObjects: +class TestShowObjects(ShowObjectsBase): views: int = 10 tables: int = 10 dynamic_tables: int = 10 @@ -66,16 +92,6 @@ def setup(self, project): run_dbt(["seed"]) run_dbt(["run"]) - @staticmethod - def list_relations_without_caching(project) -> List[SnowflakeRelation]: - my_adapter = get_adapter_by_type("snowflake") - schema = my_adapter.Relation.create( - database=project.database, schema=project.test_schema, identifier="" - ) - with get_connection(my_adapter): - relations = my_adapter.list_relations_without_caching(schema) - return relations - def test_list_relations_without_caching(self, project): relations = self.list_relations_without_caching(project) assert len([relation for relation in relations if relation.is_view]) == self.views @@ -87,3 +103,25 @@ def test_list_relations_without_caching(self, project): len([relation for relation in relations if relation.is_dynamic_table]) == self.dynamic_tables ) + + +class TestShowIcebergObjects(ShowObjectsBase): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_iceberg_materializations": True}} + + @pytest.fixture(scope="class") + def models(self): + return {"my_model.sql": _MODEL_ICEBERG} + + def test_quoting_ignore_flag_doesnt_break_iceberg_metadata(self, project): + """https://github.com/dbt-labs/dbt-snowflake/issues/1227 + + The list relations function involves a metadata sub-query. Regardless of + QUOTED_IDENTIFIERS_IGNORE_CASE, this function will fail without proper + normalization within the encapsulating python function after the macro invocation + returns. This test verifies that normalization is working. + """ + run_dbt(["run"]) + + self.list_relations_without_caching(project)