From b33ad8a421934c5a42ec40c4c658eafa885c8ece Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 8 May 2024 20:52:50 -0400 Subject: [PATCH 1/7] unskip test --- tests/functional/adapter/test_python_model.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py index 05e25c5f4..1195cbd3e 100644 --- a/tests/functional/adapter/test_python_model.py +++ b/tests/functional/adapter/test_python_model.py @@ -24,13 +24,6 @@ class TestPythonIncrementalModelSpark(BasePythonIncrementalTests): def project_config_update(self): return {} - @pytest.mark.skip( - "Databricks can't find the transaction log" - "See https://github.com/dbt-labs/dbt-spark/issues/1033" - ) - def test_incremental(self, project): - super().test_incremental(project) - models__simple_python_model = """ import pandas From 735a382c776bf26555973770c8cc47f2cc846c6c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 9 May 2024 10:14:15 -0400 Subject: [PATCH 2/7] align test pyodbc version with build deps; add pyspark from build deps --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 18ccc77fd..41229454e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,10 @@ pyhive[hive_pure_sasl]~=0.7.0 requests>=2.28.1 -pyodbc~=5.0.1 +pyodbc~=4.0.39 sqlparams>=3.0.0 thrift>=0.13.0 +pyspark>=3.0.0,<4.0.0 sqlparse>=0.4.2 # not directly required, pinned by Snyk to avoid a vulnerability types-PyYAML From dae47f4026f1a942b6329f623f22788b5d85d5da Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 9 May 2024 11:09:33 -0400 Subject: [PATCH 3/7] align test pyodbc version with build deps; add pyspark from build deps --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 41229454e..b32884c43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ pyhive[hive_pure_sasl]~=0.7.0 requests>=2.28.1 -pyodbc~=4.0.39 +pyodbc~=4.0.39 --no-binary pyodbc sqlparams>=3.0.0 thrift>=0.13.0 pyspark>=3.0.0,<4.0.0 From ab9c767ad6d892960f782d7d166bb48407b3ef48 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 13 May 2024 15:55:20 -0400 Subject: [PATCH 4/7] revert change to exclude database and schema from temp relations --- dbt/include/spark/macros/adapters.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbt/include/spark/macros/adapters.sql b/dbt/include/spark/macros/adapters.sql index a6404a2de..f9958d5a3 100644 --- a/dbt/include/spark/macros/adapters.sql +++ b/dbt/include/spark/macros/adapters.sql @@ -386,8 +386,6 @@ {% set tmp_relation = base_relation.incorporate(path = { "identifier": tmp_identifier }) -%} - - {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%} {% do return(tmp_relation) %} {% endmacro %} From c13a3344515c5b9ef0426217cc75de6f82f4bce2 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 13 May 2024 16:01:32 -0400 Subject: [PATCH 5/7] changelog entry --- .changes/unreleased/Fixes-20240513-160121.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changes/unreleased/Fixes-20240513-160121.yaml diff --git a/.changes/unreleased/Fixes-20240513-160121.yaml b/.changes/unreleased/Fixes-20240513-160121.yaml new file mode 100644 index 000000000..fd7072954 --- /dev/null +++ b/.changes/unreleased/Fixes-20240513-160121.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Fix incremental python models error where Databricks could not find the temp + table transaction logs +time: 2024-05-13T16:01:21.255833-04:00 +custom: + Author: mikealfare + Issue: "1033" From 1d7f8a738ee1ebf8ab1d6268373a3b633dd9c147 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 13 May 2024 17:40:45 -0400 Subject: [PATCH 6/7] retain existing behavior for unit tests by vendoring temp relation into the incremental materialization --- dbt/include/spark/macros/adapters.sql | 4 +++- .../spark/macros/materializations/incremental/incremental.sql | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dbt/include/spark/macros/adapters.sql b/dbt/include/spark/macros/adapters.sql index f9958d5a3..788ebc73e 100644 --- a/dbt/include/spark/macros/adapters.sql +++ b/dbt/include/spark/macros/adapters.sql @@ -138,7 +138,7 @@ {#-- We can't use temporary tables with `create ... as ()` syntax --#} {% macro spark__create_temporary_view(relation, compiled_code) -%} - create or replace temporary view {{ relation }} as + create or replace temporary view {{ relation.identifier }} as {{ compiled_code }} {%- endmacro -%} @@ -386,6 +386,8 @@ {% set tmp_relation = base_relation.incorporate(path = { "identifier": tmp_identifier }) -%} + + {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%} {% do return(tmp_relation) %} {% endmacro %} diff --git a/dbt/include/spark/macros/materializations/incremental/incremental.sql b/dbt/include/spark/macros/materializations/incremental/incremental.sql index 10d4f3ed8..9a66bab51 100644 --- a/dbt/include/spark/macros/materializations/incremental/incremental.sql +++ b/dbt/include/spark/macros/materializations/incremental/incremental.sql @@ -16,7 +16,7 @@ {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%} {%- set target_relation = this -%} {%- set existing_relation = load_relation(this) -%} - {%- set tmp_relation = make_temp_relation(this) -%} + {% set tmp_relation = this.incorporate(path = {"identifier": this.identifier ~ '__dbt_tmp'}) -%} {#-- for SQL model we will create temp view that doesn't have database and schema --#} {%- if language == 'sql'-%} From f10eb6ca1da03e4459e826a7c98844804852f89c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 13 May 2024 17:41:17 -0400 Subject: [PATCH 7/7] revert temporary view change --- dbt/include/spark/macros/adapters.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/spark/macros/adapters.sql b/dbt/include/spark/macros/adapters.sql index 788ebc73e..a6404a2de 100644 --- a/dbt/include/spark/macros/adapters.sql +++ b/dbt/include/spark/macros/adapters.sql @@ -138,7 +138,7 @@ {#-- We can't use temporary tables with `create ... as ()` syntax --#} {% macro spark__create_temporary_view(relation, compiled_code) -%} - create or replace temporary view {{ relation.identifier }} as + create or replace temporary view {{ relation }} as {{ compiled_code }} {%- endmacro -%}