From 629ea94098cea5f4c182b123048086cd2a48c767 Mon Sep 17 00:00:00 2001
From: Mike Alfare <13974384+mikealfare@users.noreply.github.com>
Date: Tue, 14 May 2024 12:28:51 -0400
Subject: [PATCH] [Bug] Fix incremental python tables - dbt can't find
 temporary table transaction logs (#1042)

* unskip test
* align test pyodbc version with build deps; add pyspark from build deps
* revert change to exclude database and schema from temp relations
* retain existing behavior for unit tests by vendoring temp relation into the incremental materialization
* changelog entry
---
 .changes/unreleased/Fixes-20240513-160121.yaml             | 7 +++++++
 .../macros/materializations/incremental/incremental.sql    | 2 +-
 requirements.txt                                           | 3 ++-
 tests/functional/adapter/test_python_model.py              | 7 -------
 4 files changed, 10 insertions(+), 9 deletions(-)
 create mode 100644 .changes/unreleased/Fixes-20240513-160121.yaml

diff --git a/.changes/unreleased/Fixes-20240513-160121.yaml b/.changes/unreleased/Fixes-20240513-160121.yaml
new file mode 100644
index 000000000..fd7072954
--- /dev/null
+++ b/.changes/unreleased/Fixes-20240513-160121.yaml
@@ -0,0 +1,7 @@
+kind: Fixes
+body: Fix incremental python models error where Databricks could not find the temp
+  table transaction logs
+time: 2024-05-13T16:01:21.255833-04:00
+custom:
+  Author: mikealfare
+  Issue: "1033"
diff --git a/dbt/include/spark/macros/materializations/incremental/incremental.sql b/dbt/include/spark/macros/materializations/incremental/incremental.sql
index 10d4f3ed8..9a66bab51 100644
--- a/dbt/include/spark/macros/materializations/incremental/incremental.sql
+++ b/dbt/include/spark/macros/materializations/incremental/incremental.sql
@@ -16,7 +16,7 @@
   {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%}
   {%- set target_relation = this -%}
   {%- set existing_relation = load_relation(this) -%}
-  {%- set tmp_relation = make_temp_relation(this) -%}
+  {% set tmp_relation = this.incorporate(path = {"identifier": this.identifier ~ '__dbt_tmp'}) -%}
 
   {#-- for SQL model we will create temp view that doesn't have database and schema --#}
   {%- if language == 'sql'-%}
diff --git a/requirements.txt b/requirements.txt
index 40335fb8a..b32884c43 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,10 @@
 pyhive[hive_pure_sasl]~=0.7.0
 requests>=2.28.1
 
-pyodbc~=5.0.1 --no-binary pyodbc
+pyodbc~=4.0.39 --no-binary pyodbc
 sqlparams>=3.0.0
 thrift>=0.13.0
+pyspark>=3.0.0,<4.0.0
 sqlparse>=0.4.2 # not directly required, pinned by Snyk to avoid a vulnerability
 
 types-PyYAML
diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py
index 05e25c5f4..1195cbd3e 100644
--- a/tests/functional/adapter/test_python_model.py
+++ b/tests/functional/adapter/test_python_model.py
@@ -24,13 +24,6 @@ class TestPythonIncrementalModelSpark(BasePythonIncrementalTests):
     def project_config_update(self):
         return {}
 
-    @pytest.mark.skip(
-        "Databricks can't find the transaction log"
-        "See https://github.com/dbt-labs/dbt-spark/issues/1033"
-    )
-    def test_incremental(self, project):
-        super().test_incremental(project)
-
 
 models__simple_python_model = """
 import pandas