diff --git a/.changes/unreleased/Fixes-20240513-160121.yaml b/.changes/unreleased/Fixes-20240513-160121.yaml new file mode 100644 index 000000000..fd7072954 --- /dev/null +++ b/.changes/unreleased/Fixes-20240513-160121.yaml @@ -0,0 +1,7 @@ +kind: Fixes +body: Fix incremental python models error where Databricks could not find the temp + table transaction logs +time: 2024-05-13T16:01:21.255833-04:00 +custom: + Author: mikealfare + Issue: "1033" diff --git a/dbt/include/spark/macros/materializations/incremental/incremental.sql b/dbt/include/spark/macros/materializations/incremental/incremental.sql index 10d4f3ed8..9a66bab51 100644 --- a/dbt/include/spark/macros/materializations/incremental/incremental.sql +++ b/dbt/include/spark/macros/materializations/incremental/incremental.sql @@ -16,7 +16,7 @@ {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%} {%- set target_relation = this -%} {%- set existing_relation = load_relation(this) -%} - {%- set tmp_relation = make_temp_relation(this) -%} + {% set tmp_relation = this.incorporate(path = {"identifier": this.identifier ~ '__dbt_tmp'}) -%} {#-- for SQL model we will create temp view that doesn't have database and schema --#} {%- if language == 'sql'-%} diff --git a/requirements.txt b/requirements.txt index 40335fb8a..b32884c43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,10 @@ pyhive[hive_pure_sasl]~=0.7.0 requests>=2.28.1 -pyodbc~=5.0.1 --no-binary pyodbc +pyodbc~=4.0.39 --no-binary pyodbc sqlparams>=3.0.0 thrift>=0.13.0 +pyspark>=3.0.0,<4.0.0 sqlparse>=0.4.2 # not directly required, pinned by Snyk to avoid a vulnerability types-PyYAML diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py index 05e25c5f4..1195cbd3e 100644 --- a/tests/functional/adapter/test_python_model.py +++ b/tests/functional/adapter/test_python_model.py @@ -24,13 +24,6 @@ class TestPythonIncrementalModelSpark(BasePythonIncrementalTests): def project_config_update(self): return {} - @pytest.mark.skip( - "Databricks can't find the transaction log" - "See https://github.com/dbt-labs/dbt-spark/issues/1033" - ) - def test_incremental(self, project): - super().test_incremental(project) - models__simple_python_model = """ import pandas