Merge branch 'main' into dbeatty/test-cast-macro

.changes/unreleased/Features-20240501-151904.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Cross-database `date` macro
+time: 2024-05-01T15:19:04.822157-06:00
+custom:
+  Author: dbeatty10
+  Issue: 1031

.changes/unreleased/Fixes-20240513-160121.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Fix incremental python models error where Databricks could not find the temp
+  table transaction logs
+time: 2024-05-13T16:01:21.255833-04:00
+custom:
+  Author: mikealfare
+  Issue: "1033"

dbt/include/spark/macros/materializations/incremental/incremental.sql

@@ -16,7 +16,7 @@
   {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%}
   {%- set target_relation = this -%}
   {%- set existing_relation = load_relation(this) -%}
-  {%- set tmp_relation = make_temp_relation(this) -%}
+  {% set tmp_relation = this.incorporate(path = {"identifier": this.identifier ~ '__dbt_tmp'}) -%}
 
   {#-- for SQL model we will create temp view that doesn't have database and schema --#}
   {%- if language == 'sql'-%}

dbt/include/spark/macros/utils/date.sql

@@ -0,0 +1,5 @@
+{% macro spark__date(year, month, day) -%}
+    {%- set dt = modules.datetime.date(year, month, day) -%}
+    {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%}
+    to_date('{{ iso_8601_formatted_date }}', 'yyyy-MM-dd')
+{%- endmacro %}

requirements.txt

@@ -1,9 +1,10 @@
 pyhive[hive_pure_sasl]~=0.7.0
 requests>=2.28.1
 
-pyodbc~=5.0.1 --no-binary pyodbc
+pyodbc~=4.0.39 --no-binary pyodbc
 sqlparams>=3.0.0
 thrift>=0.13.0
+pyspark>=3.0.0,<4.0.0
 sqlparse>=0.4.2 # not directly required, pinned by Snyk to avoid a vulnerability
 
 types-PyYAML

tests/functional/adapter/test_python_model.py

@@ -24,13 +24,6 @@ class TestPythonIncrementalModelSpark(BasePythonIncrementalTests):
     def project_config_update(self):
         return {}
 
-    @pytest.mark.skip(
-        "Databricks can't find the transaction log"
-        "See https://github.com/dbt-labs/dbt-spark/issues/1033"
-    )
-    def test_incremental(self, project):
-        super().test_incremental(project)
-
 
 models__simple_python_model = """
 import pandas

tests/functional/adapter/utils/test_utils.py

@@ -9,6 +9,7 @@
 from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText
 from dbt.tests.adapter.utils.test_concat import BaseConcat
 from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampNaive
+from dbt.tests.adapter.utils.test_date import BaseDate
 from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd
 from dbt.tests.adapter.utils.test_datediff import BaseDateDiff
 from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc
@@ -85,6 +86,10 @@ class TestCurrentTimestamp(BaseCurrentTimestampNaive):
     pass
 
 
+class TestDate(BaseDate):
+    pass
+
+
 class TestDateAdd(BaseDateAdd):
     pass