Merge branch 'main' into dependabot/pip/twine-approx-eq-5.1

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.9.0a1
+current_version = 1.9.0b1
 parse = (?P<major>[\d]+) # major version number
 	\.(?P<minor>[\d]+) # minor version number
 	\.(?P<patch>[\d]+) # patch version number

.changes/1.9.0-b1.md

@@ -0,0 +1,31 @@
+## dbt-spark 1.9.0-b1 - October 01, 2024
+
+### Features
+
+- Add tests for cross-database `cast` macro ([#1028](https://github.com/dbt-labs/dbt-spark/issues/1028))
+- Cross-database `date` macro ([#1031](https://github.com/dbt-labs/dbt-spark/issues/1031))
+- Allow configuring snapshot column names ([#1096](https://github.com/dbt-labs/dbt-spark/issues/1096))
+- Support custom ODBC connection parameters via `connection_string_suffix` config ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092))
+- Add Microbatch Strategy to dbt-spark ([#1109](https://github.com/dbt-labs/dbt-spark/issues/1109))
+
+### Fixes
+
+- Fix incremental python models error where Databricks could not find the temp table transaction logs ([#1033](https://github.com/dbt-labs/dbt-spark/issues/1033))
+
+### Under the Hood
+
+- Lazy load agate to improve performance ([#1049](https://github.com/dbt-labs/dbt-spark/issues/1049))
+
+### Dependencies
+
+- Update freezegun requirement from ~=1.3 to ~=1.4 ([#966](https://github.com/dbt-labs/dbt-spark/pull/966))
+- Bump actions/download-artifact from 3 to 4 ([#1010](https://github.com/dbt-labs/dbt-spark/pull/1010))
+- Bump actions/upload-artifact from 3 to 4 ([#1011](https://github.com/dbt-labs/dbt-spark/pull/1011))
+- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#1012](https://github.com/dbt-labs/dbt-spark/pull/1012))
+- Update wheel requirement from ~=0.42 to ~=0.43 ([#1035](https://github.com/dbt-labs/dbt-spark/pull/1035))
+- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1073](https://github.com/dbt-labs/dbt-spark/pull/1073))
+- Loosen pin on pre-commit from 3.7.0 to 3.7 ([#1074](https://github.com/dbt-labs/dbt-spark/pull/1074))
+
+### Contributors
+- [@jpoley](https://github.com/jpoley) ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092))
+- [@nilan3](https://github.com/nilan3) ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092))

.changes/1.9.0/Features-20240903-161003.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Allow configuring snapshot column names
+time: 2024-09-03T16:10:03.021221-04:00
+custom:
+  Author: gshank
+  Issue: "1096"

.changes/1.9.0/Features-20240910-175846.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Support custom ODBC connection parameters via `connection_string_suffix` config
+time: 2024-09-10T17:58:46.141332-04:00
+custom:
+  Author: colin-rogers-dbt jpoley nilan3
+  Issue: "1092"

.changes/1.9.0/Features-20240925-125242.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add Microbatch Strategy to dbt-spark
+time: 2024-09-25T12:52:42.872017+01:00
+custom:
+  Author: michelleark
+  Issue: "1109"

.changes/unreleased/Breaking Changes-20241016-184157.yaml

@@ -0,0 +1,6 @@
+kind: Breaking Changes
+body: Drop support for Python 3.8
+time: 2024-10-16T18:41:57.721002-04:00
+custom:
+  Author: mikealfare
+  Issue: "1121"

.changes/unreleased/Features-20240927-133927.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Enable setting current value of dbt_valid_to
+time: 2024-09-27T13:39:27.268886-04:00
+custom:
+  Author: gshank
+  Issue: "1112"

.changes/unreleased/Under the Hood-20240911-192845.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Isolating distribution testing
+time: 2024-09-11T19:28:45.653064-04:00
+custom:
+    Author: leahwicz
+    Issue: "1069"

.github/dependabot.yml

@@ -5,18 +5,25 @@ updates:
     schedule:
       interval: "daily"
     rebase-strategy: "disabled"
+    ignore:
+      - dependency-name: "*"
+        update-types:
+          - version-update:semver-patch
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
       interval: "weekly"
     rebase-strategy: "disabled"
+    ignore:
+      - dependency-name: "*"
+        update-types:
+          - version-update:semver-patch
   - package-ecosystem: "docker"
     directory: "/docker"
     schedule:
       interval: "weekly"
     rebase-strategy: "disabled"
-  - package-ecosystem: "docker"
-    directory: "/docker-dev"
-    schedule:
-      interval: "weekly"
-    rebase-strategy: "disabled"
+    ignore:
+      - dependency-name: "*"
+        update-types:
+          - version-update:semver-patch

.github/scripts/update_dev_dependency_branches.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -e
+set -e
+
+
+dbt_adapters_branch=$1
+dbt_core_branch=$2
+dbt_common_branch=$3
+target_req_file="dev-requirements.txt"
+core_req_sed_pattern="s|dbt-core.git.*#egg=dbt-core|dbt-core.git@${dbt_core_branch}#egg=dbt-core|g"
+adapters_req_sed_pattern="s|dbt-adapters.git|dbt-adapters.git@${dbt_adapters_branch}|g"
+common_req_sed_pattern="s|dbt-common.git|dbt-common.git@${dbt_common_branch}|g"
+if [[ "$OSTYPE" == darwin* ]]; then
+ # mac ships with a different version of sed that requires a delimiter arg
+ sed -i "" "$adapters_req_sed_pattern" $target_req_file
+ sed -i "" "$core_req_sed_pattern" $target_req_file
+ sed -i "" "$common_req_sed_pattern" $target_req_file
+else
+ sed -i "$adapters_req_sed_pattern" $target_req_file
+ sed -i "$core_req_sed_pattern" $target_req_file
+ sed -i "$common_req_sed_pattern" $target_req_file
+fi

.github/workflows/integration.yml

@@ -10,6 +10,8 @@
 
 name: Adapter Integration Tests
 
+run-name: "${{ (contains(github.event_name, 'workflow_') && inputs.name) || github.event_name }}: ${{ (contains(github.event_name, 'workflow_') &&  inputs.adapter_branch) || github.ref_name }} by @${{ github.actor }}"
+
 on:
   push:
     branches:
@@ -24,10 +26,31 @@ on:
 
   workflow_dispatch:
     inputs:
-      dbt-core-branch:
-        description: "branch of dbt-core to use in dev-requirements.txt"
+      name:
+        description: "Name to associate with run (example: 'dbt-adapters-242')"
         required: false
         type: string
+        default: "Adapter Integration Tests"
+      adapter_branch:
+        description: "The branch of this adapter repository to use"
+        type: string
+        required: false
+        default: "main"
+      dbt_adapters_branch:
+        description: "The branch of dbt-adapters to use"
+        type: string
+        required: false
+        default: "main"
+      dbt_core_branch:
+        description: "The branch of dbt-core to use"
+        type: string
+        required: false
+        default: "main"
+      dbt_common_branch:
+        description: "The branch of dbt-common to use"
+        type: string
+        required: false
+        default: "main"
 
 # explicitly turn off permissions for `GITHUB_TOKEN`
 permissions: read-all
@@ -53,6 +76,7 @@ jobs:
         test:
           - "apache_spark"
           - "spark_session"
+          - "spark_http_odbc"
           - "databricks_sql_endpoint"
           - "databricks_cluster"
           - "databricks_http_cluster"
@@ -74,12 +98,19 @@ jobs:
       DBT_TEST_USER_3: "[email protected]"
 
     steps:
-      - name: Check out the repository
-        if: github.event_name != 'pull_request_target'
+      - name: Check out the repository (push)
+        if: github.event_name == 'push'
         uses: actions/checkout@v4
         with:
           persist-credentials: false
 
+      - name: Check out the repository (workflow_dispatch)
+        if: github.event_name == 'workflow_dispatch'
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          ref: ${{ inputs.adapter_branch }}
+
       # explicitly checkout the branch for the PR,
       # this is necessary for the `pull_request` event
       - name: Check out the repository (PR)
@@ -95,17 +126,20 @@ jobs:
         with:
           python-version: "3.11"
 
+      - name: Update Adapters and Core branches (update dev_requirements.txt)
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          ./.github/scripts/update_dev_dependency_branches.sh \
+            ${{ inputs.dbt_adapters_branch }} \
+            ${{ inputs.dbt_core_branch }} \
+            ${{ inputs.dbt_common_branch }}
+          cat dev-requirements.txt
+
       - name: Install python dependencies
         run: |
           python -m pip install --user --upgrade pip
           python -m pip --version
           python -m pip install -r dagger/requirements.txt
 
-      - name: Update dev_requirements.txt
-        if: inputs.dbt-core-branch != ''
-        run: |
-          pip install bumpversion
-          ./.github/scripts/update_dbt_core_branch.sh ${{ inputs.dbt-core-branch }}
-
       - name: Run tests for ${{ matrix.test }}
         run: python dagger/run_dbt_spark_tests.py --profile ${{ matrix.test }}

.github/workflows/main.yml

@@ -49,7 +49,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install python dependencies
         run: |
@@ -75,7 +75,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - name: Check out the repository
@@ -126,7 +126,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install python dependencies
         run: |
@@ -161,7 +161,7 @@ jobs:
           overwrite: true
 
   test-build:
-    name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }}
+    name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} / ${{ matrix.dist-type }}
 
     if: needs.build.outputs.is_alpha == 0
 
@@ -172,8 +172,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-12, windows-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        os: [ubuntu-latest, macos-14, windows-latest]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        dist-type: ["whl", "gz"]
 
     steps:
       - name: Set up Python ${{ matrix.python-version }}
@@ -186,6 +187,7 @@ jobs:
           python -m pip install --user --upgrade pip
           python -m pip install --upgrade wheel
           python -m pip --version
+
       - uses: actions/download-artifact@v4
         with:
           name: dist
@@ -194,15 +196,10 @@ jobs:
       - name: Show distributions
         run: ls -lh dist/
 
-      - name: Install wheel distributions
-        run: |
-          find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
-      - name: Check wheel distributions
+      - name: Install ${{ matrix.dist-type }} distributions
         run: |
-          python -c "import dbt.adapters.spark"
-      - name: Install source distributions
-        run: |
-          find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
-      - name: Check source distributions
+          find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
+
+      - name: Check ${{ matrix.dist-type }} distributions
         run: |
           python -c "import dbt.adapters.spark"

.github/workflows/release-internal.yml

@@ -37,7 +37,7 @@ defaults:
     shell: "bash"
 
 env:
-  PYTHON_TARGET_VERSION: 3.8
+  PYTHON_TARGET_VERSION: 3.9
 
 jobs:
   run-unit-tests:
@@ -79,6 +79,7 @@ jobs:
         test:
           - "apache_spark"
           - "spark_session"
+          - "spark_http_odbc"
           - "databricks_sql_endpoint"
           - "databricks_cluster"
           - "databricks_http_cluster"

.github/workflows/release-prep.yml

@@ -84,7 +84,7 @@ defaults:
     shell: bash
 
 env:
-  PYTHON_TARGET_VERSION: 3.8
+  PYTHON_TARGET_VERSION: 3.9
   NOTIFICATION_PREFIX: "[Release Preparation]"
 
 jobs:
@@ -448,7 +448,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - name: Check out the repository
@@ -482,6 +482,7 @@ jobs:
         test:
           - "apache_spark"
           - "spark_session"
+          - "spark_http_odbc"
           - "databricks_sql_endpoint"
           - "databricks_cluster"
           - "databricks_http_cluster"

.pre-commit-config.yaml

@@ -24,10 +24,10 @@ repos:
     -   id: black
         args:
         -   --line-length=99
-        -   --target-version=py38
         -   --target-version=py39
         -   --target-version=py310
         -   --target-version=py311
+        -   --target-version=py312
         additional_dependencies: [flaky]
 
 -   repo: https://github.com/pycqa/flake8