Merge branch 'main' into cmcarthur/add-query-id-to-sqlquerystatus

.changes/unreleased/Breaking Changes-20241016-183143.yaml

@@ -0,0 +1,6 @@
+kind: Breaking Changes
+body: Drop support for Python 3.8
+time: 2024-10-16T18:31:43.4167-04:00
+custom:
+  Author: mikealfare
+  Issue: "1211"

.changes/unreleased/Under the Hood-20241016-035544.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add telemetry function
+time: 2024-10-16T03:55:44.144174-07:00
+custom:
+    Author: versusfacit
+    Issue: "301"

.github/scripts/integration-test-matrix.js

@@ -1,6 +1,6 @@
 module.exports = ({ context }) => {
-  const defaultPythonVersion = "3.8";
-  const supportedPythonVersions = ["3.8", "3.9", "3.10", "3.11", "3.12"];
+  const defaultPythonVersion = "3.9";
+  const supportedPythonVersions = ["3.9", "3.10", "3.11", "3.12"];
   const supportedAdapters = ["snowflake"];
 
   // if PR, generate matrix based on files changed and PR labels
@@ -44,7 +44,7 @@ module.exports = ({ context }) => {
 
             if (labels.includes("test macos") || testAllLabel) {
               include.push({
-                os: "macos-12",
+                os: "macos-14",
                 adapter,
                 "python-version": pythonVersion,
               });
@@ -78,7 +78,7 @@ module.exports = ({ context }) => {
   // additionally include runs for all adapters, on macos and windows,
   // but only for the default python version
   for (const adapter of supportedAdapters) {
-    for (const operatingSystem of ["windows-latest", "macos-12"]) {
+    for (const operatingSystem of ["windows-latest", "macos-14"]) {
       include.push({
         os: operatingSystem,
         adapter: adapter,

.github/workflows/main.yml

@@ -50,7 +50,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install python dependencies
         run: |
@@ -71,7 +71,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
 
     env:
       TOXENV: "unit"
@@ -127,7 +127,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install python dependencies
         run: |
@@ -174,8 +174,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-12, windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        os: [ubuntu-latest, macos-14, windows-latest]
+        python-version: ['3.9', '3.10', '3.11', '3.12']
         dist-type: ['whl', 'gz']
 
     steps:

.pre-commit-config.yaml

@@ -24,7 +24,6 @@ repos:
     -   id: black
         args:
         -   --line-length=99
-        -   --target-version=py38
         -   --target-version=py39
         -   --target-version=py310
         -   --target-version=py311

CONTRIBUTING.md

@@ -79,7 +79,7 @@ WARNING: The parameters in your `test.env` file must link to a valid Snowflake a
 There are a few methods for running tests locally.
 
 #### `tox`
-`tox` automatically runs unit tests against several Python versions using its own virtualenvs. Run `tox -p` to run unit tests for Python 3.8, Python 3.9, Python 3.10, and `flake8` in parallel. Run `tox -e py38` to invoke tests on Python version 3.8 only (use py38, py39, or py310). Tox recipes are found in `tox.ini`.
+`tox` automatically runs unit tests against several Python versions using its own virtualenvs. Run `tox -p` to run unit tests for Python 3.9 and Python 3.10, and `flake8` in parallel. Run `tox -e py39` to invoke tests on Python version 3.9 only (use py39 or py310). Tox recipes are found in `tox.ini`.
 
 #### `pytest`
 You may run a specific test or group of tests using `pytest` directly. Activate a Python virtualenv active with dev dependencies installed. Then, run tests like so:

Makefile

@@ -12,20 +12,20 @@ dev-uninstall: ## Uninstalls all packages while maintaining the virtual environm
 	pip uninstall -y dbt-snowflake
 
 .PHONY: unit
-unit: ## Runs unit tests with py38.
+unit: ## Runs unit tests with py39.
 	@\
-	tox -e py38
+	tox -e py39
 
 .PHONY: test
-test: ## Runs unit tests with py38 and code checks against staged changes.
+test: ## Runs unit tests with py39 and code checks against staged changes.
 	@\
-	tox -p -e py38; \
+	tox -p -e py39; \
 	pre-commit run --all-files
 
 .PHONY: integration
 integration: ## Runs snowflake integration tests with py38.
 	@\
-	tox -e py38-snowflake --
+	tox -e py39-snowflake --
 
 .PHONY: clean
 	@echo "cleaning repo"

dbt/adapters/snowflake/impl.py

@@ -4,6 +4,7 @@
 from dbt.adapters.base.impl import AdapterConfig, ConstraintSupport
 from dbt.adapters.base.meta import available
 from dbt.adapters.capability import CapabilityDict, CapabilitySupport, Support, Capability
+from dbt.adapters.contracts.relation import RelationConfig
 from dbt.adapters.sql import SQLAdapter
 from dbt.adapters.sql.impl import (
     LIST_SCHEMAS_MACRO_NAME,
@@ -25,6 +26,7 @@
     SnowflakeRelationType,
     TableFormat,
 )
+
 from dbt.adapters.snowflake import SnowflakeColumn
 from dbt.adapters.snowflake import SnowflakeConnectionManager
 from dbt.adapters.snowflake import SnowflakeRelation
@@ -419,3 +421,18 @@ def valid_incremental_strategies(self):
     def debug_query(self):
         """Override for DebugTask method"""
         self.execute("select 1 as id")
+
+    @classmethod
+    def _get_adapter_specific_run_info(cls, config: RelationConfig) -> Dict[str, Any]:
+        table_format: Optional[str] = None
+        if (
+            config
+            and hasattr(config, "_extra")
+            and (relation_format := config._extra.get("table_format"))
+        ):
+            table_format = relation_format
+
+        return {
+            "adapter_type": "snowflake",
+            "table_format": table_format,
+        }

dbt/include/snowflake/macros/materializations/incremental.sql

@@ -58,6 +58,7 @@
 
 {% materialization incremental, adapter='snowflake', supported_languages=['sql', 'python'] -%}
 
+  {% set original_query_tag = set_query_tag() %}
 
   {#-- Set vars --#}
   {%- set full_refresh_mode = (should_full_refresh()) -%}

dbt/include/snowflake/macros/relations/table/create.sql

@@ -1,7 +1,7 @@
 {% macro snowflake__create_table_as(temporary, relation, compiled_code, language='sql') -%}
 
   {%- if relation.is_iceberg_format and not adapter.behavior.enable_iceberg_materializations.no_warn %}
-    {% do exceptions.raise_compiler_error('Was unable to create model as Iceberg Table Format. Please set the `enable_iceberg_materializations` behavior flag to True in your dbt_project.yml. For more information, go to <url pending>.') %}
+    {% do exceptions.raise_compiler_error('Was unable to create model as Iceberg Table Format. Please set the `enable_iceberg_materializations` behavior flag to True in your dbt_project.yml. For more information, go to https://docs.getdbt.com/reference/resource-configs/snowflake-configs.') %}
   {%- endif %}
 
   {%- set materialization_prefix = relation.get_ddl_prefix_for_create(config.model.config, temporary) -%}

dev-requirements.txt

@@ -6,8 +6,7 @@ git+https://github.com/dbt-labs/dbt-common.git
 
 # dev
 ipdb~=0.13.13
-pre-commit~=3.7.0;python_version>="3.9"
-pre-commit~=3.5.0;python_version<"3.9"
+pre-commit~=3.7.0
 
 # test
 ddtrace==2.3.0

docker/Dockerfile

@@ -1,7 +1,7 @@
 # this image gets published to GHCR for production use
 ARG py_version=3.11.2
 
-FROM python:$py_version-slim-bullseye as base
+FROM python:$py_version-slim-bullseye AS base
 
 RUN apt-get update \
   && apt-get dist-upgrade -y \
@@ -25,7 +25,7 @@ ENV LANG=C.UTF-8
 RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir
 
 
-FROM base as dbt-snowflake
+FROM base AS dbt-snowflake
 
 ARG commit_ref=main
 

docker/dev.Dockerfile

@@ -1,43 +1,43 @@
 # this image does not get published, it is intended for local development only, see `Makefile` for usage
-FROM ubuntu:22.04 as base
+FROM ubuntu:24.04 AS base
 
 # prevent python installation from asking for time zone region
 ARG DEBIAN_FRONTEND=noninteractive
 
 # add python repository
 RUN apt-get update \
-  && apt-get install -y software-properties-common=0.99.22.9 \
-  && add-apt-repository -y ppa:deadsnakes/ppa \
-  && apt-get clean \
-  && rm -rf \
-    /var/lib/apt/lists/* \
-    /tmp/* \
-    /var/tmp/*
+    && apt-get install -y software-properties-common=0.99.48 \
+    && add-apt-repository -y ppa:deadsnakes/ppa \
+    && apt-get clean \
+    && rm -rf \
+        /var/lib/apt/lists/* \
+        /tmp/* \
+        /var/tmp/*
 
 # install python
 RUN apt-get update \
-  && apt-get install -y --no-install-recommends \
-    build-essential=12.9ubuntu3 \
-    git-all=1:2.34.1-1ubuntu1.10 \
-    python3.8=3.8.19-1+jammy1 \
-    python3.8-dev=3.8.19-1+jammy1 \
-    python3.8-distutils=3.8.19-1+jammy1 \
-    python3.8-venv=3.8.19-1+jammy1 \
-    python3-pip=22.0.2+dfsg-1ubuntu0.4 \
-    python3-wheel=0.37.1-2ubuntu0.22.04.1 \
-  && apt-get clean \
-  && rm -rf \
-    /var/lib/apt/lists/* \
-    /tmp/* \
-    /var/tmp/*
+    && apt-get install -y --no-install-recommends \
+        build-essential=12.10ubuntu1 \
+        git-all=1:2.43.0-1ubuntu7.1 \
+        python3.9=3.9.20-1+noble1 \
+        python3.9-dev=3.9.20-1+noble1 \
+        python3.9-distutils=3.9.20-1+noble1 \
+        python3.9-venv=3.9.20-1+noble1 \
+        python3-pip=24.0+dfsg-1ubuntu1 \
+        python3-wheel=0.42.0-2 \
+    && apt-get clean \
+    && rm -rf \
+        /var/lib/apt/lists/* \
+        /tmp/* \
+        /var/tmp/* \
 
 # update the default system interpreter to the newly installed version
-RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1
 
 
-FROM base as dbt-snowflake-dev
+FROM base AS dbt-snowflake-dev
 
-HEALTHCHECK CMD python3 --version || exit 1
+HEALTHCHECK CMD python --version || exit 1
 
 # send stdout/stderr to terminal
 ENV PYTHONUNBUFFERED=1
@@ -47,4 +47,4 @@ WORKDIR /opt/code
 VOLUME /opt/code
 
 # create a virtual environment
-RUN python3 -m venv /opt/venv
+RUN python -m venv /opt/venv

setup.py

@@ -5,9 +5,9 @@
 import sys
 
 # require python 3.8 or newer
-if sys.version_info < (3, 8):
+if sys.version_info < (3, 9):
     print("Error: dbt does not support this version of Python.")
-    print("Please upgrade to Python 3.8 or higher.")
+    print("Please upgrade to Python 3.9 or higher.")
     sys.exit(1)
 
 
@@ -72,12 +72,10 @@ def _plugin_version() -> str:
         "Operating System :: Microsoft :: Windows",
         "Operating System :: MacOS :: MacOS X",
         "Operating System :: POSIX :: Linux",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
     ],
-    python_requires=">=3.8",
+    python_requires=">=3.9",
 )

tests/functional/generic_test_tests/_files.py

@@ -0,0 +1,90 @@
+SCHEMA__CONTROL = """
+version: 2
+models:
+  - name: colors
+    columns:
+      - name: color
+        data_tests:
+          - not_null
+"""
+
+
+SCHEMA__EXPLICIT_WAREHOUSE = """
+version: 2
+models:
+  - name: colors
+    columns:
+      - name: color
+        data_tests:
+          - not_null:
+              config:
+                snowflake_warehouse: DBT_TESTING_ALT
+"""
+
+
+SCHEMA__NOT_NULL = """
+version: 2
+models:
+  - name: facts
+    columns:
+      - name: value
+        data_tests:
+          - not_null:
+              config:
+                snowflake_warehouse: DBT_TESTING_ALT
+"""
+
+
+SCHEMA__RELATIONSHIPS = """
+version: 2
+models:
+  - name: facts
+    columns:
+      - name: color
+        data_tests:
+          - relationships:
+              to: ref('my_colors')
+              field: color
+              config:
+                snowflake_warehouse: DBT_TESTING_ALT
+"""
+
+
+SCHEMA__ACCEPTED_VALUES = """
+version: 2
+models:
+  - name: colors
+    columns:
+      - name: color
+        data_tests:
+          - accepted_values:
+              values: ['blue', 'red', 'green']
+              config:
+                snowflake_warehouse: DBT_TESTING_ALT
+"""
+
+
+SEED__COLORS = """
+color
+blue
+green
+red
+yellow
+""".strip()
+
+
+# record 10 is missing a value
+# record 7 has a color that's not on COLORS
+SEED__FACTS = """
+id,color,value
+1,blue,10
+2,red,20
+3,green,30
+4,yellow,40
+5,blue,50
+6,red,60
+7,orange,70
+8,green,80
+9,yellow,90
+10,blue,
+""".strip()