CT-1524 - Add support for python 3.11 (dbt-labs#236)

* added python 3.11 support
   * added change log for python 3.11 support
   * added python 3.11 support in docs
* updated `setup.py`
   * replaced regular expressions with direct parsing with `exec()` for version
   * replaced os.path with pathlib
   * made `setup.py` functions private
   * decomposed the dbt-redshift > dbt-core function to make it clear that we're pinning the plugin to dbt-core at the minor
* pinned all dependencies to the minor version
* added `pip install -e .` to `make dev` so that packages in `setup.py` get picked up as well, namely boto3 
* readability updates
   * added comments for clarity
   * fixed typos in comments
   * fixed white space convention
* general cleanup
   * removed file because it did nothing; it contained a test that always passes

.changes/unreleased/Dependencies-20221209-233905.yaml

@@ -0,0 +1,7 @@
+kind: Dependencies
+body: Add support for python 3.11
+time: 2022-12-09T23:39:05.296196-05:00
+custom:
+  Author: mikealfare
+  Issue: "225"
+  PR: "236"

.github/scripts/integration-test-matrix.js

@@ -1,6 +1,6 @@
 module.exports = ({ context }) => {
   const defaultPythonVersion = "3.8";
-  const supportedPythonVersions = ["3.7", "3.8", "3.9", "3.10"];
+  const supportedPythonVersions = ["3.7", "3.8", "3.9", "3.10", "3.11"];
   const supportedAdapters = ["redshift"];
 
   // if PR, generate matrix based on files changed and PR labels

.github/workflows/main.yml

@@ -72,7 +72,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
 
     env:
       TOXENV: "unit"
@@ -174,7 +174,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
 
     steps:
       - name: Set up Python ${{ matrix.python-version }}

Makefile

@@ -1,9 +1,9 @@
 .DEFAULT_GOAL:=help
 
 .PHONY: dev
-dev: ## Installs adapter in develop mode along with development depedencies
+dev: ## Installs adapter in develop mode along with development dependencies
 	@\
-	pip install -r dev-requirements.txt && pre-commit install
+	pip install -e . -r dev-requirements.txt && pre-commit install
 
 .PHONY: mypy
 mypy: ## Runs mypy against staged changes for static type checking.

dev-requirements.txt

@@ -4,22 +4,22 @@ git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
 git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter
 git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-postgres&subdirectory=plugins/postgres
 
-black==22.8.0
+black~=22.8.0
 click~=8.1.3
-bumpversion
+bumpversion~=0.6.0
 flake8
-flaky
-freezegun==0.3.12
-ipdb
-mypy==0.971
-pip-tools
-pre-commit
-pytest
-pytest-dotenv
-pytest-logbook
-pytest-csv
-pytest-xdist
-pytz
-tox>=3.13
-twine
-wheel
+flaky~=3.7.0
+freezegun~=0.3.12
+ipdb~=0.13.9
+mypy~=0.971.0
+pip-tools~=6.11.0
+pre-commit~=2.20.0
+pytest~=7.2.0
+pytest-dotenv~=0.5.2
+pytest-logbook~=1.2.0
+pytest-csv~=3.0.0
+pytest-xdist~=3.1.0
+pytz~=2022.6.0
+tox~=4.0.0
+twine~=4.0.2
+wheel~=0.37.1

setup.py

@@ -1,75 +1,89 @@
 #!/usr/bin/env python
-import os
 import sys
-import re
 
-# require python 3.7 or newer
 if sys.version_info < (3, 7):
     print("Error: dbt does not support this version of Python.")
     print("Please upgrade to Python 3.7 or higher.")
     sys.exit(1)
 
 
-# require version of setuptools that supports find_namespace_packages
-from setuptools import setup
-
 try:
     from setuptools import find_namespace_packages
 except ImportError:
-    # the user has a downlevel version of setuptools.
     print("Error: dbt requires setuptools v40.1.0 or higher.")
-    print('Please upgrade setuptools with "pip install --upgrade setuptools" ' "and try again")
+    print('Please upgrade setuptools with "pip install --upgrade setuptools" and try again')
     sys.exit(1)
 
 
-# pull long description from README
-this_directory = os.path.abspath(os.path.dirname(__file__))
-with open(os.path.join(this_directory, "README.md")) as f:
-    long_description = f.read()
+from pathlib import Path
+from setuptools import setup
+
+
+# pull the long description from the README
+README = Path(__file__).parent / "README.md"
+
+
+# used for this adapter's version and in determining the compatible dbt-core version
+VERSION = Path(__file__).parent / "dbt/adapters/redshift/__version__.py"
+
+
+def _plugin_version() -> str:
+    """
+    Pull the package version from the main package version file
+    """
+    attributes = {}
+    exec(VERSION.read_text(), attributes)
+    return attributes["version"]
+
+
+def _core_patch(plugin_patch: str):
+    """
+    Determines the compatible dbt-core patch given this plugin's patch
+
+    Args:
+        plugin_patch: the version patch of this plugin
+    """
+    pre_release_phase = "".join([i for i in plugin_patch if not i.isdigit()])
+    if pre_release_phase:
+        if pre_release_phase not in ["a", "b", "rc"]:
+            raise ValueError(f"Invalid prerelease patch: {plugin_patch}")
+        return f"0{pre_release_phase}1"
+    return "0"
 
 
-# get this package's version from dbt/adapters/<name>/__version__.py
-def _get_plugin_version_dict():
-    _version_path = os.path.join(this_directory, "dbt", "adapters", "redshift", "__version__.py")
-    _semver = r"""(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)"""
-    _pre = r"""((?P<prekind>a|b|rc)(?P<pre>\d+))?"""
-    _version_pattern = fr"""version\s*=\s*["']{_semver}{_pre}["']"""
-    with open(_version_path) as f:
-        match = re.search(_version_pattern, f.read().strip())
-        if match is None:
-            raise ValueError(f"invalid version at {_version_path}")
-        return match.groupdict()
+# require a compatible minor version (~=) and prerelease if this is a prerelease
+def _core_version(plugin_version: str = _plugin_version()) -> str:
+    """
+    Determine the compatible dbt-core version give this plugin's version.
 
+    We assume that the plugin must agree with `dbt-core` down to the minor version.
 
-# require a compatible minor version (~=), prerelease if this is a prerelease
-def _get_dbt_core_version():
-    parts = _get_plugin_version_dict()
-    minor = "{major}.{minor}.0".format(**parts)
-    pre = parts["prekind"] + "1" if parts["prekind"] else ""
-    return f"{minor}{pre}"
+    Args:
+        plugin_version: the version of this plugin, this is an argument in case we ever want to unit test this
+    """
+    try:
+        major, minor, plugin_patch = plugin_version.split(".")
+    except ValueError:
+        raise ValueError(f"Invalid version: {plugin_version}")
 
+    return f"{major}.{minor}.{_core_patch(plugin_patch)}"
 
-package_name = "dbt-redshift"
-package_version = "1.4.0b1"
-dbt_core_version = _get_dbt_core_version()
-description = """The Redshift adapter plugin for dbt"""
 
 setup(
-    name=package_name,
-    version=package_version,
-    description=description,
-    long_description=long_description,
+    name="dbt-redshift",
+    version=_plugin_version(),
+    description="The Redshift adapter plugin for dbt",
+    long_description=README.read_text(),
     long_description_content_type="text/markdown",
     author="dbt Labs",
     author_email="[email protected]",
     url="https://github.com/dbt-labs/dbt-redshift",
     packages=find_namespace_packages(include=["dbt", "dbt.*"]),
     include_package_data=True,
     install_requires=[
-        "dbt-core~={}".format(dbt_core_version),
-        "dbt-postgres~={}".format(dbt_core_version),
-        # the following are all to match snowflake-connector-python
-        "boto3>=1.4.4,<2.0.0",
+        f"dbt-core~={_core_version()}",
+        f"dbt-postgres~={_core_version()}",
+        "boto3~=1.26.26",
     ],
     zip_safe=False,
     classifiers=[
@@ -82,6 +96,7 @@ def _get_dbt_core_version():
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
     python_requires=">=3.7",
 )

tests/conftest.py

@@ -1,11 +1,12 @@
 import pytest
 import os
 
-# Import the fuctional fixtures as a plugin
+# Import the functional fixtures as a plugin
 # Note: fixtures with session scope need to be local
 
 pytest_plugins = ["dbt.tests.fixtures.project"]
 
+
 # The profile dictionary, used to write out profiles.yml
 @pytest.fixture(scope="class")
 def dbt_profile_target():
@@ -19,5 +20,3 @@ def dbt_profile_target():
         'pass': os.getenv('REDSHIFT_TEST_PASS'),
         'dbname': os.getenv('REDSHIFT_TEST_DBNAME'),
     }
-
-

tests/functional/adapter/test_basic.py

@@ -1,12 +1,9 @@
 import pytest
 
-from dbt.tests.util import AnyStringWith, AnyFloat
-
+from dbt.tests.util import AnyStringWith
 from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations
 from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests
-from dbt.tests.adapter.basic.test_singular_tests_ephemeral import (
-    BaseSingularTestsEphemeral,
-)
+from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral
 from dbt.tests.adapter.basic.test_empty import BaseEmpty
 from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral
 from dbt.tests.adapter.basic.test_incremental import BaseIncremental
@@ -16,12 +13,12 @@
 from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod
 from dbt.tests.adapter.basic.test_docs_generate import BaseDocsGenerate, BaseDocsGenReferences
 from dbt.tests.adapter.basic.expected_catalog import base_expected_catalog, no_stats, expected_references_catalog
+from dbt.tests.adapter.basic.files import seeds_base_csv, seeds_added_csv, seeds_newcolumns_csv
+
 from tests.functional.adapter.expected_stats import redshift_stats, redshift_ephemeral_summary_stats
 
-from dbt.tests.adapter.basic.files import seeds_base_csv, seeds_added_csv, seeds_newcolumns_csv
 
-# set the datatype of the name column in the 'added' seed so it
-# can hold the '_update' that's added
+# set the datatype of the name column in the 'added' seed so that it can hold the '_update' that's added
 schema_seed_added_yml = """
 version: 2
 seeds:
@@ -32,6 +29,7 @@
 """
 
 
+# TODO: update these with test cases or remove them if not needed
 class TestSimpleMaterializationsRedshift(BaseSimpleMaterializations):
     pass
 
@@ -61,8 +59,7 @@ class TestGenericTestsRedshift(BaseGenericTests):
 
 
 class TestSnapshotCheckColsRedshift(BaseSnapshotCheckCols):
-    # Redshift defines the 'name' column such that it's not big enough
-    # to hold the '_update' added in the test.
+    # Redshift defines the 'name' column such that it's not big enough to hold the '_update' added in the test.
     @pytest.fixture(scope="class")
     def models(self):
         return {
@@ -73,8 +70,7 @@ def models(self):
 
 
 class TestSnapshotTimestampRedshift(BaseSnapshotTimestamp):
-    # Redshift defines the 'name' column such that it's not big enough
-    # to hold the '_update' added in the test.
+    # Redshift defines the 'name' column such that it's not big enough to hold the '_update' added in the test.
     @pytest.fixture(scope="class")
     def models(self):
         return {
@@ -84,11 +80,11 @@ def models(self):
             "seeds.yml": schema_seed_added_yml,
         }
 
+
 class TestBaseAdapterMethod(BaseAdapterMethod):
     pass
 
 
-
 class TestDocsGenerateRedshift(BaseDocsGenerate):
     @pytest.fixture(scope="class")                               
     def expected_catalog(self, project, profile_user):
@@ -105,6 +101,7 @@ def expected_catalog(self, project, profile_user):
         )             
 
 
+# TODO: update this or delete it
 @pytest.mark.skip(reason="Needs updated dbt-core code")
 class TestDocsGenReferencesRedshift(BaseDocsGenReferences):
     @pytest.fixture(scope="class")
@@ -123,4 +120,3 @@ def expected_catalog(self, project, profile_user):
             view_summary_stats=no_stats(),
             ephemeral_summary_stats=redshift_ephemeral_summary_stats(),
         )
-

tests/unit/mock_adapter.py

@@ -1,8 +1,8 @@
 from unittest import mock
-
-from dbt.adapters.base import BaseAdapter
 from contextlib import contextmanager
 
+from dbt.adapters.base import BaseAdapter, PythonJobHelper
+
 
 def adapter_factory():
     class MockAdapter(BaseAdapter):

tests/unit/test_context.py

@@ -53,6 +53,7 @@ def setUp(self):
     },
 }
 
+
 PROJECT_DATA = {
     'name': 'root',
     'version': '0.1',