type fixing, hatch updates

.github/workflows/ci_code_quality.yml

@@ -50,4 +50,4 @@ jobs:
         run: pip3 install hatch
 
       - name: Run Pre-commit Hooks
-        run: hatch run dev-env:pre-commit run --show-diff-on-failure --color=always --all-files
+        run: hatch run lint-all

.github/workflows/ci_tests.yml

@@ -55,7 +55,7 @@ jobs:
         run: pip3 install hatch
 
       - name: "Run Tests"
-        run: hatch run dev-env:pytest tests
+        run: hatch run unit-tests
 
       - name: "Get current date"
         if: always()

.gitignore

@@ -143,6 +143,9 @@ venv.bak/
 .dmypy.json
 dmypy.json
 
+# ruff
+.ruff_cache/
+
 # Pyre type checker
 .pyre/
 

.pre-commit-config.yaml

@@ -18,37 +18,10 @@ repos:
     exclude_types:
       - "markdown"
   - id: check-case-conflict
-- repo: https://github.com/psf/black
-  rev: 22.3.0
-  hooks:
-  - id: black
-  - id: black
-    alias: black-check
-    stages: [manual]
-    args:
-    - "--check"
-    - "--diff"
-- repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.4.1
-  hooks:
-  - id: mypy
-    # N.B.: Mypy is... a bit fragile.
-    #
-    # By using `language: system` we run this hook in the local
-    # environment instead of a pre-commit isolated one.  This is needed
-    # to ensure mypy correctly parses the project.
-
-    # It may cause trouble
-    # in that it adds environmental variables out of our control to the
-    # mix.  Unfortunately, there's nothing we can do about per pre-commit's
-    # author.
-    # See https://github.com/pre-commit/pre-commit/issues/730 for details.
-    args: [--show-error-codes]
-    files: ^dbt_common/
-    language: system
-  - id: mypy
-    alias: mypy-check
-    stages: [manual]
-    args: [--show-error-codes, --pretty]
-    files: ^dbt_common
-    language: system
+  - id: format
+    name: format
+    entry: hatch run lint-all
+    language: python
+    types: [python]
+    pass_filenames: false
+    verbose: true

dbt_common/__about__.py

@@ -0,0 +1 @@
+version = "0.0.1"

dbt_common/clients/agate_helper.py

@@ -17,7 +17,7 @@ def cast(self, d):
         # by default agate will cast none as a Number
         # but we need to cast it as an Integer to preserve
         # the type when merging and unioning tables
-        if isinstance(d, int) or d is None:
+        if type(d) == int or d is None:  # noqa [E721]
             return d
         else:
             raise agate.exceptions.CastError('Can not parse value "%s" as Integer.' % d)
@@ -30,7 +30,7 @@ class Number(agate.data_types.Number):
     # undo the change in https://github.com/wireservice/agate/pull/733
     # i.e. do not cast True and False to numeric 1 and 0
     def cast(self, d):
-        if isinstance(d, bool):
+        if type(d) == bool:  # noqa [E721]
             raise agate.exceptions.CastError("Do not cast True to 1 or False to 0.")
         else:
             return super().cast(d)

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dbt-common"
-version = "0.0.1"
+dynamic = ["version"]
 description = "The shared common utilities that dbt-core and adapter implementations use"
 readme = "README.md"
 requires-python = ">=3.8"
@@ -9,8 +9,15 @@ keywords = []
 authors = [
   { name = "dbt Labs", email = "[email protected]" },
 ]
+maintainers = [
+    { name = "dbt Labs", email = "[email protected]" },
+]
 classifiers = [
-  "Development Status :: 4 - Beta",
+  "Development Status :: 2 - Pre-Alpha",
+  "License :: OSI Approved :: Apache Software License",
+  "Operating System :: MacOS :: MacOS X",
+  "Operating System :: Microsoft :: Windows",
+  "Operating System :: POSIX :: Linux",
   "Programming Language :: Python",
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
@@ -32,6 +39,33 @@ dependencies = [
   "requests<3.0.0",
   "typing-extensions~=4.4",
 ]
+[project.optional-dependencies]
+lint = [
+    "black~=23.3",
+    "flake8",
+    "Flake8-pyproject",
+    "mypy~=1.3",
+    "ruff==0.1.11",
+    "types-Jinja2~=2.11",
+    "types-jsonschema~=4.17",
+    "types-protobuf~=4.24.0",
+    "types-python-dateutil~=2.8",
+    "types-PyYAML~=6.0",
+    "types-requests<2.31.0" # types-requests 2.31.0.8 requires urllib3>=2, but we pin urllib3 ~= 1.0 because of openssl requirement for requests
+
+]
+test = [
+    "pytest~=7.3",
+#     "pytest-dotenv",
+    "pytest-xdist~=3.2",
+    "hypothesis~=6.87"
+]
+
+[project.urls]
+Homepage = "https://github.com/dbt-labs/dbt-common"  # TODO: should this be dbt's homepage?
+Repository = "https://github.com/dbt-labs/dbt-common.git"
+Issues = "https://github.com/dbt-labs/dbt-common/issues"
+Changelog = "https://github.com/dbt-labs/dbt-common/blob/main/CHANGELOG.md"
 
 [build-system]
 requires = ["hatchling"]
@@ -45,36 +79,40 @@ exclude = [
   ".gitignore",
   ".pre-commit-config.yaml",
   "CONTRIBUTING.md",
-  "MAKEFILE",
+  "MAKEFILE",  # TODO: remove once we remove the makefile
   "/tests",
 ]
 
 [tool.hatch.build.targets.wheel]
 packages = ["dbt_common"]
 
-[tool.hatch.envs.dev-env.scripts]
-all = ["pre-commit run --all-files"]
+[tool.hatch.version]
+path = "dbt_common/__about__.py"
 
-[tool.hatch.envs.dev-env]
-description = "Env for running development commands like pytest / pre-commit"
-dependencies = [
-  "pytest~=7.3",
-  "pytest-xdist~=3.2",
-  "httpx~=0.24",
-  "hypothesis~=6.87",
-  "pre-commit~=3.2",
-  "isort~=5.12",
-  "black~=23.3",
-  "ruff==0.1.11",
-  "mypy~=1.3",
-  "pytest~=7.3",
-  "types-Jinja2~=2.11",
-  "types-jsonschema~=4.17",
-  "types-protobuf~=4.24.0",
-  "types-python-dateutil~=2.8",
-  "types-PyYAML~=6.0",
-  "types-requests<2.31.0" # types-requests 2.31.0.8 requires urllib3>=2, but we pin urllib3 ~= 1.0 because of openssl requirement for requests
+# [tool.hatch.envs.dev.scripts]
+# all = ["pre-commit run --all-files"]
+
+[tool.hatch.envs.default]
+description = "Env for running development commands like pytest & linting"
+features = ["lint", "test"]
+
+# these are the commands that you run with `hatch run <cmd>` for local dev & CI
+[tool.hatch.envs.default.scripts]
+# This edits your local pre-commit hook file to use Hatch when executing.
+# TODO: move to it's own env that only installs pre-commit - may need to live here though?
+setup-pre-commit = 'sed -i -e "s/exec /exec hatch run /g" .git/hooks/pre-commit'
+unit-tests = "- python -m pytest {args:tests/unit}"
+lint-all = [
+    "- lint-black",
+    "- lint-flake8",
+    "- lint-mypy",
 ]
+lint-black = "python -m black ."
+lint-flake8 = "python -m flake8 ."
+lint-mypy = "python -m mypy ."
+proto = "protoc -I=./dbt_common/events --python_out=./dbt_common/events ./dbt_common/events/types.proto"
+lint = ["black --check --diff {args:.}", "ruff check {args:.}", "mypy {args:.}"]
+format = ["black {args:.}", "ruff --fix --exit-non-zero-on-fix {args:.}"]
 
 # ruff replaces flake8!
 [tool.ruff]
@@ -108,13 +146,34 @@ ignore = [
 # E501 - Line too long.
 # W292 - No newline at end of file.
 fixable = ["F401", "E501", "W292"]
+exclude = [
+    "dbt_common/events/types_pb2.py",
+    "env*",
+    "third-party-stubs/*",
+]
 
 [tool.ruff.per-file-ignores]
 "types_pb2.py" = ["E501", "E712", "F821"]
 
 [tool.ruff.pydocstyle]
 convention = "google"
 
+[tool.black]
+extend-exclude = "dbt_common/events/types_pb2.py"
+line-length = 99
+target-version = ['py38']
+
+[tool.flake8]
+select = ["E", "W", "F"]
+ignore = ["E203", "E501", "E741", "W503", "W504"]
+exclude = [
+    "dbt_common/events/types_pb2.py",
+    "tests",
+    "venv",
+    "env*"
+]
+per-file-ignores = ["*/__init__.py: F401"]
+
 [tool.mypy]
 mypy_path = "third-party-stubs/"
 namespace_packages = true
@@ -123,14 +182,12 @@ show_error_codes = true
 disable_error_code = "attr-defined"  # TODO: revisit once other mypy errors resolved
 disallow_untyped_defs = false # TODO: add type annotations everywhere
 warn_redundant_casts = true
+exclude = [
+    "dbt_common/events/types_pb2.py",
+    "env*",
+    "third-party-stubs/*",
+]
 
-# Don't run the extensive mypy checks on custom stubs
 [[tool.mypy.overrides]]
-module = ["logbook.*"]
-disallow_untyped_defs = false
-
-[tool.isort]
-profile = "black"
-
-[tool.black]
-line-length = 99
+module = ["dbt_common.events.types_pb2.py"]
+follow_imports = "skip"

tests/unit/test_agate_helper.py

@@ -222,6 +222,5 @@ def test_nocast_bool_01(self):
             [True, Decimal(1)],
             [False, Decimal(0)],
         ]
-
         for i, row in enumerate(tbl):
             self.assertEqual(list(row), expected[i])

tests/unit/test_core_dbt_utils.py

@@ -8,66 +8,66 @@
 
 class TestCommonDbtUtils(unittest.TestCase):
     def test_connection_exception_retry_none(self):
-        Counter._reset()
-        connection_exception_retry(lambda: Counter._add(), 5)
+        Counter._reset(self)
+        connection_exception_retry(lambda: Counter._add(self), 5)
         self.assertEqual(1, counter)
 
     def test_connection_exception_retry_success_requests_exception(self):
-        Counter._reset()
-        connection_exception_retry(lambda: Counter._add_with_requests_exception(), 5)
+        Counter._reset(self)
+        connection_exception_retry(lambda: Counter._add_with_requests_exception(self), 5)
         self.assertEqual(2, counter)  # 2 = original attempt returned None, plus 1 retry
 
     def test_connection_exception_retry_max(self):
-        Counter._reset()
+        Counter._reset(self)
         with self.assertRaises(ConnectionError):
-            connection_exception_retry(lambda: Counter._add_with_exception(), 5)
+            connection_exception_retry(lambda: Counter._add_with_exception(self), 5)
         self.assertEqual(6, counter)  # 6 = original attempt plus 5 retries
 
     def test_connection_exception_retry_success_failed_untar(self):
-        Counter._reset()
-        connection_exception_retry(lambda: Counter._add_with_untar_exception(), 5)
+        Counter._reset(self)
+        connection_exception_retry(lambda: Counter._add_with_untar_exception(self), 5)
         self.assertEqual(2, counter)  # 2 = original attempt returned ReadError, plus 1 retry
 
     def test_connection_exception_retry_success_failed_eofexception(self):
-        Counter._reset()
-        connection_exception_retry(lambda: Counter._add_with_eof_exception(), 5)
+        Counter._reset(self)
+        connection_exception_retry(lambda: Counter._add_with_eof_exception(self), 5)
         self.assertEqual(2, counter)  # 2 = original attempt returned EOFError, plus 1 retry
 
 
 counter: int = 0
 
 
 class Counter:
-    def _add():
+    def _add(self):
         global counter
         counter += 1
 
     # All exceptions that Requests explicitly raises inherit from
     # requests.exceptions.RequestException so we want to make sure that raises plus one exception
     # that inherit from it for sanity
-    def _add_with_requests_exception():
+    def _add_with_requests_exception(self):
         global counter
         counter += 1
         if counter < 2:
             raise requests.exceptions.RequestException
 
-    def _add_with_exception():
+    def _add_with_exception(self):
         global counter
         counter += 1
         raise requests.exceptions.ConnectionError
 
-    def _add_with_untar_exception():
+    def _add_with_untar_exception(self):
         global counter
         counter += 1
         if counter < 2:
             raise tarfile.ReadError
 
-    def _add_with_eof_exception():
+    def _add_with_eof_exception(self):
         global counter
         counter += 1
         if counter < 2:
             raise EOFError
 
-    def _reset():
+    def _reset(self):
         global counter
         counter = 0