Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Point testing dependencies to main #12

Merged
merged 7 commits into from
Feb 15, 2024
Merged

Point testing dependencies to main #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e38ba30
remove experimental parser tests
mikealfare Feb 12, 2024
1fb5a00
migrate get catalog unit tests to dbt-adapters
mikealfare Feb 12, 2024
397b855
add get catalog unit tests back, run set_invocation_context first
mikealfare Feb 12, 2024
15897be
move all dependencies into hatch dependencies, point to dev dependenc…
mikealfare Feb 12, 2024
58ba4fe
point to feature branch for dbt-adapters for testing
mikealfare Feb 12, 2024
4baa89a
Merge branch 'main' into tests/experimental-parser
mikealfare Feb 12, 2024
d9f2f9c
point back to main
mikealfare Feb 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 43 additions & 56 deletions pyproject.toml
View file
Open in desktop
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Dependencies were moved down into the hatch environments, in alignment with dbt-adapters. This was done for a combination of reasons:

  • we cannot publish a package to PyPI with dependencies installed from GitHub, so development versions of internal packages would need to be identified as hatch environment dependencies
  • there is not a lot of value in installing dev and test tooling without the development versions of internal packages
  • having just the tooling as installable extras in the production distribution is confusing

Original file line number Diff line number Diff line change
Expand Up @@ -23,44 +23,12 @@ classifiers = [
"Programming Language :: Python :: 3.11",
]
dependencies = [
"dbt-adapters",
"psycopg2~=2.9",
# installed via dbt-adapters but used directly, unpin minor to avoid version conflicts
"dbt-adapters>=0.1.0a6,<0.2.0",
"psycopg2>=2.9,<3.0",
# installed via dbt-adapters but used directly
"dbt-common<1.0",
"agate<2.0",
"agate>=1.0,<2.0",
]
[project.optional-dependencies]
dev = [
"dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git",
]
lint = [
"black",
"flake8",
"Flake8-pyproject",
]
typecheck = [
"mypy",
"types-protobuf",
"types-pytz",
]
test = [
# TODO: remove `dbt-core` dependencies from unit tests
"dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core",
"freezegun",
"pytest",
"pytest-dotenv",
"pytest-mock",
"pytest-xdist",
]
integration = [
"dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter",
]
build = [
"wheel",
"twine",
"check-wheel-contents",
]

[project.urls]
Homepage = "https://github.com/dbt-labs/dbt-postgres"
Documentation = "https://docs.getdbt.com"
Expand All @@ -72,11 +40,6 @@ Changelog = "https://github.com/dbt-labs/dbt-postgres/blob/main/CHANGELOG.md"
requires = ["hatchling"]
build-backend = "hatchling.build"

# TODO: this is needed to install from github in optoinal-dependencies
# alternatively, we can stick the github dependencies directly in the hatch envs
[tool.hatch.metadata]
allow-direct-references = true

[tool.hatch.build.targets.sdist]
include = ["dbt"]

Expand All @@ -87,44 +50,68 @@ packages = ["dbt"]
path = "dbt/adapters/postgres/__version__.py"

[tool.hatch.envs.default]
features = [
"lint",
"typecheck",
"test",
"integration",
"build",
dependencies = [
"dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git",
"dbt_common @ git+https://github.com/dbt-labs/dbt-common.git",
]

[tool.hatch.envs.lint]
detached = true
features = ["lint"]
dependencies = [
"black",
"flake8",
"Flake8-pyproject",
]
[tool.hatch.envs.lint.scripts]
all = ["black", "flake8"]
all = [
"black",
"flake8",
]
black = "python -m black ."
flake8 = "python -m flake8 ."

[tool.hatch.envs.typecheck]
features = ["typecheck"]
dependencies = [
"mypy",
"types-protobuf",
"types-pytz",
]
[tool.hatch.envs.typecheck.scripts]
all = "python -m mypy ."

[tool.hatch.envs.unit-tests]
# TODO: confirm this works for production testing or add appropriate hatch envs
features = ["dev", "test"]
dependencies = [
# TODO: remove `dbt-core` dependencies from unit tests
"dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

kinda surprised we don't need dbt-tests-adapter for unit tests

"freezegun",
"pytest",
"pytest-dotenv",
"pytest-mock",
"pytest-xdist",
]
[tool.hatch.envs.unit-tests.scripts]
all = "python -m pytest {args:tests/unit}"

[tool.hatch.envs.integration-tests]
# TODO: confirm this works for production testing or add appropriate hatch envs
features = ["dev", "test", "integration"]
template = "unit-tests"
extra-dependencies = [
"dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter",
]
[tool.hatch.envs.integration-tests.scripts]
all = "python -m pytest {args:tests/functional}"

[tool.hatch.envs.build]
detached = true
features = ["build"]
dependencies = [
"wheel",
"twine",
"check-wheel-contents",
]
[tool.hatch.envs.build.scripts]
check-all = ["- check-wheel", "- check-sdist"]
check-all = [
"- check-wheel",
"- check-sdist",
]
check-wheel = [
"twine check dist/*",
"find ./dist/dbt_postgres-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/",
Expand Down
48 changes: 48 additions & 0 deletions tests/unit/test_adapter.py
View file
Open in desktop
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The test cases added here are being migrated back to dbt-postgres (for now). It was determined that the issue was likely being caused by a combination of not looking at main for internal dependencies and not instantiating the invocation context variables before checking them. set_invocation_context({}) is run on line 338 now, right before getting the catalog, and tests are now passing.

Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
import dataclasses
from multiprocessing import get_context
from unittest import TestCase, mock

import agate
from dbt.adapters.base import BaseRelation
from dbt.adapters.contracts.relation import Path
from dbt_common.context import set_invocation_context
from dbt_common.exceptions import DbtValidationError

from dbt.adapters.postgres import Plugin as PostgresPlugin, PostgresAdapter
Expand Down Expand Up @@ -302,3 +307,46 @@ def test_set_zero_keepalive(self, psycopg2):
connect_timeout=10,
application_name="dbt",
)

@mock.patch.object(PostgresAdapter, "execute_macro")
@mock.patch.object(PostgresAdapter, "_get_catalog_relations")
def test_get_catalog_various_schemas(self, mock_get_relations, mock_execute):
self.catalog_test(mock_get_relations, mock_execute, False)

@mock.patch.object(PostgresAdapter, "execute_macro")
@mock.patch.object(PostgresAdapter, "_get_catalog_relations")
def test_get_filtered_catalog(self, mock_get_relations, mock_execute):
self.catalog_test(mock_get_relations, mock_execute, True)

def catalog_test(self, mock_get_relations, mock_execute, filtered=False):
column_names = ["table_database", "table_schema", "table_name"]
relations = [
BaseRelation(path=Path(database="dbt", schema="foo", identifier="bar")),
BaseRelation(path=Path(database="dbt", schema="FOO", identifier="baz")),
BaseRelation(path=Path(database="dbt", schema=None, identifier="bar")),
BaseRelation(path=Path(database="dbt", schema="quux", identifier="bar")),
BaseRelation(path=Path(database="dbt", schema="skip", identifier="bar")),
]
rows = list(map(lambda x: dataclasses.astuple(x.path), relations))
mock_execute.return_value = agate.Table(rows=rows, column_names=column_names)

mock_get_relations.return_value = relations

relation_configs = []
used_schemas = {("dbt", "foo"), ("dbt", "quux")}

set_invocation_context({})
if filtered:
catalog, exceptions = self.adapter.get_filtered_catalog(
relation_configs, used_schemas, set([relations[0], relations[3]])
)
else:
catalog, exceptions = self.adapter.get_catalog(relation_configs, used_schemas)

tupled_catalog = set(map(tuple, catalog))
if filtered:
self.assertEqual(tupled_catalog, {rows[0], rows[3]})
else:
self.assertEqual(tupled_catalog, {rows[0], rows[1], rows[3]})

self.assertEqual(exceptions, [])
Loading