From e38ba3040abb58f14a433d61dbc8d703dd57241c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 12 Feb 2024 12:31:20 -0500 Subject: [PATCH 1/6] remove experimental parser tests --- tests/functional/test_experimental_parser.py | 304 ------------------- 1 file changed, 304 deletions(-) delete mode 100644 tests/functional/test_experimental_parser.py diff --git a/tests/functional/test_experimental_parser.py b/tests/functional/test_experimental_parser.py deleted file mode 100644 index 9064ee4f..00000000 --- a/tests/functional/test_experimental_parser.py +++ /dev/null @@ -1,304 +0,0 @@ -import os - -from dbt.context.providers import RefArgs -from dbt.contracts.graph.manifest import Manifest -import pytest - -from tests.functional.utils import run_dbt, run_dbt_and_capture - - -def get_manifest(): - path = "./target/partial_parse.msgpack" - if os.path.exists(path): - with open(path, "rb") as fp: - manifest_mp = fp.read() - manifest: Manifest = Manifest.from_msgpack(manifest_mp) - return manifest - else: - return None - - -basic__schema_yml = """ -version: 2 - -sources: - - name: my_src - schema: "{{ target.schema }}" - tables: - - name: my_tbl - -models: - - name: model_a - columns: - - name: fun - -""" - -basic__model_a_sql = """ -{{ config(tags='hello', x=False) }} -{{ config(tags='world', x=True) }} - -select * from {{ ref('model_b') }} -cross join {{ source('my_src', 'my_tbl') }} -where false as boop - -""" - -basic__model_b_sql = """ -select 1 as fun -""" - - -ref_macro__schema_yml = """ -version: 2 - -""" - -ref_macro__models__model_a_sql = """ -select 1 as id - -""" - -source_macro__macros__source_sql = """ -{% macro source(source_name, table_name) %} - -{% endmacro %} -""" - -source_macro__schema_yml = """ -version: 2 - -""" - -source_macro__models__model_a_sql = """ -select 1 as id - -""" - -config_macro__macros__config_sql = """ -{% macro config() %} - -{% endmacro %} -""" - -config_macro__schema_yml = """ -version: 2 - -""" - -config_macro__models__model_a_sql = """ -select 1 as id - -""" - - -class BasicExperimentalParser: - @pytest.fixture(scope="class") - def models(self): - return { - "model_a.sql": basic__model_a_sql, - "model_b.sql": basic__model_b_sql, - "schema.yml": basic__schema_yml, - } - - -class TestBasicExperimentalParserFlag(BasicExperimentalParser): - @pytest.fixture(scope="class", autouse=True) - def setup(self, project): - os.environ["DBT_USE_EXPERIMENTAL_PARSER"] = "true" - yield - del os.environ["DBT_USE_EXPERIMENTAL_PARSER"] - - def test_env_use_experimental_parser(self, project): - _, log_output = run_dbt_and_capture(["--debug", "parse"]) - - # successful stable static parsing - assert not ("1699: " in log_output) - # successful experimental static parsing - assert "1698: " in log_output - # experimental parser failed - assert not ("1604: " in log_output) - # static parser failed - assert not ("1603: " in log_output) - # jinja rendering - assert not ("1602: " in log_output) - - -class TestBasicStaticParserFlag(BasicExperimentalParser): - @pytest.fixture(scope="class", autouse=True) - def setup(self, project): - os.environ["DBT_STATIC_PARSER"] = "false" - yield - del os.environ["DBT_STATIC_PARSER"] - - def test_env_static_parser(self, project): - _, log_output = run_dbt_and_capture(["--debug", "parse"]) - - print(log_output) - - # jinja rendering because of --no-static-parser - assert "1605: " in log_output - # successful stable static parsing - assert not ("1699: " in log_output) - # successful experimental static parsing - assert not ("1698: " in log_output) - # experimental parser failed - assert not ("1604: " in log_output) - # static parser failed - assert not ("1603: " in log_output) - # fallback jinja rendering - assert not ("1602: " in log_output) - - -class TestBasicExperimentalParser(BasicExperimentalParser): - # test that the experimental parser extracts some basic ref, source, and config calls. - def test_experimental_parser_basic( - self, - project, - ): - run_dbt(["--use-experimental-parser", "parse"]) - manifest = get_manifest() - node = manifest.nodes["model.test.model_a"] - assert node.refs == [RefArgs(name="model_b")] - assert node.sources == [["my_src", "my_tbl"]] - assert node.config._extra == {"x": True} - assert node.config.tags == ["hello", "world"] - - -class TestBasicStaticParser(BasicExperimentalParser): - # test that the static parser extracts some basic ref, source, and config calls by default - # without the experimental flag and without rendering jinja - def test_static_parser_basic(self, project): - _, log_output = run_dbt_and_capture(["--debug", "parse"]) - - # successful stable static parsing - assert "1699: " in log_output - # successful experimental static parsing - assert not ("1698: " in log_output) - # experimental parser failed - assert not ("1604: " in log_output) - # static parser failed - assert not ("1603: " in log_output) - # jinja rendering - assert not ("1602: " in log_output) - - manifest = get_manifest() - node = manifest.nodes["model.test.model_a"] - assert node.refs == [RefArgs(name="model_b")] - assert node.sources == [["my_src", "my_tbl"]] - assert node.config._extra == {"x": True} - assert node.config.tags == ["hello", "world"] - - -class TestBasicNoStaticParser(BasicExperimentalParser): - # test that the static parser doesn't run when the flag is set - def test_static_parser_is_disabled(self, project): - _, log_output = run_dbt_and_capture(["--debug", "--no-static-parser", "parse"]) - - # jinja rendering because of --no-static-parser - assert "1605: " in log_output - # successful stable static parsing - assert not ("1699: " in log_output) - # successful experimental static parsing - assert not ("1698: " in log_output) - # experimental parser failed - assert not ("1604: " in log_output) - # static parser failed - assert not ("1603: " in log_output) - # fallback jinja rendering - assert not ("1602: " in log_output) - - -class TestRefOverrideExperimentalParser: - @pytest.fixture(scope="class") - def models(self): - return { - "model_a.sql": ref_macro__models__model_a_sql, - "schema.yml": ref_macro__schema_yml, - } - - @pytest.fixture(scope="class") - def macros(self): - return { - "source.sql": source_macro__macros__source_sql, - } - - # test that the experimental parser doesn't run if the ref built-in is overriden with a macro - def test_experimental_parser_ref_override( - self, - project, - ): - _, log_output = run_dbt_and_capture(["--debug", "--use-experimental-parser", "parse"]) - - print(log_output) - - # successful experimental static parsing - assert not ("1698: " in log_output) - # fallback to jinja rendering - assert "1602: " in log_output - # experimental parser failed - assert not ("1604: " in log_output) - # didn't run static parser because dbt detected a built-in macro override - assert "1601: " in log_output - - -class TestSourceOverrideExperimentalParser: - @pytest.fixture(scope="class") - def models(self): - return { - "model_a.sql": source_macro__models__model_a_sql, - "schema.yml": source_macro__schema_yml, - } - - @pytest.fixture(scope="class") - def macros(self): - return { - "source.sql": source_macro__macros__source_sql, - } - - # test that the experimental parser doesn't run if the source built-in is overriden with a macro - def test_experimental_parser_source_override( - self, - project, - ): - _, log_output = run_dbt_and_capture(["--debug", "--use-experimental-parser", "parse"]) - - # successful experimental static parsing - assert not ("1698: " in log_output) - # fallback to jinja rendering - assert "1602: " in log_output - # experimental parser failed - assert not ("1604: " in log_output) - # didn't run static parser because dbt detected a built-in macro override - assert "1601: " in log_output - - -class TestConfigOverrideExperimentalParser: - @pytest.fixture(scope="class") - def models(self): - return { - "model_a.sql": config_macro__models__model_a_sql, - "schema.yml": config_macro__schema_yml, - } - - @pytest.fixture(scope="class") - def macros(self): - return { - "config.sql": config_macro__macros__config_sql, - } - - # test that the experimental parser doesn't run if the config built-in is overriden with a macro - def test_experimental_parser_config_override( - self, - project, - ): - _, log_output = run_dbt_and_capture(["--debug", "--use-experimental-parser", "parse"]) - - # successful experimental static parsing - assert not ("1698: " in log_output) - # fallback to jinja rendering - assert "1602: " in log_output - # experimental parser failed - assert not ("1604: " in log_output) - # didn't run static parser because dbt detected a built-in macro override - assert "1601: " in log_output From 1fb5a00640b65b6721ebee0214d7ad3d7ea9024b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 12 Feb 2024 14:04:50 -0500 Subject: [PATCH 2/6] migrate get catalog unit tests to dbt-adapters --- tests/unit/test_adapter.py | 46 -------------------------------------- 1 file changed, 46 deletions(-) diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index be2eef14..d73ed54c 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -1,10 +1,6 @@ -import dataclasses from multiprocessing import get_context from unittest import TestCase, mock -import agate -from dbt.adapters.base import BaseRelation -from dbt.adapters.contracts.relation import Path from dbt_common.exceptions import DbtValidationError from dbt.adapters.postgres import Plugin as PostgresPlugin, PostgresAdapter @@ -306,45 +302,3 @@ def test_set_zero_keepalive(self, psycopg2): connect_timeout=10, application_name="dbt", ) - - @mock.patch.object(PostgresAdapter, "execute_macro") - @mock.patch.object(PostgresAdapter, "_get_catalog_relations") - def test_get_catalog_various_schemas(self, mock_get_relations, mock_execute): - self.catalog_test(mock_get_relations, mock_execute, False) - - @mock.patch.object(PostgresAdapter, "execute_macro") - @mock.patch.object(PostgresAdapter, "_get_catalog_relations") - def test_get_filtered_catalog(self, mock_get_relations, mock_execute): - self.catalog_test(mock_get_relations, mock_execute, True) - - def catalog_test(self, mock_get_relations, mock_execute, filtered=False): - column_names = ["table_database", "table_schema", "table_name"] - relations = [ - BaseRelation(path=Path(database="dbt", schema="foo", identifier="bar")), - BaseRelation(path=Path(database="dbt", schema="FOO", identifier="baz")), - BaseRelation(path=Path(database="dbt", schema=None, identifier="bar")), - BaseRelation(path=Path(database="dbt", schema="quux", identifier="bar")), - BaseRelation(path=Path(database="dbt", schema="skip", identifier="bar")), - ] - rows = list(map(lambda x: dataclasses.astuple(x.path), relations)) - mock_execute.return_value = agate.Table(rows=rows, column_names=column_names) - - mock_get_relations.return_value = relations - - relation_configs = [] - used_schemas = {("dbt", "foo"), ("dbt", "quux")} - - if filtered: - catalog, exceptions = self.adapter.get_filtered_catalog( - relation_configs, used_schemas, set([relations[0], relations[3]]) - ) - else: - catalog, exceptions = self.adapter.get_catalog(relation_configs, used_schemas) - - tupled_catalog = set(map(tuple, catalog)) - if filtered: - self.assertEqual(tupled_catalog, {rows[0], rows[3]}) - else: - self.assertEqual(tupled_catalog, {rows[0], rows[1], rows[3]}) - - self.assertEqual(exceptions, []) From 397b8557a31161a6b8f01b75413f2b789fc7f594 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 12 Feb 2024 16:37:50 -0500 Subject: [PATCH 3/6] add get catalog unit tests back, run set_invocation_context first --- tests/unit/test_adapter.py | 48 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/unit/test_adapter.py b/tests/unit/test_adapter.py index d73ed54c..1c375ac0 100644 --- a/tests/unit/test_adapter.py +++ b/tests/unit/test_adapter.py @@ -1,6 +1,11 @@ +import dataclasses from multiprocessing import get_context from unittest import TestCase, mock +import agate +from dbt.adapters.base import BaseRelation +from dbt.adapters.contracts.relation import Path +from dbt_common.context import set_invocation_context from dbt_common.exceptions import DbtValidationError from dbt.adapters.postgres import Plugin as PostgresPlugin, PostgresAdapter @@ -302,3 +307,46 @@ def test_set_zero_keepalive(self, psycopg2): connect_timeout=10, application_name="dbt", ) + + @mock.patch.object(PostgresAdapter, "execute_macro") + @mock.patch.object(PostgresAdapter, "_get_catalog_relations") + def test_get_catalog_various_schemas(self, mock_get_relations, mock_execute): + self.catalog_test(mock_get_relations, mock_execute, False) + + @mock.patch.object(PostgresAdapter, "execute_macro") + @mock.patch.object(PostgresAdapter, "_get_catalog_relations") + def test_get_filtered_catalog(self, mock_get_relations, mock_execute): + self.catalog_test(mock_get_relations, mock_execute, True) + + def catalog_test(self, mock_get_relations, mock_execute, filtered=False): + column_names = ["table_database", "table_schema", "table_name"] + relations = [ + BaseRelation(path=Path(database="dbt", schema="foo", identifier="bar")), + BaseRelation(path=Path(database="dbt", schema="FOO", identifier="baz")), + BaseRelation(path=Path(database="dbt", schema=None, identifier="bar")), + BaseRelation(path=Path(database="dbt", schema="quux", identifier="bar")), + BaseRelation(path=Path(database="dbt", schema="skip", identifier="bar")), + ] + rows = list(map(lambda x: dataclasses.astuple(x.path), relations)) + mock_execute.return_value = agate.Table(rows=rows, column_names=column_names) + + mock_get_relations.return_value = relations + + relation_configs = [] + used_schemas = {("dbt", "foo"), ("dbt", "quux")} + + set_invocation_context({}) + if filtered: + catalog, exceptions = self.adapter.get_filtered_catalog( + relation_configs, used_schemas, set([relations[0], relations[3]]) + ) + else: + catalog, exceptions = self.adapter.get_catalog(relation_configs, used_schemas) + + tupled_catalog = set(map(tuple, catalog)) + if filtered: + self.assertEqual(tupled_catalog, {rows[0], rows[3]}) + else: + self.assertEqual(tupled_catalog, {rows[0], rows[1], rows[3]}) + + self.assertEqual(exceptions, []) From 15897beedc416db117d3f458f68da321dd637496 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 12 Feb 2024 17:37:08 -0500 Subject: [PATCH 4/6] move all dependencies into hatch dependencies, point to dev dependencies for testing --- pyproject.toml | 99 ++++++++++++++++++++++---------------------------- 1 file changed, 43 insertions(+), 56 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3240b861..71ef119c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,44 +23,12 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ - "dbt-adapters", - "psycopg2~=2.9", - # installed via dbt-adapters but used directly, unpin minor to avoid version conflicts + "dbt-adapters>=0.1.0a6,<0.2.0", + "psycopg2>=2.9,<3.0", + # installed via dbt-adapters but used directly "dbt-common<1.0", - "agate<2.0", + "agate>=1.0,<2.0", ] -[project.optional-dependencies] -dev = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", -] -lint = [ - "black", - "flake8", - "Flake8-pyproject", -] -typecheck = [ - "mypy", - "types-protobuf", - "types-pytz", -] -test = [ - # TODO: remove `dbt-core` dependencies from unit tests - "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", - "freezegun", - "pytest", - "pytest-dotenv", - "pytest-mock", - "pytest-xdist", -] -integration = [ - "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", -] -build = [ - "wheel", - "twine", - "check-wheel-contents", -] - [project.urls] Homepage = "https://github.com/dbt-labs/dbt-postgres" Documentation = "https://docs.getdbt.com" @@ -72,11 +40,6 @@ Changelog = "https://github.com/dbt-labs/dbt-postgres/blob/main/CHANGELOG.md" requires = ["hatchling"] build-backend = "hatchling.build" -# TODO: this is needed to install from github in optoinal-dependencies -# alternatively, we can stick the github dependencies directly in the hatch envs -[tool.hatch.metadata] -allow-direct-references = true - [tool.hatch.build.targets.sdist] include = ["dbt"] @@ -87,44 +50,68 @@ packages = ["dbt"] path = "dbt/adapters/postgres/__version__.py" [tool.hatch.envs.default] -features = [ - "lint", - "typecheck", - "test", - "integration", - "build", +dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", ] [tool.hatch.envs.lint] detached = true -features = ["lint"] +dependencies = [ + "black", + "flake8", + "Flake8-pyproject", +] [tool.hatch.envs.lint.scripts] -all = ["black", "flake8"] +all = [ + "black", + "flake8", +] black = "python -m black ." flake8 = "python -m flake8 ." [tool.hatch.envs.typecheck] -features = ["typecheck"] +dependencies = [ + "mypy", + "types-protobuf", + "types-pytz", +] [tool.hatch.envs.typecheck.scripts] all = "python -m mypy ." [tool.hatch.envs.unit-tests] -# TODO: confirm this works for production testing or add appropriate hatch envs -features = ["dev", "test"] +dependencies = [ + # TODO: remove `dbt-core` dependencies from unit tests + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", + "freezegun", + "pytest", + "pytest-dotenv", + "pytest-mock", + "pytest-xdist", +] [tool.hatch.envs.unit-tests.scripts] all = "python -m pytest {args:tests/unit}" [tool.hatch.envs.integration-tests] -# TODO: confirm this works for production testing or add appropriate hatch envs -features = ["dev", "test", "integration"] +template = "unit-tests" +extra-dependencies = [ + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", +] [tool.hatch.envs.integration-tests.scripts] all = "python -m pytest {args:tests/functional}" [tool.hatch.envs.build] detached = true -features = ["build"] +dependencies = [ + "wheel", + "twine", + "check-wheel-contents", +] [tool.hatch.envs.build.scripts] -check-all = ["- check-wheel", "- check-sdist"] +check-all = [ + "- check-wheel", + "- check-sdist", +] check-wheel = [ "twine check dist/*", "find ./dist/dbt_postgres-*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", From 58ba4fe4dd6e83a68796dfb1f42c1ac9c801b0a5 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 12 Feb 2024 17:37:35 -0500 Subject: [PATCH 5/6] point to feature branch for dbt-adapters for testing --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 71ef119c..0d82cca8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ path = "dbt/adapters/postgres/__version__.py" [tool.hatch.envs.default] dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@tests/get-catalog", "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", ] @@ -95,7 +95,7 @@ all = "python -m pytest {args:tests/unit}" [tool.hatch.envs.integration-tests] template = "unit-tests" extra-dependencies = [ - "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git@tests/get-catalog#subdirectory=dbt-tests-adapter", ] [tool.hatch.envs.integration-tests.scripts] all = "python -m pytest {args:tests/functional}" From d9f2f9c831dd29ea22e19ee2a11104386843219f Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Tue, 13 Feb 2024 17:58:59 -0500 Subject: [PATCH 6/6] point back to main --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0d82cca8..71ef119c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,7 @@ path = "dbt/adapters/postgres/__version__.py" [tool.hatch.envs.default] dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@tests/get-catalog", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", "dbt_common @ git+https://github.com/dbt-labs/dbt-common.git", ] @@ -95,7 +95,7 @@ all = "python -m pytest {args:tests/unit}" [tool.hatch.envs.integration-tests] template = "unit-tests" extra-dependencies = [ - "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git@tests/get-catalog#subdirectory=dbt-tests-adapter", + "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", ] [tool.hatch.envs.integration-tests.scripts] all = "python -m pytest {args:tests/functional}"