From 8d16c4ddf3d5994e11f92b9f96a308580c86053d Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Fri, 30 Sep 2022 14:49:22 -0500 Subject: [PATCH 1/5] init pr 051 test conversion in bigquery --- dev-requirements.txt | 4 +-- .../query_comment_test/test_query_comment.py | 29 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 tests/functional/adapter/query_comment_test/test_query_comment.py diff --git a/dev-requirements.txt b/dev-requirements.txt index 676703d3e..06f410d57 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git@mcknight/CT-1271#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@mcknight/CT-1271#egg=dbt-tests-adapter&subdirectory=tests/adapter black==22.8.0 bumpversion diff --git a/tests/functional/adapter/query_comment_test/test_query_comment.py b/tests/functional/adapter/query_comment_test/test_query_comment.py new file mode 100644 index 000000000..8c4f710a8 --- /dev/null +++ b/tests/functional/adapter/query_comment_test/test_query_comment.py @@ -0,0 +1,29 @@ +import pytest + +from dbt.tests.adapter.query_comment.test_query_comment import ( + BaseQueryComments, + BaseMacroQueryComments, + BaseMacroArgsQueryComments, + BaseMacroInvalidQueryComments, + BaseNullQueryComments, + BaseEmptyQueryComments, +) + + +class TestQueryCommentsBigQuery(BaseQueryComments): + pass + +class TestMacroQueryCommentsBigQuery(BaseMacroQueryComments): + pass + +class TestMacroArgsQueryCommentsBigQuery(BaseMacroArgsQueryComments): + pass + +class TestMacroInvalidQueryCommentsBigQuery(BaseMacroInvalidQueryComments): + pass + +class TestNullQueryCommentsBigQuery(BaseNullQueryComments): + pass + +class TestEmptyQueryCommentsBigQuery(BaseEmptyQueryComments): + pass \ No newline at end of file From 6d8511720fcef4161e3ee14965960b3ad24b538d Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Mon, 3 Oct 2022 09:38:54 -0500 Subject: [PATCH 2/5] update to main From 8abc1e526230600edc98f86b7002891031e42435 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Mon, 3 Oct 2022 11:05:20 -0500 Subject: [PATCH 3/5] remove old test --- .../query_comments_test/macros/macro.sql | 25 --- .../query_comments_test/models/x.sql | 24 --- .../test_query_comments.py | 169 ------------------ 3 files changed, 218 deletions(-) delete mode 100644 tests/integration/query_comments_test/macros/macro.sql delete mode 100644 tests/integration/query_comments_test/models/x.sql delete mode 100644 tests/integration/query_comments_test/test_query_comments.py diff --git a/tests/integration/query_comments_test/macros/macro.sql b/tests/integration/query_comments_test/macros/macro.sql deleted file mode 100644 index 6e8a1900c..000000000 --- a/tests/integration/query_comments_test/macros/macro.sql +++ /dev/null @@ -1,25 +0,0 @@ -{%- macro query_header_no_args() -%} -{%- set x = "are pretty cool" -%} -{{ "dbt macros" }} -{{ x }} -{%- endmacro -%} - - -{%- macro query_header_args(message) -%} - {%- set comment_dict = dict( - app='dbt++', - macro_version='0.1.0', - dbt_version=dbt_version, - message='blah: '~ message) -%} - {{ return(comment_dict) }} -{%- endmacro -%} - - -{%- macro ordered_to_json(dct) -%} -{{ tojson(dct, sort_keys=True) }} -{%- endmacro %} - - -{% macro invalid_query_header() -%} -{{ "Here is an invalid character for you: */" }} -{% endmacro %} diff --git a/tests/integration/query_comments_test/models/x.sql b/tests/integration/query_comments_test/models/x.sql deleted file mode 100644 index e719a30b5..000000000 --- a/tests/integration/query_comments_test/models/x.sql +++ /dev/null @@ -1,24 +0,0 @@ -{% set blacklist = ['pass', 'password', 'keyfile', 'keyfile.json', 'password', 'private_key_passphrase'] %} -{% for key in blacklist %} - {% if key in blacklist and blacklist[key] %} - {% do exceptions.raise_compiler_error('invalid target, found banned key "' ~ key ~ '"') %} - {% endif %} -{% endfor %} - -{% if 'type' not in target %} - {% do exceptions.raise_compiler_error('invalid target, missing "type"') %} -{% endif %} - -{% set required = ['name', 'schema', 'type', 'threads'] %} - -{# Require what we document at https://docs.getdbt.com/docs/target #} -{% do required.extend(['project']) %} - -{% for value in required %} - {% if value not in target %} - {% do exceptions.raise_compiler_error('invalid target, missing "' ~ value ~ '"') %} - {% endif %} -{% endfor %} - -{% do run_query('select 2 as inner_id') %} -select 1 as outer_id diff --git a/tests/integration/query_comments_test/test_query_comments.py b/tests/integration/query_comments_test/test_query_comments.py deleted file mode 100644 index bc813a6ae..000000000 --- a/tests/integration/query_comments_test/test_query_comments.py +++ /dev/null @@ -1,169 +0,0 @@ -from tests.integration.base import DBTIntegrationTest, use_profile -import io -import json -import os -import re - -import dbt.exceptions -from dbt.version import __version__ as dbt_version -from dbt.logger import log_manager - - -class TestDefaultQueryComments(DBTIntegrationTest): - def matches_comment(self, msg) -> bool: - if not msg.startswith('/* '): - return False - # our blob is the first line of the query comments, minus the comment - json_str = msg.split('\n')[0][3:-3] - data = json.loads(json_str) - return ( - data['app'] == 'dbt' and - data['dbt_version'] == dbt_version and - data['node_id'] == 'model.test.x' - ) - - @property - def project_config(self): - return { - 'config-version': 2, - 'macro-paths': ['macros'] - } - - @property - def schema(self): - return 'dbt_query_comments' - - @staticmethod - def dir(value): - return os.path.normpath(value) - - @property - def models(self): - return self.dir('models') - - def setUp(self): - super().setUp() - self.initial_stdout = log_manager.stdout - self.initial_stderr = log_manager.stderr - self.stringbuf = io.StringIO() - log_manager.set_output_stream(self.stringbuf) - - def tearDown(self): - log_manager.set_output_stream(self.initial_stdout, self.initial_stderr) - super().tearDown() - - def run_get_json(self, expect_pass=True): - res, raw_logs = self.run_dbt_and_capture( - ['--debug', '--log-format=json', 'run'], - expect_pass=expect_pass - ) - parsed_logs = [] - for line in raw_logs.split('\n'): - try: - log = json.loads(line) - except ValueError: - continue - - parsed_logs.append(log) - - # empty lists evaluate as False - self.assertTrue(parsed_logs) - return parsed_logs - - def query_comment(self, model_name, log): - log_msg = re.sub("(?:[01]\d|2[0123]):(?:[012345]\d):(?:[012345]\d \| )", "", log['msg']) - prefix = 'On {}: '.format(model_name) - - if log_msg.startswith(prefix): - msg = log_msg[len(prefix):] - if msg in {'COMMIT', 'BEGIN', 'ROLLBACK'}: - return None - return msg - return None - - def run_assert_comments(self): - logs = self.run_get_json() - - seen = False - for log in logs: - msg = self.query_comment('model.test.x', log) - if msg is not None and self.matches_comment(msg): - seen = True - - self.assertTrue(seen, 'Never saw a matching log message! Logs:\n{}'.format('\n'.join(l['msg'] for l in logs))) - - @use_profile('bigquery') - def test_bigquery_comments(self): - self.run_assert_comments() - - -class TestQueryComments(TestDefaultQueryComments): - @property - def project_config(self): - cfg = super().project_config - cfg.update({'query-comment': 'dbt\nrules!\n'}) - return cfg - - def matches_comment(self, msg) -> bool: - return msg.startswith('/* dbt\nrules! */\n') - - -class TestMacroQueryComments(TestDefaultQueryComments): - @property - def project_config(self): - cfg = super().project_config - cfg.update({'query-comment': '{{ query_header_no_args() }}'}) - return cfg - - def matches_comment(self, msg) -> bool: - start_with = '/* dbt macros\nare pretty cool */\n' - return msg.startswith(start_with) - - -class TestMacroArgsQueryComments(TestDefaultQueryComments): - @property - def project_config(self): - cfg = super().project_config - cfg.update( - {'query-comment': '{{ return(ordered_to_json(query_header_args(target.name))) }}'} - ) - return cfg - - def matches_comment(self, msg) -> bool: - expected_dct = {'app': 'dbt++', 'dbt_version': dbt_version, 'macro_version': '0.1.0', 'message': 'blah: default2'} - expected = '/* {} */\n'.format(json.dumps(expected_dct, sort_keys=True)) - return msg.startswith(expected) - - -class TestMacroInvalidQueryComments(TestDefaultQueryComments): - @property - def project_config(self): - cfg = super().project_config - cfg.update({'query-comment': '{{ invalid_query_header() }}'}) - return cfg - - def run_assert_comments(self): - with self.assertRaises(dbt.exceptions.RuntimeException): - self.run_get_json(expect_pass=False) - - -class TestNullQueryComments(TestDefaultQueryComments): - @property - def project_config(self): - cfg = super().project_config - cfg.update({'query-comment': ''}) - return cfg - - def matches_comment(self, msg) -> bool: - return not ('/*' in msg or '*/' in msg) - - -class TestEmptyQueryComments(TestDefaultQueryComments): - @property - def project_config(self): - cfg = super().project_config - cfg.update({'query-comment': ''}) - return cfg - - def matches_comment(self, msg) -> bool: - return not ('/*' in msg or '*/' in msg) From 06fcff5bff69cbf9c4370095a043951063a253e4 Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 11 Oct 2022 14:59:57 -0500 Subject: [PATCH 4/5] point back to main in core --- dev-requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 06f410d57..676703d3e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@mcknight/CT-1271#egg=dbt-core&subdirectory=core -git+https://github.com/dbt-labs/dbt-core.git@mcknight/CT-1271#egg=dbt-tests-adapter&subdirectory=tests/adapter +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter black==22.8.0 bumpversion From a22f88252408889def788d5fd63c32cf48ba002f Mon Sep 17 00:00:00 2001 From: Matthew McKnight Date: Tue, 11 Oct 2022 15:36:02 -0500 Subject: [PATCH 5/5] trying to fix the custom quota error --- .../functional/adapter/query_comment_test/test_query_comment.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/adapter/query_comment_test/test_query_comment.py b/tests/functional/adapter/query_comment_test/test_query_comment.py index 8c4f710a8..aa7061150 100644 --- a/tests/functional/adapter/query_comment_test/test_query_comment.py +++ b/tests/functional/adapter/query_comment_test/test_query_comment.py @@ -1,5 +1,4 @@ import pytest - from dbt.tests.adapter.query_comment.test_query_comment import ( BaseQueryComments, BaseMacroQueryComments,