From 0e6ac5baf1aa8290aed478f489a008edaddd131a Mon Sep 17 00:00:00 2001 From: Kyle Wigley Date: Wed, 18 Nov 2020 14:30:30 -0500 Subject: [PATCH] can we just default materialization to `test`? --- core/dbt/contracts/graph/model_config.py | 1 + .../test_docs_generate.py | 81 +++++--- test/integration/047_dbt_ls_test/test_ls.py | 6 +- test/unit/test_contracts_graph_compiled.py | 46 +++-- test/unit/test_contracts_graph_parsed.py | 4 +- test/unit/test_graph_selector_methods.py | 182 ++++++++++++------ 6 files changed, 206 insertions(+), 114 deletions(-) diff --git a/core/dbt/contracts/graph/model_config.py b/core/dbt/contracts/graph/model_config.py index d712a809b5e..d15440ff257 100644 --- a/core/dbt/contracts/graph/model_config.py +++ b/core/dbt/contracts/graph/model_config.py @@ -450,6 +450,7 @@ class SeedConfig(NodeConfig): @dataclass class TestConfig(NodeConfig): + materialized: str = 'test' severity: Severity = Severity('ERROR') diff --git a/test/integration/029_docs_generate_tests/test_docs_generate.py b/test/integration/029_docs_generate_tests/test_docs_generate.py index 8e5c9ce9600..56907b3e259 100644 --- a/test/integration/029_docs_generate_tests/test_docs_generate.py +++ b/test/integration/029_docs_generate_tests/test_docs_generate.py @@ -91,7 +91,8 @@ def test_postgres_include_schema(self): self.assertEqual(len(manifest['nodes']), 1) self.assertIn('model.test.model', manifest['nodes']) self.assertIn('schema', manifest['nodes']['model.test.model']) - self.assertEqual('pg', manifest['nodes']['model.test.model']['schema'][:2]) + self.assertEqual('pg', manifest['nodes'] + ['model.test.model']['schema'][:2]) class TestDocsGenerate(DBTIntegrationTest): @@ -117,7 +118,6 @@ def tearDown(self): super().tearDown() del os.environ['DBT_ENV_CUSTOM_ENV_env_key'] - @property def schema(self): return 'docs_generate_029' @@ -209,11 +209,11 @@ def _redshift_stats(self): "include": True }, "diststyle": { - "id": "diststyle", - "label": "Dist Style", - "value": AnyStringWith('AUTO'), - "description": "Distribution style or distribution key column, if key distribution is defined.", - "include": True + "id": "diststyle", + "label": "Dist Style", + "value": AnyStringWith('AUTO'), + "description": "Distribution style or distribution key column, if key distribution is defined.", + "include": True }, "max_varchar": { "id": "max_varchar", @@ -349,7 +349,7 @@ def _expected_catalog(self, id_type, text_type, time_type, view_type, table_type, model_stats, seed_stats=None, case=None, case_columns=False, model_database=None): if case is None: - case = lambda x: x + def case(x): return x col_case = case if case_columns else lambda x: x if seed_stats is None: @@ -888,7 +888,8 @@ def verify_catalog(self, expected): assert set(catalog) == {'errors', 'metadata', 'nodes', 'sources'} - self.verify_metadata(catalog['metadata'], 'https://schemas.getdbt.com/dbt/catalog/v1.json') + self.verify_metadata( + catalog['metadata'], 'https://schemas.getdbt.com/dbt/catalog/v1.json') assert not catalog['errors'] for key in 'nodes', 'sources': @@ -988,7 +989,7 @@ def rendered_tst_config(self, **updates): result = { 'column_types': {}, 'enabled': True, - 'materialized': 'view', + 'materialized': 'test', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], @@ -1071,10 +1072,13 @@ def expected_seeded_manifest(self, model_database=None, quote_model=False): model_database = self.alternative_database model_config = self.rendered_model_config(database=model_database) - second_config = self.rendered_model_config(schema=self.alternate_schema[-4:]) + second_config = self.rendered_model_config( + schema=self.alternate_schema[-4:]) - unrendered_model_config = self.unrendered_model_config(database=model_database, materialized='view') - unrendered_second_config = self.unrendered_model_config(schema=self.alternate_schema[-4:], materialized='view') + unrendered_model_config = self.unrendered_model_config( + database=model_database, materialized='view') + unrendered_second_config = self.unrendered_model_config( + schema=self.alternate_schema[-4:], materialized='view') seed_config = self.rendered_seed_config() unrendered_seed_config = self.unrendered_seed_config() @@ -2143,7 +2147,8 @@ def expected_bigquery_complex_manifest(self): 'config': self.rendered_model_config( cluster_by=['first_name'], materialized='table', - partition_by={'field': 'updated_at', 'data_type': 'date'}, + partition_by={'field': 'updated_at', + 'data_type': 'date'}, ), 'sources': [], 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, @@ -2219,7 +2224,8 @@ def expected_bigquery_complex_manifest(self): 'unrendered_config': self.unrendered_model_config( cluster_by=['first_name'], materialized='table', - partition_by={'field': 'updated_at', 'data_type': 'date'}, + partition_by={'field': 'updated_at', + 'data_type': 'date'}, ), }, 'model.test.multi_clustered': { @@ -2228,7 +2234,8 @@ def expected_bigquery_complex_manifest(self): 'config': self.rendered_model_config( cluster_by=['first_name', 'email'], materialized='table', - partition_by={'field': 'updated_at', 'data_type': 'date'} + partition_by={'field': 'updated_at', + 'data_type': 'date'} ), 'sources': [], 'depends_on': {'macros': [], 'nodes': ['seed.test.seed']}, @@ -2303,7 +2310,8 @@ def expected_bigquery_complex_manifest(self): 'unrendered_config': self.unrendered_model_config( cluster_by=['first_name', 'email'], materialized='table', - partition_by={'field': 'updated_at', 'data_type': 'date'} + partition_by={'field': 'updated_at', + 'data_type': 'date'} ), }, 'model.test.nested_view': { @@ -2864,12 +2872,16 @@ def verify_manifest(self, expected_manifest): for key in manifest_keys: if key == 'macros': - self.verify_manifest_macros(manifest, expected_manifest.get('macros')) + self.verify_manifest_macros( + manifest, expected_manifest.get('macros')) elif key == 'metadata': metadata = manifest['metadata'] - self.verify_metadata(metadata, 'https://schemas.getdbt.com/dbt/manifest/v1.json') - assert 'project_id' in metadata and metadata['project_id'] == '098f6bcd4621d373cade4e832627b4f6' - assert 'send_anonymous_usage_stats' in metadata and metadata['send_anonymous_usage_stats'] is False + self.verify_metadata( + metadata, 'https://schemas.getdbt.com/dbt/manifest/v1.json') + assert 'project_id' in metadata and metadata[ + 'project_id'] == '098f6bcd4621d373cade4e832627b4f6' + assert 'send_anonymous_usage_stats' in metadata and metadata[ + 'send_anonymous_usage_stats'] is False assert 'user_id' in metadata and metadata['user_id'] is None assert 'adapter_type' in metadata and metadata['adapter_type'] == self.adapter_type else: @@ -2895,9 +2907,12 @@ def expected_run_results(self, quote_schema=True, quote_model=False, model_database = self.alternative_database model_config = self.rendered_model_config(database=model_database) - second_model_config = self.rendered_model_config(schema=self.alternate_schema[-4:]) - unrendered_model_config = self.unrendered_model_config(database=model_database, materialized='view') - unrendered_second_model_config = self.unrendered_model_config(schema=self.alternate_schema[-4:], materialized='view') + second_model_config = self.rendered_model_config( + schema=self.alternate_schema[-4:]) + unrendered_model_config = self.unrendered_model_config( + database=model_database, materialized='view') + unrendered_second_model_config = self.unrendered_model_config( + schema=self.alternate_schema[-4:], materialized='view') schema = self.unique_schema() # we are selecting from the seed, which is always in the default db @@ -3735,7 +3750,8 @@ def verify_run_results(self, expected_run_results): run_results = _read_json('./target/run_results.json') assert 'metadata' in run_results - self.verify_metadata(run_results['metadata'], 'https://schemas.getdbt.com/dbt/run-results/v1.json') + self.verify_metadata( + run_results['metadata'], 'https://schemas.getdbt.com/dbt/run-results/v1.json') self.assertIn('elapsed_time', run_results) self.assertGreater(run_results['elapsed_time'], 0) self.assertTrue( @@ -3752,7 +3768,8 @@ def verify_run_results(self, expected_run_results): @use_profile('postgres') def test__postgres__run_and_generate_no_compile(self): - self.run_and_generate(alternate_db=self.default_database, args=['--no-compile']) + self.run_and_generate( + alternate_db=self.default_database, args=['--no-compile']) self.verify_catalog(self.expected_postgres_catalog()) self.assertFalse(os.path.exists('./target/manifest.json')) @@ -3777,12 +3794,14 @@ def test__postgres_references(self): self.verify_catalog(self.expected_postgres_references_catalog()) self.verify_manifest(self.expected_postgres_references_manifest()) - self.verify_run_results(self.expected_postgres_references_run_results()) + self.verify_run_results( + self.expected_postgres_references_run_results()) @use_profile('postgres') def test_postgres_asset_paths_copied(self): self.run_and_generate( - {'asset-paths': [self.dir('assets'), self.dir('non-existent-assets')]}, + {'asset-paths': [self.dir('assets'), + self.dir('non-existent-assets')]}, ) assert os.path.exists('./target/assets') @@ -3823,7 +3842,8 @@ def connect(*args, **kwargs): self.verify_catalog(self.expected_snowflake_catalog(case_columns=True)) self.verify_manifest(self.expected_seeded_manifest(quote_model=True)) - self.verify_run_results(self.expected_run_results(quote_schema=False, quote_model=True)) + self.verify_run_results(self.expected_run_results( + quote_schema=False, quote_model=True)) @use_profile('bigquery') def test__bigquery__run_and_generate(self): @@ -3862,7 +3882,8 @@ def test__redshift__incremental_view(self): model_count=1, ) self.verify_catalog(self.expected_redshift_incremental_catalog()) - self.verify_manifest(self.expected_redshift_incremental_view_manifest()) + self.verify_manifest( + self.expected_redshift_incremental_view_manifest()) @use_profile('presto') def test__presto__run_and_generate(self): diff --git a/test/integration/047_dbt_ls_test/test_ls.py b/test/integration/047_dbt_ls_test/test_ls.py index 302afd9e1c5..f4ef4726a07 100644 --- a/test/integration/047_dbt_ls_test/test_ls.py +++ b/test/integration/047_dbt_ls_test/test_ls.py @@ -332,7 +332,7 @@ def expect_test_output(self): 'tags': ['schema'], 'config': { 'enabled': True, - 'materialized': 'view', + 'materialized': 'test', 'post-hook': [], 'severity': 'ERROR', 'tags': [], @@ -356,7 +356,7 @@ def expect_test_output(self): 'tags': ['data'], 'config': { 'enabled': True, - 'materialized': 'view', + 'materialized': 'test', 'post-hook': [], 'severity': 'ERROR', 'tags': [], @@ -380,7 +380,7 @@ def expect_test_output(self): 'tags': ['schema'], 'config': { 'enabled': True, - 'materialized': 'view', + 'materialized': 'test', 'post-hook': [], 'severity': 'ERROR', 'tags': [], diff --git a/test/unit/test_contracts_graph_compiled.py b/test/unit/test_contracts_graph_compiled.py index ad01cdb50c0..c1c1653bac7 100644 --- a/test/unit/test_contracts_graph_compiled.py +++ b/test/unit/test_contracts_graph_compiled.py @@ -235,31 +235,40 @@ def test_invalid_bad_type_model(minimal_uncompiled_dict): lambda u: (u, u.replace(alias='nope')), # None -> False is a config change even though it's pretty much the same - lambda u: (u.replace(config=u.config.replace(persist_docs={'relation': False})), u.replace(config=u.config.replace(persist_docs={'relation': False}))), - lambda u: (u.replace(config=u.config.replace(persist_docs={'columns': False})), u.replace(config=u.config.replace(persist_docs={'columns': False}))), + lambda u: (u.replace(config=u.config.replace(persist_docs={'relation': False})), u.replace( + config=u.config.replace(persist_docs={'relation': False}))), + lambda u: (u.replace(config=u.config.replace(persist_docs={'columns': False})), u.replace( + config=u.config.replace(persist_docs={'columns': False}))), # True -> True - lambda u: (u.replace(config=u.config.replace(persist_docs={'relation': True})), u.replace(config=u.config.replace(persist_docs={'relation': True}))), - lambda u: (u.replace(config=u.config.replace(persist_docs={'columns': True})), u.replace(config=u.config.replace(persist_docs={'columns': True}))), + lambda u: (u.replace(config=u.config.replace(persist_docs={'relation': True})), u.replace( + config=u.config.replace(persist_docs={'relation': True}))), + lambda u: (u.replace(config=u.config.replace(persist_docs={'columns': True})), u.replace( + config=u.config.replace(persist_docs={'columns': True}))), # only columns docs enabled, but description changed - lambda u: (u.replace(config=u.config.replace(persist_docs={'columns': True})), u.replace(config=u.config.replace(persist_docs={'columns': True}), description='a model description')), + lambda u: (u.replace(config=u.config.replace(persist_docs={'columns': True})), u.replace( + config=u.config.replace(persist_docs={'columns': True}), description='a model description')), # only relation docs eanbled, but columns changed - lambda u: (u.replace(config=u.config.replace(persist_docs={'relation': True})), u.replace(config=u.config.replace(persist_docs={'relation': True}), columns={'a': ColumnInfo(name='a', description='a column description')})) + lambda u: (u.replace(config=u.config.replace(persist_docs={'relation': True})), u.replace(config=u.config.replace( + persist_docs={'relation': True}), columns={'a': ColumnInfo(name='a', description='a column description')})) ] changed_compiled_models = [ lambda u: (u, None), lambda u: (u, u.replace(raw_sql='select * from wherever')), - lambda u: (u, u.replace(fqn=['test', 'models', 'subdir', 'foo'], original_file_path='models/subdir/foo.sql', path='/root/models/subdir/foo.sql')), + lambda u: (u, u.replace(fqn=['test', 'models', 'subdir', 'foo'], + original_file_path='models/subdir/foo.sql', path='/root/models/subdir/foo.sql')), lambda u: (u, replace_config(u, full_refresh=True)), lambda u: (u, replace_config(u, post_hook=['select 1 as id'])), lambda u: (u, replace_config(u, pre_hook=['select 1 as id'])), - lambda u: (u, replace_config(u, quoting={'database': True, 'schema': False, 'identifier': False})), + lambda u: (u, replace_config( + u, quoting={'database': True, 'schema': False, 'identifier': False})), # we changed persist_docs values lambda u: (u, replace_config(u, persist_docs={'relation': True})), lambda u: (u, replace_config(u, persist_docs={'columns': True})), - lambda u: (u, replace_config(u, persist_docs={'columns': True, 'relation': True})), + lambda u: (u, replace_config(u, persist_docs={ + 'columns': True, 'relation': True})), # None -> False is a config change even though it's pretty much the same lambda u: (u, replace_config(u, persist_docs={'relation': False})), @@ -267,12 +276,14 @@ def test_invalid_bad_type_model(minimal_uncompiled_dict): # persist docs was true for the relation and we changed the model description lambda u: ( replace_config(u, persist_docs={'relation': True}), - replace_config(u, persist_docs={'relation': True}, description='a model description'), + replace_config(u, persist_docs={ + 'relation': True}, description='a model description'), ), # persist docs was true for columns and we changed the model description lambda u: ( replace_config(u, persist_docs={'columns': True}), - replace_config(u, persist_docs={'columns': True}, columns={'a': ColumnInfo(name='a', description='a column description')}) + replace_config(u, persist_docs={'columns': True}, columns={ + 'a': ColumnInfo(name='a', description='a column description')}) ), # changing alias/schema/database on the config level is a change lambda u: (u, replace_config(u, database='nope')), @@ -408,7 +419,7 @@ def basic_uncompiled_schema_test_dict(): 'config': { 'column_types': {}, 'enabled': True, - 'materialized': 'view', + 'materialized': 'test', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], @@ -457,7 +468,7 @@ def basic_compiled_schema_test_dict(): 'config': { 'column_types': {}, 'enabled': True, - 'materialized': 'view', + 'materialized': 'test', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], @@ -539,13 +550,15 @@ def test_invalid_resource_type_schema_test(minimal_schema_test_dict): lambda u: replace_config(u, full_refresh=True), lambda u: replace_config(u, post_hook=['select 1 as id']), lambda u: replace_config(u, pre_hook=['select 1 as id']), - lambda u: replace_config(u, quoting={'database': True, 'schema': False, 'identifier': False}), + lambda u: replace_config( + u, quoting={'database': True, 'schema': False, 'identifier': False}), ] changed_schema_tests = [ lambda u: None, - lambda u: u.replace(fqn=['test', 'models', 'subdir', 'foo'], original_file_path='models/subdir/foo.sql', path='/root/models/subdir/foo.sql'), + lambda u: u.replace(fqn=['test', 'models', 'subdir', 'foo'], + original_file_path='models/subdir/foo.sql', path='/root/models/subdir/foo.sql'), lambda u: replace_config(u, severity='warn'), # If we checked test metadata, these would caount. But we don't, because these changes would all change the unique ID, so it's irrelevant. # lambda u: u.replace(test_metadata=u.test_metadata.replace(namespace='something')), @@ -572,5 +585,6 @@ def test_compare_to_compiled(basic_uncompiled_schema_test_node, basic_compiled_s compiled = basic_compiled_schema_test_node assert not uncompiled.same_contents(compiled) fixed_config = compiled.config.replace(severity=uncompiled.config.severity) - fixed_compiled = compiled.replace(config=fixed_config, unrendered_config=uncompiled.unrendered_config) + fixed_compiled = compiled.replace( + config=fixed_config, unrendered_config=uncompiled.unrendered_config) assert uncompiled.same_contents(fixed_compiled) diff --git a/test/unit/test_contracts_graph_parsed.py b/test/unit/test_contracts_graph_parsed.py index f4bd3844c21..9161af7ae4a 100644 --- a/test/unit/test_contracts_graph_parsed.py +++ b/test/unit/test_contracts_graph_parsed.py @@ -982,7 +982,7 @@ def basic_parsed_schema_test_dict(): 'config': { 'column_types': {}, 'enabled': True, - 'materialized': 'view', + 'materialized': 'test', 'persist_docs': {}, 'post-hook': [], 'pre-hook': [], @@ -1137,7 +1137,7 @@ def test_basic_schema_test_node(minimal_parsed_schema_test_dict, basic_parsed_sc assert node.empty is False assert node.is_ephemeral is False assert node.is_refable is False - assert node.get_materialization() == 'view' + assert node.get_materialization() == 'test' assert_from_dict(node, minimum, ParsedSchemaTestNode) pickle.loads(pickle.dumps(node)) diff --git a/test/unit/test_graph_selector_methods.py b/test/unit/test_graph_selector_methods.py index 69d1fe2bf54..d712d69435d 100644 --- a/test/unit/test_graph_selector_methods.py +++ b/test/unit/test_graph_selector_methods.py @@ -2,7 +2,6 @@ import pytest from unittest import mock -from datetime import datetime from pathlib import Path from dbt.contracts.files import FileHash @@ -12,7 +11,6 @@ ParsedModelNode, ParsedExposure, ParsedSeedNode, - ParsedSnapshotNode, ParsedDataTestNode, ParsedSchemaTestNode, ParsedSourceDefinition, @@ -173,7 +171,8 @@ def make_schema_test(pkg, test_name, test_model, test_kwargs, path=None, refs=No source_values = [] # this doesn't really have to be correct if isinstance(test_model, ParsedSourceDefinition): - kwargs['model'] = "{{ source('" + test_model.source_name + "', '" + test_model.name + "') }}" + kwargs['model'] = "{{ source('" + test_model.source_name + \ + "', '" + test_model.name + "') }}" source_values.append([test_model.source_name, test_model.name]) else: kwargs['model'] = "{{ ref('" + test_model.name + "')}}" @@ -186,7 +185,8 @@ def make_schema_test(pkg, test_name, test_model, test_kwargs, path=None, refs=No if column_name is not None: args_name += '_' + column_name node_name = f'{test_name}_{args_name}' - raw_sql = '{{ config(severity="ERROR") }}{{ test_' + test_name + '(**dbt_schema_test_kwargs) }}' + raw_sql = '{{ config(severity="ERROR") }}{{ test_' + \ + test_name + '(**dbt_schema_test_kwargs) }}' name_parts = test_name.split('.') if len(name_parts) == 2: @@ -461,8 +461,10 @@ def view_test_nothing(view_model): @pytest.fixture def manifest(seed, source, ephemeral_model, view_model, table_model, ext_source, ext_model, union_model, ext_source_2, ext_source_other, ext_source_other_2, table_id_unique, table_id_not_null, view_id_unique, ext_source_id_unique, view_test_nothing): - nodes = [seed, ephemeral_model, view_model, table_model, union_model, ext_model, table_id_unique, table_id_not_null, view_id_unique, ext_source_id_unique, view_test_nothing] - sources = [source, ext_source, ext_source_2, ext_source_other, ext_source_other_2] + nodes = [seed, ephemeral_model, view_model, table_model, union_model, ext_model, + table_id_unique, table_id_not_null, view_id_unique, ext_source_id_unique, view_test_nothing] + sources = [source, ext_source, ext_source_2, + ext_source_other, ext_source_other_2] manifest = Manifest( nodes={n.unique_id: n for n in nodes}, sources={s.unique_id: s for s in sources}, @@ -477,7 +479,8 @@ def manifest(seed, source, ephemeral_model, view_model, table_model, ext_source, def search_manifest_using_method(manifest, method, selection): - selected = method.search(set(manifest.nodes) | set(manifest.sources) | set(manifest.exposures), selection) + selected = method.search(set(manifest.nodes) | set( + manifest.sources) | set(manifest.exposures), selection) results = {manifest.expect(uid).search_name for uid in selected} return results @@ -488,11 +491,14 @@ def test_select_fqn(manifest): assert isinstance(method, QualifiedNameSelectorMethod) assert method.arguments == [] - assert search_manifest_using_method(manifest, method, 'pkg.unions') == {'union_model'} + assert search_manifest_using_method( + manifest, method, 'pkg.unions') == {'union_model'} assert not search_manifest_using_method(manifest, method, 'ext.unions') # sources don't show up, because selection pretends they have no FQN. Should it? - assert search_manifest_using_method(manifest, method, 'pkg') == {'union_model', 'table_model', 'view_model', 'ephemeral_model', 'seed'} - assert search_manifest_using_method(manifest, method, 'ext') == {'ext_model'} + assert search_manifest_using_method(manifest, method, 'pkg') == { + 'union_model', 'table_model', 'view_model', 'ephemeral_model', 'seed'} + assert search_manifest_using_method( + manifest, method, 'ext') == {'ext_model'} def test_select_tag(manifest): @@ -501,7 +507,8 @@ def test_select_tag(manifest): assert isinstance(method, TagSelectorMethod) assert method.arguments == [] - assert search_manifest_using_method(manifest, method, 'uses_ephemeral') == {'view_model', 'table_model'} + assert search_manifest_using_method(manifest, method, 'uses_ephemeral') == { + 'view_model', 'table_model'} assert not search_manifest_using_method(manifest, method, 'missing') @@ -512,19 +519,29 @@ def test_select_source(manifest): assert method.arguments == [] # the lookup is based on how many components you provide: source, source.table, package.source.table - assert search_manifest_using_method(manifest, method, 'raw') == {'raw.seed', 'raw.ext_source', 'raw.ext_source_2'} - assert search_manifest_using_method(manifest, method, 'raw.seed') == {'raw.seed'} - assert search_manifest_using_method(manifest, method, 'pkg.raw.seed') == {'raw.seed'} - assert search_manifest_using_method(manifest, method, 'pkg.*.*') == {'raw.seed'} - assert search_manifest_using_method(manifest, method, 'raw.*') == {'raw.seed', 'raw.ext_source', 'raw.ext_source_2'} - assert search_manifest_using_method(manifest, method, 'ext.raw.*') == {'raw.ext_source', 'raw.ext_source_2'} + assert search_manifest_using_method(manifest, method, 'raw') == { + 'raw.seed', 'raw.ext_source', 'raw.ext_source_2'} + assert search_manifest_using_method( + manifest, method, 'raw.seed') == {'raw.seed'} + assert search_manifest_using_method( + manifest, method, 'pkg.raw.seed') == {'raw.seed'} + assert search_manifest_using_method( + manifest, method, 'pkg.*.*') == {'raw.seed'} + assert search_manifest_using_method( + manifest, method, 'raw.*') == {'raw.seed', 'raw.ext_source', 'raw.ext_source_2'} + assert search_manifest_using_method( + manifest, method, 'ext.raw.*') == {'raw.ext_source', 'raw.ext_source_2'} assert not search_manifest_using_method(manifest, method, 'missing') assert not search_manifest_using_method(manifest, method, 'raw.missing') - assert not search_manifest_using_method(manifest, method, 'missing.raw.seed') - - assert search_manifest_using_method(manifest, method, 'ext.*.*') == {'ext_raw.ext_source', 'ext_raw.ext_source_2', 'raw.ext_source', 'raw.ext_source_2'} - assert search_manifest_using_method(manifest, method, 'ext_raw') == {'ext_raw.ext_source', 'ext_raw.ext_source_2'} - assert search_manifest_using_method(manifest, method, 'ext.ext_raw.*') == {'ext_raw.ext_source', 'ext_raw.ext_source_2'} + assert not search_manifest_using_method( + manifest, method, 'missing.raw.seed') + + assert search_manifest_using_method(manifest, method, 'ext.*.*') == { + 'ext_raw.ext_source', 'ext_raw.ext_source_2', 'raw.ext_source', 'raw.ext_source_2'} + assert search_manifest_using_method(manifest, method, 'ext_raw') == { + 'ext_raw.ext_source', 'ext_raw.ext_source_2'} + assert search_manifest_using_method( + manifest, method, 'ext.ext_raw.*') == {'ext_raw.ext_source', 'ext_raw.ext_source_2'} assert not search_manifest_using_method(manifest, method, 'pkg.ext_raw.*') @@ -536,12 +553,17 @@ def test_select_path(manifest): assert isinstance(method, PathSelectorMethod) assert method.arguments == [] - assert search_manifest_using_method(manifest, method, 'subdirectory/*.sql') == {'union_model', 'table_model'} - assert search_manifest_using_method(manifest, method, 'subdirectory/union_model.sql') == {'union_model'} - assert search_manifest_using_method(manifest, method, 'models/*.sql') == {'view_model', 'ephemeral_model'} + assert search_manifest_using_method( + manifest, method, 'subdirectory/*.sql') == {'union_model', 'table_model'} + assert search_manifest_using_method( + manifest, method, 'subdirectory/union_model.sql') == {'union_model'} + assert search_manifest_using_method( + manifest, method, 'models/*.sql') == {'view_model', 'ephemeral_model'} assert not search_manifest_using_method(manifest, method, 'missing') - assert not search_manifest_using_method(manifest, method, 'models/missing.sql') - assert not search_manifest_using_method(manifest, method, 'models/missing*') + assert not search_manifest_using_method( + manifest, method, 'models/missing.sql') + assert not search_manifest_using_method( + manifest, method, 'models/missing*') def test_select_package(manifest): @@ -550,8 +572,10 @@ def test_select_package(manifest): assert isinstance(method, PackageSelectorMethod) assert method.arguments == [] - assert search_manifest_using_method(manifest, method, 'pkg') == {'union_model', 'table_model', 'view_model', 'ephemeral_model', 'seed', 'raw.seed', 'unique_table_model_id', 'not_null_table_model_id', 'unique_view_model_id', 'view_test_nothing'} - assert search_manifest_using_method(manifest, method, 'ext') == {'ext_model', 'ext_raw.ext_source', 'ext_raw.ext_source_2', 'raw.ext_source', 'raw.ext_source_2', 'unique_ext_raw_ext_source_id'} + assert search_manifest_using_method(manifest, method, 'pkg') == {'union_model', 'table_model', 'view_model', 'ephemeral_model', + 'seed', 'raw.seed', 'unique_table_model_id', 'not_null_table_model_id', 'unique_view_model_id', 'view_test_nothing'} + assert search_manifest_using_method(manifest, method, 'ext') == { + 'ext_model', 'ext_raw.ext_source', 'ext_raw.ext_source_2', 'raw.ext_source', 'raw.ext_source_2', 'unique_ext_raw_ext_source_id'} assert not search_manifest_using_method(manifest, method, 'missing') @@ -562,9 +586,10 @@ def test_select_config_materialized(manifest): assert isinstance(method, ConfigSelectorMethod) assert method.arguments == ['materialized'] - # yes, technically tests are "views" - assert search_manifest_using_method(manifest, method, 'view') == {'view_model', 'ext_model', 'unique_table_model_id', 'not_null_table_model_id', 'unique_view_model_id', 'unique_ext_raw_ext_source_id', 'view_test_nothing'} - assert search_manifest_using_method(manifest, method, 'table') == {'table_model', 'union_model'} + assert search_manifest_using_method(manifest, method, 'view') == { + 'view_model', 'ext_model'} + assert search_manifest_using_method(manifest, method, 'table') == { + 'table_model', 'union_model'} def test_select_test_name(manifest): @@ -573,8 +598,10 @@ def test_select_test_name(manifest): assert isinstance(method, TestNameSelectorMethod) assert method.arguments == [] - assert search_manifest_using_method(manifest, method, 'unique') == {'unique_table_model_id', 'unique_view_model_id', 'unique_ext_raw_ext_source_id'} - assert search_manifest_using_method(manifest, method, 'not_null') == {'not_null_table_model_id'} + assert search_manifest_using_method(manifest, method, 'unique') == { + 'unique_table_model_id', 'unique_view_model_id', 'unique_ext_raw_ext_source_id'} + assert search_manifest_using_method(manifest, method, 'not_null') == { + 'not_null_table_model_id'} assert not search_manifest_using_method(manifest, method, 'notatest') @@ -583,8 +610,10 @@ def test_select_test_type(manifest): method = methods.get_method('test_type', []) assert isinstance(method, TestTypeSelectorMethod) assert method.arguments == [] - assert search_manifest_using_method(manifest, method, 'schema') == {'unique_table_model_id', 'not_null_table_model_id', 'unique_view_model_id', 'unique_ext_raw_ext_source_id'} - assert search_manifest_using_method(manifest, method, 'data') == {'view_test_nothing'} + assert search_manifest_using_method(manifest, method, 'schema') == { + 'unique_table_model_id', 'not_null_table_model_id', 'unique_view_model_id', 'unique_ext_raw_ext_source_id'} + assert search_manifest_using_method(manifest, method, 'data') == { + 'view_test_nothing'} def test_select_exposure(manifest): @@ -593,8 +622,10 @@ def test_select_exposure(manifest): methods = MethodManager(manifest, None) method = methods.get_method('exposure', []) assert isinstance(method, ExposureSelectorMethod) - assert search_manifest_using_method(manifest, method, 'my_exposure') == {'my_exposure'} - assert not search_manifest_using_method(manifest, method, 'not_my_exposure') + assert search_manifest_using_method( + manifest, method, 'my_exposure') == {'my_exposure'} + assert not search_manifest_using_method( + manifest, method, 'not_my_exposure') @pytest.fixture @@ -644,41 +675,52 @@ def test_select_state_nothing(manifest, previous_state): def test_select_state_added_model(manifest, previous_state): add_node(manifest, make_model('pkg', 'another_model', 'select 1 as id')) method = statemethod(manifest, previous_state) - assert search_manifest_using_method(manifest, method, 'modified') == {'another_model'} - assert search_manifest_using_method(manifest, method, 'new') == {'another_model'} + assert search_manifest_using_method( + manifest, method, 'modified') == {'another_model'} + assert search_manifest_using_method( + manifest, method, 'new') == {'another_model'} def test_select_state_changed_model_sql(manifest, previous_state, view_model): change_node(manifest, view_model.replace(raw_sql='select 1 as id')) method = statemethod(manifest, previous_state) - assert search_manifest_using_method(manifest, method, 'modified') == {'view_model'} + assert search_manifest_using_method( + manifest, method, 'modified') == {'view_model'} assert not search_manifest_using_method(manifest, method, 'new') def test_select_state_changed_model_fqn(manifest, previous_state, view_model): - change_node(manifest, view_model.replace(fqn=view_model.fqn[:-1]+['nested']+view_model.fqn[-1:])) + change_node(manifest, view_model.replace( + fqn=view_model.fqn[:-1]+['nested']+view_model.fqn[-1:])) method = statemethod(manifest, previous_state) - assert search_manifest_using_method(manifest, method, 'modified') == {'view_model'} + assert search_manifest_using_method( + manifest, method, 'modified') == {'view_model'} assert not search_manifest_using_method(manifest, method, 'new') def test_select_state_added_seed(manifest, previous_state): add_node(manifest, make_seed('pkg', 'another_seed')) method = statemethod(manifest, previous_state) - assert search_manifest_using_method(manifest, method, 'modified') == {'another_seed'} - assert search_manifest_using_method(manifest, method, 'new') == {'another_seed'} + assert search_manifest_using_method( + manifest, method, 'modified') == {'another_seed'} + assert search_manifest_using_method( + manifest, method, 'new') == {'another_seed'} def test_select_state_changed_seed_checksum_sha_to_sha(manifest, previous_state, seed): - change_node(manifest, seed.replace(checksum=FileHash.from_contents('changed'))) + change_node(manifest, seed.replace( + checksum=FileHash.from_contents('changed'))) method = statemethod(manifest, previous_state) - assert search_manifest_using_method(manifest, method, 'modified') == {'seed'} + assert search_manifest_using_method( + manifest, method, 'modified') == {'seed'} assert not search_manifest_using_method(manifest, method, 'new') def test_select_state_changed_seed_checksum_path_to_path(manifest, previous_state, seed): - change_node(previous_state.manifest, seed.replace(checksum=FileHash(name='path', checksum=seed.original_file_path))) - change_node(manifest, seed.replace(checksum=FileHash(name='path', checksum=seed.original_file_path))) + change_node(previous_state.manifest, seed.replace( + checksum=FileHash(name='path', checksum=seed.original_file_path))) + change_node(manifest, seed.replace(checksum=FileHash( + name='path', checksum=seed.original_file_path))) method = statemethod(manifest, previous_state) with mock.patch('dbt.contracts.graph.parsed.warn_or_error') as warn_or_error_patch: assert not search_manifest_using_method(manifest, method, 'modified') @@ -691,10 +733,12 @@ def test_select_state_changed_seed_checksum_path_to_path(manifest, previous_stat def test_select_state_changed_seed_checksum_sha_to_path(manifest, previous_state, seed): - change_node(manifest, seed.replace(checksum=FileHash(name='path', checksum=seed.original_file_path))) + change_node(manifest, seed.replace(checksum=FileHash( + name='path', checksum=seed.original_file_path))) method = statemethod(manifest, previous_state) with mock.patch('dbt.contracts.graph.parsed.warn_or_error') as warn_or_error_patch: - assert search_manifest_using_method(manifest, method, 'modified') == {'seed'} + assert search_manifest_using_method( + manifest, method, 'modified') == {'seed'} warn_or_error_patch.assert_called_once() msg = warn_or_error_patch.call_args[0][0] assert msg.startswith('Found a seed (pkg.seed) >1MB in size') @@ -704,10 +748,12 @@ def test_select_state_changed_seed_checksum_sha_to_path(manifest, previous_state def test_select_state_changed_seed_checksum_path_to_sha(manifest, previous_state, seed): - change_node(previous_state.manifest, seed.replace(checksum=FileHash(name='path', checksum=seed.original_file_path))) + change_node(previous_state.manifest, seed.replace( + checksum=FileHash(name='path', checksum=seed.original_file_path))) method = statemethod(manifest, previous_state) with mock.patch('dbt.contracts.graph.parsed.warn_or_error') as warn_or_error_patch: - assert search_manifest_using_method(manifest, method, 'modified') == {'seed'} + assert search_manifest_using_method( + manifest, method, 'modified') == {'seed'} warn_or_error_patch.assert_not_called() with mock.patch('dbt.contracts.graph.parsed.warn_or_error') as warn_or_error_patch: assert not search_manifest_using_method(manifest, method, 'new') @@ -715,9 +761,11 @@ def test_select_state_changed_seed_checksum_path_to_sha(manifest, previous_state def test_select_state_changed_seed_fqn(manifest, previous_state, seed): - change_node(manifest, seed.replace(fqn=seed.fqn[:-1]+['nested']+seed.fqn[-1:])) + change_node(manifest, seed.replace( + fqn=seed.fqn[:-1]+['nested']+seed.fqn[-1:])) method = statemethod(manifest, previous_state) - assert search_manifest_using_method(manifest, method, 'modified') == {'seed'} + assert search_manifest_using_method( + manifest, method, 'modified') == {'seed'} assert not search_manifest_using_method(manifest, method, 'new') @@ -725,27 +773,32 @@ def test_select_state_changed_seed_relation_documented(manifest, previous_state, seed_doc_relation = replace_config(seed, persist_docs={'relation': True}) change_node(manifest, seed_doc_relation) method = statemethod(manifest, previous_state) - assert search_manifest_using_method(manifest, method, 'modified') == {'seed'} + assert search_manifest_using_method( + manifest, method, 'modified') == {'seed'} assert not search_manifest_using_method(manifest, method, 'new') def test_select_state_changed_seed_relation_documented_nodocs(manifest, previous_state, seed): seed_doc_relation = replace_config(seed, persist_docs={'relation': True}) - seed_doc_relation_documented = seed_doc_relation.replace(description='a description') + seed_doc_relation_documented = seed_doc_relation.replace( + description='a description') change_node(previous_state.manifest, seed_doc_relation) change_node(manifest, seed_doc_relation_documented) method = statemethod(manifest, previous_state) - assert search_manifest_using_method(manifest, method, 'modified') == {'seed'} + assert search_manifest_using_method( + manifest, method, 'modified') == {'seed'} assert not search_manifest_using_method(manifest, method, 'new') def test_select_state_changed_seed_relation_documented_withdocs(manifest, previous_state, seed): seed_doc_relation = replace_config(seed, persist_docs={'relation': True}) - seed_doc_relation_documented = seed_doc_relation.replace(description='a description') + seed_doc_relation_documented = seed_doc_relation.replace( + description='a description') change_node(previous_state.manifest, seed_doc_relation_documented) change_node(manifest, seed_doc_relation) method = statemethod(manifest, previous_state) - assert search_manifest_using_method(manifest, method, 'modified') == {'seed'} + assert search_manifest_using_method( + manifest, method, 'modified') == {'seed'} assert not search_manifest_using_method(manifest, method, 'new') @@ -754,7 +807,8 @@ def test_select_state_changed_seed_columns_documented(manifest, previous_state, seed_doc_columns = replace_config(seed, persist_docs={'columns': True}) change_node(manifest, seed_doc_columns) method = statemethod(manifest, previous_state) - assert search_manifest_using_method(manifest, method, 'modified') == {'seed'} + assert search_manifest_using_method( + manifest, method, 'modified') == {'seed'} assert not search_manifest_using_method(manifest, method, 'new') @@ -768,7 +822,8 @@ def test_select_state_changed_seed_columns_documented_nodocs(manifest, previous_ change_node(manifest, seed_doc_columns_documented_columns) method = statemethod(manifest, previous_state) - assert search_manifest_using_method(manifest, method, 'modified') == {'seed'} + assert search_manifest_using_method( + manifest, method, 'modified') == {'seed'} assert not search_manifest_using_method(manifest, method, 'new') @@ -782,5 +837,6 @@ def test_select_state_changed_seed_columns_documented_withdocs(manifest, previou change_node(previous_state.manifest, seed_doc_columns_documented_columns) method = statemethod(manifest, previous_state) - assert search_manifest_using_method(manifest, method, 'modified') == {'seed'} + assert search_manifest_using_method( + manifest, method, 'modified') == {'seed'} assert not search_manifest_using_method(manifest, method, 'new')