From b4215007f6eadd535357d3e4acae0fc434c7f5d2 Mon Sep 17 00:00:00 2001 From: Pedro Sousa Date: Tue, 25 Jul 2023 17:10:33 +0100 Subject: [PATCH 1/5] Add support for ref with packages --- .../models/model_b_references_subpackage.sql | 1 + .../tests/unit/mock_input_styles.sql | 6 ++--- integration-tests/tests/unit/model_31.sql | 2 +- .../unit/model_b_references_subpackage.sql | 17 ++++++++++++ ...odel_f_references_a_non_nullable_field.sql | 2 +- macros/mock_builders.sql | 21 ++++++++++++--- macros/output.sql | 16 +++++++----- macros/overrides.sql | 19 +++++++++++--- macros/sql_builders.sql | 22 +++++++++------- macros/tests.sql | 8 +++--- macros/utils.sql | 26 +++++++++---------- 11 files changed, 97 insertions(+), 43 deletions(-) create mode 100644 integration-tests/models/model_b_references_subpackage.sql create mode 100644 integration-tests/tests/unit/model_b_references_subpackage.sql diff --git a/integration-tests/models/model_b_references_subpackage.sql b/integration-tests/models/model_b_references_subpackage.sql new file mode 100644 index 0000000..4219675 --- /dev/null +++ b/integration-tests/models/model_b_references_subpackage.sql @@ -0,0 +1 @@ +select * from {{ dbt_unit_testing.ref('dbt_unit_testing_sub_package', 'sub_package_model_a') }} where a >=1 diff --git a/integration-tests/tests/unit/mock_input_styles.sql b/integration-tests/tests/unit/mock_input_styles.sql index 3115891..c5fdaad 100644 --- a/integration-tests/tests/unit/mock_input_styles.sql +++ b/integration-tests/tests/unit/mock_input_styles.sql @@ -5,7 +5,7 @@ }} {% call dbt_unit_testing.test('model_b_references_a', 'csv input') %} - {% call dbt_unit_testing.mock_ref ('model_a',{"input_format": "CSV"}) %} + {% call dbt_unit_testing.mock_ref ('model_a', options={"input_format": "CSV"}) %} a,b 0,'a' 1,'b' @@ -19,7 +19,7 @@ UNION ALL {% call dbt_unit_testing.test('model_b_references_a', 'csv input with type cast on columns') %} - {% call dbt_unit_testing.mock_ref ('model_a',{"input_format": "csv"}) %} + {% call dbt_unit_testing.mock_ref ('model_a', options={"input_format": "csv"}) %} a::numeric,b 0,'a' 1,'b' @@ -33,7 +33,7 @@ UNION ALL UNION ALL {% call dbt_unit_testing.test('model_b_references_a', 'csv input with different separator') %} - {% call dbt_unit_testing.mock_ref ('model_a',{"input_format": "csv","column_separator": "|"}) %} + {% call dbt_unit_testing.mock_ref ('model_a', options={"input_format": "csv","column_separator": "|"}) %} a | b 0 | 'a' 1 | 'b' diff --git a/integration-tests/tests/unit/model_31.sql b/integration-tests/tests/unit/model_31.sql index 4169b5a..3f8e144 100644 --- a/integration-tests/tests/unit/model_31.sql +++ b/integration-tests/tests/unit/model_31.sql @@ -17,7 +17,7 @@ UNION ALL {% call dbt_unit_testing.test('model_31', "unless we need to fetch missing columns") %} - {% call dbt_unit_testing.mock_ref ('model_21', {"include_missing_columns": true}) %} + {% call dbt_unit_testing.mock_ref ('model_21', options={"include_missing_columns": true}) %} select 1 as id {% endcall %} {% call dbt_unit_testing.expect() %} diff --git a/integration-tests/tests/unit/model_b_references_subpackage.sql b/integration-tests/tests/unit/model_b_references_subpackage.sql new file mode 100644 index 0000000..49f2955 --- /dev/null +++ b/integration-tests/tests/unit/model_b_references_subpackage.sql @@ -0,0 +1,17 @@ +{{ + config( + tags=['unit-test', 'bigquery', 'snowflake', 'postgres'] + ) +}} + +{% call dbt_unit_testing.test('model_b_references_subpackage', 'sample test') %} + {% call dbt_unit_testing.mock_ref ('dbt_unit_testing_sub_package', 'sub_package_model_a') %} + select 0 as a, 'a' as b + UNION ALL + select 1 as a, 'b' as b + {% endcall %} + {% call dbt_unit_testing.expect() %} + select 1 as a, 'b' as b + {% endcall %} +{% endcall %} + \ No newline at end of file diff --git a/integration-tests/tests/unit/model_f_references_a_non_nullable_field.sql b/integration-tests/tests/unit/model_f_references_a_non_nullable_field.sql index 8562bcd..ad1ab2a 100644 --- a/integration-tests/tests/unit/model_f_references_a_non_nullable_field.sql +++ b/integration-tests/tests/unit/model_f_references_a_non_nullable_field.sql @@ -18,7 +18,7 @@ UNION ALL {% call dbt_unit_testing.test('model_f_references_a_non_nullable_field', 'sample test passes if we include extra columns') %} - {% call dbt_unit_testing.mock_ref ('model_a', {"include_missing_columns": true}) %} + {% call dbt_unit_testing.mock_ref ('model_a', options={"include_missing_columns": true}) %} select 0 as a UNION ALL select 1 as a diff --git a/macros/mock_builders.sql b/macros/mock_builders.sql index 7f10315..035b148 100644 --- a/macros/mock_builders.sql +++ b/macros/mock_builders.sql @@ -1,8 +1,20 @@ -{% macro mock_ref(model_name, options={}) %} +{% macro mock_ref(project_or_package, model_name, options={}) %} + {% if model_name is mapping %} + {% set options = model_name %} + {% set model_name = project_or_package %} + {% set project_or_package = model.package_name %} + {{ dbt_unit_testing.print_warning("Use keyword 'options' when passing options to mock_ref" ~ " (in " ~ model_name ~ ")") }} + {% else %} + {% set project_or_package, model_name = dbt_unit_testing.setup_project_and_model_name(project_or_package, model_name) %} + {% endif %} + {% if model_name is undefined %} + {{ dbt_unit_testing.raise_error('model_name must be provided for mock_ref') }} + {% endif %} {% set mock = { "type": 'mock', "resource_type": 'model', "name": model_name, + "package_name": project_or_package, "options": options, "input_values": caller(), } @@ -11,8 +23,11 @@ {% endmacro %} {% macro mock_source(source_name, table_name, options={}) %} - {% if not table_name %} - {{ dbt_unit_testing.raise_error('Table name must be provided for source') }} + {% if source_name is undefined %} + {{ dbt_unit_testing.raise_error('source_name must be provided for mock_source') }} + {% endif %} + {% if table_name is undefined %} + {{ dbt_unit_testing.raise_error('table_name must be provided for mock_source') }} {% endif %} {% set mock = { "type": 'mock', diff --git a/macros/output.sql b/macros/output.sql index 10d34ee..3d3ad0a 100644 --- a/macros/output.sql +++ b/macros/output.sql @@ -51,12 +51,12 @@ {% set cells = [] %} {% for cell_value in row %} {% set col_index = loop.index0 %} - {% set padded = dbt_unit_testing.pad(cell_value, columns_info[col_index].max_length, pad_right=cell_value is string) %} - {% if columns_info[col_index].has_differences %} - {% do cells.append("{RED}" ~ padded ~ "{RESET}") %} - {% else %} - {% do cells.append(padded) %} - {% endif %} + {% set padded = dbt_unit_testing.pad(cell_value, columns_info[col_index].max_length, pad_right=cell_value is string) %} + {% if columns_info[col_index].has_differences %} + {% do cells.append("{RED}" ~ padded ~ "{RESET}") %} + {% else %} + {% do cells.append(padded) %} + {% endif %} {% endfor %} {{ dbt_unit_testing.println("| " ~ cells | join(" | ") ~ " |")}} {% endfor %} @@ -83,3 +83,7 @@ {% macro println(s) %} {% do log(dbt_unit_testing.parse_colors(s ~ "{RESET}"), info=true) %} {% endmacro %} + +{% macro print_warning(s) %} + {% do log(dbt_unit_testing.parse_colors("{YELLOW}" ~ s ~ "{RESET}"), info=true) %} +{% endmacro %} diff --git a/macros/overrides.sql b/macros/overrides.sql index e55b0dd..34580d6 100644 --- a/macros/overrides.sql +++ b/macros/overrides.sql @@ -1,14 +1,16 @@ -{% macro ref(model_name) %} +{% macro ref(project_or_package, model_name) %} + {% set project_or_package, model_name = dbt_unit_testing.setup_project_and_model_name(project_or_package, model_name) %} {% if dbt_unit_testing.running_unit_test() %} - {{ return (dbt_unit_testing.ref_cte_name(model_name)) }} + {% set node = {"package_name": project_or_package, "name": model_name} %} + {{ return (dbt_unit_testing.ref_cte_name(node)) }} {% else %} - {{ return (builtins.ref(model_name)) }} + {{ return (builtins.ref(project_or_package, model_name)) }} {% endif %} {% endmacro %} {% macro source(source, table_name) %} {% if dbt_unit_testing.running_unit_test() %} - {{ return (dbt_unit_testing.source_cte_name(source, table_name)) }} + {{ return (dbt_unit_testing.source_cte_name({"source_name": source, "name": table_name})) }} {% else %} {{ return (builtins.source(source, table_name)) }} {% endif %} @@ -28,3 +30,12 @@ {% macro running_unit_test() %} {{ return ('unit-test' in config.get('tags', {})) }} {% endmacro %} + +{% macro setup_project_and_model_name(project_or_package, model_name) %} + {% set updated_project_or_package = project_or_package if model_name is defined else model.package_name %} + {% set updated_model_name = model_name if model_name is defined else project_or_package %} + {{ return((updated_project_or_package, updated_model_name)) }} +{% endmacro %} + + + diff --git a/macros/sql_builders.sql b/macros/sql_builders.sql index 0d7061b..b53fffb 100644 --- a/macros/sql_builders.sql +++ b/macros/sql_builders.sql @@ -30,23 +30,27 @@ {% macro cte_name(node) %} {% if node.resource_type in ('source') %} - {{ return (dbt_unit_testing.source_cte_name(node.source_name, node.name)) }} + {{ return (dbt_unit_testing.source_cte_name(node)) }} {% else %} - {{ return (dbt_unit_testing.ref_cte_name(node.name)) }} + {{ return (dbt_unit_testing.ref_cte_name(node)) }} {% endif %} {% endmacro %} -{% macro ref_cte_name(model_name) %} - {{ return (dbt_unit_testing.quote_identifier(model_name)) }} +{% macro ref_cte_name(node) %} + {% if node.project_or_package == model.package_name %} + {{ return (dbt_unit_testing.quote_identifier(node.name)) }} + {% else %} + {{ return (dbt_unit_testing.quote_identifier([node.package_name, node.name] | join("__"))) }} + {% endif %} {% endmacro %} -{% macro source_cte_name(source, table_name) %} +{% macro source_cte_name(node) %} {%- set cte_name -%} {%- if dbt_unit_testing.config_is_true("use_qualified_sources") -%} - {%- set source_node = dbt_unit_testing.source_node(source, table_name) -%} - {{ [source, table_name] | join("__") }} + {%- set source_node = dbt_unit_testing.source_node(node) -%} + {{ [source_node.source_name, source_node.name] | join("__") }} {%- else -%} - {{ table_name }} + {{ node.name }} {%- endif -%} {%- endset -%} {{ return (dbt_unit_testing.quote_identifier(cte_name)) }} @@ -100,7 +104,7 @@ {% macro render_node_for_model_being_tested(node) %} {% set model_sql = node.raw_sql if node.raw_sql is defined else node.raw_code %} - {% set model_name = node.name %} + {% set model_name = dbt_unit_testing.cte_name(node) %} {% set this_name = this | string %} -- {# If the model contains a 'this' property, we will replace its result with the name of the model being tested #} -- {# but first we mask previous occurrences that could be there before the render (very unlikely to happen) #} diff --git a/macros/tests.sql b/macros/tests.sql index 4e3333e..e69a01e 100644 --- a/macros/tests.sql +++ b/macros/tests.sql @@ -16,7 +16,8 @@ {% endmacro %} {% macro build_configuration_and_test_queries(model_name, test_description, options, mocks_and_expectations_json_str) %} - {{ dbt_unit_testing.set_test_context("model_being_tested", model_name) }} + {% set node = {"package_name": model.package_name, "name": model_name} %} + {{ dbt_unit_testing.set_test_context("model_being_tested", dbt_unit_testing.ref_cte_name(node)) }} {% set test_configuration = { "model_name": model_name, "description": test_description, @@ -45,7 +46,7 @@ {% set expectations = mocks_and_expectations | selectattr("type", "==", "expectations") | first %} {% for mock in mocks %} - {% do mock.update({"unique_id": dbt_unit_testing.graph_node(mock.source_name, mock.name).unique_id}) %} + {% do mock.update({"unique_id": dbt_unit_testing.graph_node(mock).unique_id}) %} {% if mock.options.include_missing_columns %} {% do dbt_unit_testing.enrich_mock_sql_with_missing_columns(mock, test_configuration.options) %} {% endif %} @@ -73,7 +74,8 @@ {% macro build_test_queries(test_configuration) %} {% set expectations = test_configuration.expectations %} - {% set model_node = dbt_unit_testing.model_node(test_configuration.model_name) %} + {% set node = {"package_name": model.package_name, "name": test_configuration.model_name} %} + {% set model_node = dbt_unit_testing.model_node(node) %} {%- set model_complete_sql = dbt_unit_testing.build_model_complete_sql(model_node, test_configuration.mocks, test_configuration.options) -%} {% set columns = dbt_unit_testing.quote_and_join_columns(dbt_unit_testing.extract_columns_list(expectations.input_values)) %} diff --git a/macros/utils.sql b/macros/utils.sql index b897542..79223e9 100644 --- a/macros/utils.sql +++ b/macros/utils.sql @@ -66,30 +66,30 @@ {{ return (graph.nodes[node_id] if node_id in graph.nodes else graph.sources[node_id]) }} {% endmacro %} -{% macro graph_node_by_prefix (prefix, name) %} - {{ return (graph.nodes[prefix ~ "." ~ model.package_name ~ "." ~ name])}} +{% macro graph_node_by_prefix (prefix, node) %} + {{ return (graph.nodes[prefix ~ "." ~ node.package_name ~ "." ~ node.name])}} {% endmacro %} -{% macro model_node (model_name) %} +{% macro model_node (node) %} {% set node = nil - | default(dbt_unit_testing.graph_node_by_prefix("model", model_name)) - | default(dbt_unit_testing.graph_node_by_prefix("snapshot", model_name)) - | default(dbt_unit_testing.graph_node_by_prefix("seed", model_name)) %} + | default(dbt_unit_testing.graph_node_by_prefix("model", node)) + | default(dbt_unit_testing.graph_node_by_prefix("snapshot", node)) + | default(dbt_unit_testing.graph_node_by_prefix("seed", node)) %} {% if not node %} - {{ dbt_unit_testing.raise_error("Node " ~ model.package_name ~ "." ~ model_name ~ " not found.") }} + {{ dbt_unit_testing.raise_error("Node " ~ node.package_name ~ "." ~ node.name ~ " not found.") }} {% endif %} {{ return (node) }} {% endmacro %} -{% macro source_node(source_name, model_name) %} - {{ return (graph.sources["source." ~ model.package_name ~ "." ~ source_name ~ "." ~ model_name]) }} +{% macro source_node(node) %} + {{ return (graph.sources["source." ~ model.package_name ~ "." ~ node.source_name ~ "." ~ node.name]) }} {% endmacro %} -{% macro graph_node(source_name, model_name) %} - {% if source_name %} - {{ return (dbt_unit_testing.source_node(source_name, model_name)) }} +{% macro graph_node(node) %} + {% if node.resource_type in ('source') %} + {{ return (dbt_unit_testing.source_node(node)) }} {% else %} - {{ return (dbt_unit_testing.model_node(model_name)) }} + {{ return (dbt_unit_testing.model_node(node)) }} {% endif %} {% endmacro %} From 9f11746a1394662eccd27b3a9f746dec68b07eb5 Mon Sep 17 00:00:00 2001 From: Pedro Sousa Date: Sun, 30 Jul 2023 11:36:42 +0100 Subject: [PATCH 2/5] Add support for model versions --- .../model_references_model_with_version.sql | 3 ++ .../model_references_model_with_version_1.sql | 3 ++ .../model_references_model_with_version_2.sql | 4 ++ .../models/model_with_version_v1.sql | 1 + .../models/model_with_version_v2.sql | 1 + integration-tests/models/schema.yml | 8 ++++ .../model_references_model_with_version.sql | 43 +++++++++++++++++++ macros/mock_builders.sql | 2 + macros/overrides.sql | 7 +-- macros/sql_builders.sql | 12 ++++-- macros/utils.sql | 34 ++++++++++----- run-integration-tests.sh | 10 ++++- run-tests-helper.sh | 2 +- run-tests-on-dbt-version.sh | 2 +- 14 files changed, 110 insertions(+), 22 deletions(-) create mode 100644 integration-tests/models/model_references_model_with_version.sql create mode 100644 integration-tests/models/model_references_model_with_version_1.sql create mode 100644 integration-tests/models/model_references_model_with_version_2.sql create mode 100644 integration-tests/models/model_with_version_v1.sql create mode 100644 integration-tests/models/model_with_version_v2.sql create mode 100644 integration-tests/models/schema.yml create mode 100644 integration-tests/tests/unit/model_references_model_with_version.sql diff --git a/integration-tests/models/model_references_model_with_version.sql b/integration-tests/models/model_references_model_with_version.sql new file mode 100644 index 0000000..be450ec --- /dev/null +++ b/integration-tests/models/model_references_model_with_version.sql @@ -0,0 +1,3 @@ +{% if dbt_unit_testing.version_bigger_or_equal_to("1.5") %} + select * from {{ dbt_unit_testing.ref('model_with_version') }} where a > 0 +{% endif %} diff --git a/integration-tests/models/model_references_model_with_version_1.sql b/integration-tests/models/model_references_model_with_version_1.sql new file mode 100644 index 0000000..7cdc91d --- /dev/null +++ b/integration-tests/models/model_references_model_with_version_1.sql @@ -0,0 +1,3 @@ +{% if dbt_unit_testing.version_bigger_or_equal_to("1.5") %} + select * from {{ dbt_unit_testing.ref('model_with_version', v=1) }} where a > 1 +{% endif %} diff --git a/integration-tests/models/model_references_model_with_version_2.sql b/integration-tests/models/model_references_model_with_version_2.sql new file mode 100644 index 0000000..7eaba77 --- /dev/null +++ b/integration-tests/models/model_references_model_with_version_2.sql @@ -0,0 +1,4 @@ +{% if dbt_unit_testing.version_bigger_or_equal_to("1.5") %} + select * from {{ dbt_unit_testing.ref('model_with_version', version=2) }} where a > 2 +{% endif %} + diff --git a/integration-tests/models/model_with_version_v1.sql b/integration-tests/models/model_with_version_v1.sql new file mode 100644 index 0000000..afd685c --- /dev/null +++ b/integration-tests/models/model_with_version_v1.sql @@ -0,0 +1 @@ +select 1 as a \ No newline at end of file diff --git a/integration-tests/models/model_with_version_v2.sql b/integration-tests/models/model_with_version_v2.sql new file mode 100644 index 0000000..f5f8dec --- /dev/null +++ b/integration-tests/models/model_with_version_v2.sql @@ -0,0 +1 @@ +select 2 as a \ No newline at end of file diff --git a/integration-tests/models/schema.yml b/integration-tests/models/schema.yml new file mode 100644 index 0000000..e5e2199 --- /dev/null +++ b/integration-tests/models/schema.yml @@ -0,0 +1,8 @@ +version: 2 + +models: + - name: model_with_version + + versions: + - v: 1 + - v: 2 diff --git a/integration-tests/tests/unit/model_references_model_with_version.sql b/integration-tests/tests/unit/model_references_model_with_version.sql new file mode 100644 index 0000000..ff5c0b1 --- /dev/null +++ b/integration-tests/tests/unit/model_references_model_with_version.sql @@ -0,0 +1,43 @@ +{{ + config( + tags=['unit-test', 'bigquery', 'snowflake', 'postgres', 'versioned', '1.5.4'] + ) +}} + +{% call dbt_unit_testing.test('model_references_model_with_version', 'latest version') %} + {% call dbt_unit_testing.mock_ref ('model_with_version') %} + select 0 as a + UNION ALL + select 1234 as a + {% endcall %} + {% call dbt_unit_testing.expect() %} + select 1234 as a + {% endcall %} +{% endcall %} + +UNION ALL + +{% call dbt_unit_testing.test('model_references_model_with_version_1', 'version 1') %} + {% call dbt_unit_testing.mock_ref ('model_with_version', v=1) %} + select 1 as a + UNION ALL + select 1234 as a + {% endcall %} + {% call dbt_unit_testing.expect() %} + select 1234 as a + {% endcall %} +{% endcall %} + +UNION ALL + +{% call dbt_unit_testing.test('model_references_model_with_version_2', 'version 2') %} + {% call dbt_unit_testing.mock_ref ('model_with_version', version=2) %} + select 2 as a + UNION ALL + select 1234 as a + {% endcall %} + {% call dbt_unit_testing.expect() %} + select 1234 as a + {% endcall %} +{% endcall %} + \ No newline at end of file diff --git a/macros/mock_builders.sql b/macros/mock_builders.sql index 035b148..883752d 100644 --- a/macros/mock_builders.sql +++ b/macros/mock_builders.sql @@ -10,11 +10,13 @@ {% if model_name is undefined %} {{ dbt_unit_testing.raise_error('model_name must be provided for mock_ref') }} {% endif %} + {% set node_version = kwargs["version"] | default(kwargs["v"]) | default(none) %} {% set mock = { "type": 'mock', "resource_type": 'model', "name": model_name, "package_name": project_or_package, + "version": node_version, "options": options, "input_values": caller(), } diff --git a/macros/overrides.sql b/macros/overrides.sql index 34580d6..81f6c98 100644 --- a/macros/overrides.sql +++ b/macros/overrides.sql @@ -1,10 +1,11 @@ {% macro ref(project_or_package, model_name) %} {% set project_or_package, model_name = dbt_unit_testing.setup_project_and_model_name(project_or_package, model_name) %} {% if dbt_unit_testing.running_unit_test() %} - {% set node = {"package_name": project_or_package, "name": model_name} %} - {{ return (dbt_unit_testing.ref_cte_name(node)) }} + {% set node_version = kwargs["version"] | default (kwargs["v"]) %} + {% set node = {"package_name": project_or_package, "name": model_name, "version": node_version} %} + {{ return (dbt_unit_testing.ref_cte_name(node)) }} {% else %} - {{ return (builtins.ref(project_or_package, model_name)) }} + {{ return (builtins.ref(project_or_package, model_name, **kwargs)) }} {% endif %} {% endmacro %} diff --git a/macros/sql_builders.sql b/macros/sql_builders.sql index b53fffb..033939e 100644 --- a/macros/sql_builders.sql +++ b/macros/sql_builders.sql @@ -37,11 +37,15 @@ {% endmacro %} {% macro ref_cte_name(node) %} - {% if node.project_or_package == model.package_name %} - {{ return (dbt_unit_testing.quote_identifier(node.name)) }} - {% else %} - {{ return (dbt_unit_testing.quote_identifier([node.package_name, node.name] | join("__"))) }} + {% set node = dbt_unit_testing.model_node(node) %} + {% set parts = [node.name] %} + {% if node.package_name != model.package_name %} + {% set parts = [node.package_name] + parts %} + {% endif %} + {% if node.version is not none %} + {% set parts = parts + [node.version] %} {% endif %} + {{ return (dbt_unit_testing.quote_identifier(parts | join("__"))) }} {% endmacro %} {% macro source_cte_name(node) %} diff --git a/macros/utils.sql b/macros/utils.sql index 79223e9..1b1b5fd 100644 --- a/macros/utils.sql +++ b/macros/utils.sql @@ -66,19 +66,27 @@ {{ return (graph.nodes[node_id] if node_id in graph.nodes else graph.sources[node_id]) }} {% endmacro %} -{% macro graph_node_by_prefix (prefix, node) %} - {{ return (graph.nodes[prefix ~ "." ~ node.package_name ~ "." ~ node.name])}} -{% endmacro %} - {% macro model_node (node) %} - {% set node = nil - | default(dbt_unit_testing.graph_node_by_prefix("model", node)) - | default(dbt_unit_testing.graph_node_by_prefix("snapshot", node)) - | default(dbt_unit_testing.graph_node_by_prefix("seed", node)) %} - {% if not node %} - {{ dbt_unit_testing.raise_error("Node " ~ node.package_name ~ "." ~ node.name ~ " not found.") }} + {% set graph_nodes = graph.nodes.values() | + selectattr('package_name', 'equalto', node.package_name) | + selectattr('name', 'equalto', node.name) | + list %} + {% if graph_nodes | length > 0 %} + {% if node.version is defined and node.version is not none %} + {% set graph_nodes = graph_nodes | selectattr('version', 'equalto', node.version) | list %} + {% else %} + {% set latest_version = graph_nodes[0].latest_version %} + {% if latest_version is defined and latest_version is not none %} + {% set graph_nodes = graph_nodes | selectattr('version', 'equalto', latest_version) | list %} + {% endif %} + {% endif %} {% endif %} - {{ return (node) }} + {% if graph_nodes | length == 0 %} + {% set node_version = '_v' ~ node.version if node.version is defined and node.version is not none else '' %} + {{ dbt_unit_testing.raise_error("Node " ~ node.package_name ~ "." ~ node.name ~ node_version ~ " not found.") }} + {% endif %} + {% set graph_node = graph_nodes[0] %} + {{ return (graph_node) }} {% endmacro %} {% macro source_node(node) %} @@ -190,3 +198,7 @@ {% do context.update({test_key: {}}) %} {% endmacro %} +{% macro version_bigger_or_equal_to(v) %} + {{ return (dbt_version >= v )}} +{% endmacro %} + diff --git a/run-integration-tests.sh b/run-integration-tests.sh index d1bdcc4..897b268 100755 --- a/run-integration-tests.sh +++ b/run-integration-tests.sh @@ -7,6 +7,11 @@ if [ -z "$1" ]; then exit 1 fi +if [ -z "$2" ]; then + echo 'Please provide dbt version' + exit 1 +fi + PROFILE=$1 cd integration-tests || exit @@ -23,12 +28,13 @@ dbt run-operation macro_with_ref --target "$PROFILE" # create seeds in the database dbt seed --target "$PROFILE" --select seeds/real_seeds # run tests with no database dependency -dbt test --target "$PROFILE" --select tag:unit-test,tag:"$PROFILE" --exclude tag:db-dependency +dbt test --target "$PROFILE" --select tag:unit-test,tag:"$PROFILE" --exclude tag:db-dependency --exclude tag:versioned +dbt test --target "$PROFILE" --select tag:unit-test,tag:"$PROFILE",tag:"$DBT_VERSION" --exclude tag:db-dependency # create sources in the database dbt seed --target "$PROFILE" --select seeds/existing_sources # create models in the database for tests that depends on database models -dbt run --target "$PROFILE" --select tag:add-to-database +dbt run --target "$PROFILE" --select tag:add-to-database # run tests with database dependency dbt test --target "$PROFILE" --select tag:unit-test,tag:"$PROFILE",tag:db-dependency diff --git a/run-tests-helper.sh b/run-tests-helper.sh index 80b1f4d..4fa8dab 100644 --- a/run-tests-helper.sh +++ b/run-tests-helper.sh @@ -5,7 +5,7 @@ function run_tests() { local PROFILE=$2 if [ "$PROFILE" == "postgres" ]; then - VERSIONS="1.3.3 1.4.5" + VERSIONS="1.3.4 1.4.6 1.5.4" elif [ "$PROFILE" == "bigquery" ]; then VERSIONS="1.3.2" elif [ "$PROFILE" == "snowflake" ]; then diff --git a/run-tests-on-dbt-version.sh b/run-tests-on-dbt-version.sh index bd57a0a..cc828d0 100755 --- a/run-tests-on-dbt-version.sh +++ b/run-tests-on-dbt-version.sh @@ -33,4 +33,4 @@ pip install "dbt-$PROFILE==$DBT_VERSION" source "$VENV_FOLDER/bin/activate" -"$SCRIPT_DIR/$TEST_SCRIPT.sh" "$PROFILE" +"$SCRIPT_DIR/$TEST_SCRIPT.sh" "$PROFILE" "$DBT_VERSION" From 845a76e58c57be4a6c734decef4fd253f45aaa7f Mon Sep 17 00:00:00 2001 From: Pedro Sousa Date: Sun, 30 Jul 2023 12:18:55 +0100 Subject: [PATCH 3/5] Fix integration tests --- run-integration-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-integration-tests.sh b/run-integration-tests.sh index 897b268..2674a59 100755 --- a/run-integration-tests.sh +++ b/run-integration-tests.sh @@ -28,7 +28,7 @@ dbt run-operation macro_with_ref --target "$PROFILE" # create seeds in the database dbt seed --target "$PROFILE" --select seeds/real_seeds # run tests with no database dependency -dbt test --target "$PROFILE" --select tag:unit-test,tag:"$PROFILE" --exclude tag:db-dependency --exclude tag:versioned +dbt test --target "$PROFILE" --select tag:unit-test,tag:"$PROFILE" --exclude tag:versioned tag:db-dependency dbt test --target "$PROFILE" --select tag:unit-test,tag:"$PROFILE",tag:"$DBT_VERSION" --exclude tag:db-dependency # create sources in the database From e52582b22fde2f21d6760b4fb532c2da4d698478 Mon Sep 17 00:00:00 2001 From: Pedro Sousa Date: Sun, 30 Jul 2023 23:04:52 +0100 Subject: [PATCH 4/5] Add support for model versions --- README.md | 60 +++++++++++++++++-- integration-tests/macros/refs.sql | 7 +++ integration-tests/macros/test_helpers.sql | 6 +- ...el_b_references_model_with_builtin_ref.sql | 4 +- .../model_references_model_with_version.sql | 2 +- .../model_references_model_with_version_1.sql | 2 +- .../model_references_model_with_version_2.sql | 2 +- .../tests/unit/model_with_version.sql | 27 +++++++++ macros/tests.sql | 15 ++--- macros/utils.sql | 1 + 10 files changed, 107 insertions(+), 19 deletions(-) create mode 100644 integration-tests/macros/refs.sql create mode 100644 integration-tests/tests/unit/model_with_version.sql diff --git a/README.md b/README.md index 0985e92..1ac288d 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Add the following to packages.yml ```yaml packages: - git: "https://github.com/EqualExperts/dbt-unit-testing" - revision: v0.2.9 + revision: v0.3.2 ``` [read the docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. @@ -355,12 +355,12 @@ To be able to mock the models and sources in tests, in your dbt models you **mus Alternatively, if you prefer to keep using the standard `ref` macro in the models, you can add these macros to your project: ```jinja -{% macro ref(model_name) %} - {{ return(dbt_unit_testing.ref(model_name)) }} +{% macro ref() %} + {{ return(dbt_unit_testing.ref(*varargs, **kwargs)) }} {% endmacro %} -{% macro source(source, model_name) %} - {{ return(dbt_unit_testing.source(source, model_name)) }} +{% macro source() %} + {{ return(dbt_unit_testing.source(*varargs, **kwargs)) }} {% endmacro %} ``` @@ -371,6 +371,56 @@ select {{ dbt_utils.star(builtins.ref('some_model')) }} from {{ ref('some_model') }} ``` +## Model versions + +You can specify a model version on the `dbt_unit_testing.ref` macro, the same way you do on the dbt ref macro: + +```jinja +{% call dbt_unit_testing.ref('some_model', version=3) %} +``` + +or + +```jinja +{% call dbt_unit_testing.ref('some_model', v=3) %} +``` + +if you are overriding the ref and source macros in your project, please use the new way of doing it ([here](#requirement)). This is necessary for the version parameter to work: + +```jinja +{% macro ref() %} + {{ return(dbt_unit_testing.ref(*varargs, **kwargs)) }} +{% endmacro %} + +{% macro source() %} + {{ return(dbt_unit_testing.source(*varargs, **kwargs)) }} +{% endmacro %} +``` + +### Testing Model versions + +You can test a specific model version by specifying the `version` parameter on the `dbt_unit_testing.test` macro: + +```jinja +{% call dbt_unit_testing.test('some_model', 'should return 1', version=3) %} + {% call dbt_unit_testing.expect() %} + select 1 + {% endcall %} +{% endcall %} +``` + +If `version` is not specified, the test will run against the latest version of the model. + +It is also possible to mock a specific model version, again by specifying the `version` parameter on the `dbt_unit_testing.mock_ref` macro: + +```jinja +{% call dbt_unit_testing.mock_ref('some_model', version=3) %} + select 1 +{% endcall %} +``` + +If `version` is not specified, the latest version of the model will be mocked. + ## Incremental models You can write unit tests for incremental models. To enable this functionality, you should add the following code to your project: diff --git a/integration-tests/macros/refs.sql b/integration-tests/macros/refs.sql new file mode 100644 index 0000000..76ab8ab --- /dev/null +++ b/integration-tests/macros/refs.sql @@ -0,0 +1,7 @@ +{% macro ref() %} + {{ return(dbt_unit_testing.ref(*varargs, **kwargs)) }} +{% endmacro %} + +{% macro source() %} + {{ return(dbt_unit_testing.source(*varargs, **kwargs)) }} +{% endmacro %} diff --git a/integration-tests/macros/test_helpers.sql b/integration-tests/macros/test_helpers.sql index cbebc1f..9252515 100644 --- a/integration-tests/macros/test_helpers.sql +++ b/integration-tests/macros/test_helpers.sql @@ -2,7 +2,8 @@ {{ dbt_unit_testing.ref_tested_model(model_name) }} {% if execute %} {% set mocks_and_expectations_json_str = caller() %} - {% set test_configuration, test_queries = dbt_unit_testing.build_configuration_and_test_queries(model_name, test_description, {}, mocks_and_expectations_json_str) %} + {% set model_node = {"package_name": model.package_name, "name": model_name} %} + {% set test_configuration, test_queries = dbt_unit_testing.build_configuration_and_test_queries(model_node, test_description, {}, mocks_and_expectations_json_str) %} {% set test_report = dbt_unit_testing.build_test_report(test_configuration, test_queries) %} {{ dbt_unit_testing.verbose("-------------------- " ~ test_configuration.model_name ~ " --------------------" ) }} @@ -20,7 +21,8 @@ {{ dbt_unit_testing.ref_tested_model(model_name) }} {% if execute %} {% set mocks_and_expectations_json_str = caller() %} - {% set test_configuration, test_queries = dbt_unit_testing.build_configuration_and_test_queries(model_name, test_description, options, mocks_and_expectations_json_str) %} + {% set model_node = {"package_name": model.package_name, "name": model_name} %} + {% set test_configuration, test_queries = dbt_unit_testing.build_configuration_and_test_queries(model_node, test_description, options, mocks_and_expectations_json_str) %} {% set model_query = test_queries["model_query"] %} diff --git a/integration-tests/models/model_b_references_model_with_builtin_ref.sql b/integration-tests/models/model_b_references_model_with_builtin_ref.sql index 1cba387..2f156d6 100644 --- a/integration-tests/models/model_b_references_model_with_builtin_ref.sql +++ b/integration-tests/models/model_b_references_model_with_builtin_ref.sql @@ -1,5 +1,5 @@ select - {{ dbt_utils.star(ref('model_in_database')) }}, - {{ dbt_utils.star(source('dbt_unit_testing', 'sample_source_name')) }} + {{ dbt_utils.star(builtins.ref('model_in_database')) }}, + {{ dbt_utils.star(builtins.source('dbt_unit_testing', 'sample_source_name')) }} from {{ dbt_unit_testing.ref('model_in_database') }} left join {{ dbt_unit_testing.source('dbt_unit_testing', 'sample_source_name') }} on false diff --git a/integration-tests/models/model_references_model_with_version.sql b/integration-tests/models/model_references_model_with_version.sql index be450ec..8efd59b 100644 --- a/integration-tests/models/model_references_model_with_version.sql +++ b/integration-tests/models/model_references_model_with_version.sql @@ -1,3 +1,3 @@ {% if dbt_unit_testing.version_bigger_or_equal_to("1.5") %} - select * from {{ dbt_unit_testing.ref('model_with_version') }} where a > 0 + select * from {{ ref('model_with_version') }} where a > 0 {% endif %} diff --git a/integration-tests/models/model_references_model_with_version_1.sql b/integration-tests/models/model_references_model_with_version_1.sql index 7cdc91d..cdd2f0e 100644 --- a/integration-tests/models/model_references_model_with_version_1.sql +++ b/integration-tests/models/model_references_model_with_version_1.sql @@ -1,3 +1,3 @@ {% if dbt_unit_testing.version_bigger_or_equal_to("1.5") %} - select * from {{ dbt_unit_testing.ref('model_with_version', v=1) }} where a > 1 + select * from {{ ref('model_with_version', v=1) }} where a > 1 {% endif %} diff --git a/integration-tests/models/model_references_model_with_version_2.sql b/integration-tests/models/model_references_model_with_version_2.sql index 7eaba77..2e7514a 100644 --- a/integration-tests/models/model_references_model_with_version_2.sql +++ b/integration-tests/models/model_references_model_with_version_2.sql @@ -1,4 +1,4 @@ {% if dbt_unit_testing.version_bigger_or_equal_to("1.5") %} - select * from {{ dbt_unit_testing.ref('model_with_version', version=2) }} where a > 2 + select * from {{ ref('model_with_version', version=2) }} where a > 2 {% endif %} diff --git a/integration-tests/tests/unit/model_with_version.sql b/integration-tests/tests/unit/model_with_version.sql new file mode 100644 index 0000000..bf78d53 --- /dev/null +++ b/integration-tests/tests/unit/model_with_version.sql @@ -0,0 +1,27 @@ +{{ + config( + tags=['unit-test', 'bigquery', 'snowflake', 'postgres', 'versioned', '1.5.4'] + ) +}} + +{% call dbt_unit_testing.test('model_with_version', 'version1 ', version=1) %} + {% call dbt_unit_testing.expect() %} + select 1 as a + {% endcall %} +{% endcall %} + +UNION ALL + +{% call dbt_unit_testing.test('model_with_version', 'version 2', v=2) %} + {% call dbt_unit_testing.expect() %} + select 2 as a + {% endcall %} +{% endcall %} + +UNION ALL + +{% call dbt_unit_testing.test('model_with_version', 'latest version') %} + {% call dbt_unit_testing.expect() %} + select 2 as a + {% endcall %} +{% endcall %} diff --git a/macros/tests.sql b/macros/tests.sql index e69a01e..ac56896 100644 --- a/macros/tests.sql +++ b/macros/tests.sql @@ -3,7 +3,9 @@ {% if execute %} {% set mocks_and_expectations_json_str = caller() %} - {% set test_configuration, test_queries = dbt_unit_testing.build_configuration_and_test_queries(model_name, test_description, options, mocks_and_expectations_json_str) %} + {% set model_version = kwargs["version"] | default(kwargs["v"]) | default(none) %} + {% set model_node = {"package_name": model.package_name, "name": model_name, "version": model_version} %} + {% set test_configuration, test_queries = dbt_unit_testing.build_configuration_and_test_queries(model_node, test_description, options, mocks_and_expectations_json_str) %} {% set test_report = dbt_unit_testing.build_test_report(test_configuration, test_queries) %} {% if not test_report.succeeded %} @@ -15,12 +17,12 @@ {% endif %} {% endmacro %} -{% macro build_configuration_and_test_queries(model_name, test_description, options, mocks_and_expectations_json_str) %} - {% set node = {"package_name": model.package_name, "name": model_name} %} - {{ dbt_unit_testing.set_test_context("model_being_tested", dbt_unit_testing.ref_cte_name(node)) }} +{% macro build_configuration_and_test_queries(model_node, test_description, options, mocks_and_expectations_json_str) %} + {{ dbt_unit_testing.set_test_context("model_being_tested", dbt_unit_testing.ref_cte_name(model_node)) }} {% set test_configuration = { - "model_name": model_name, + "model_name": model_node.model_name, "description": test_description, + "model_node": model_node, "options": dbt_unit_testing.merge_configs([options])} %} {{ dbt_unit_testing.set_test_context("options", test_configuration.options) }} @@ -74,8 +76,7 @@ {% macro build_test_queries(test_configuration) %} {% set expectations = test_configuration.expectations %} - {% set node = {"package_name": model.package_name, "name": test_configuration.model_name} %} - {% set model_node = dbt_unit_testing.model_node(node) %} + {% set model_node = dbt_unit_testing.model_node(test_configuration.model_node) %} {%- set model_complete_sql = dbt_unit_testing.build_model_complete_sql(model_node, test_configuration.mocks, test_configuration.options) -%} {% set columns = dbt_unit_testing.quote_and_join_columns(dbt_unit_testing.extract_columns_list(expectations.input_values)) %} diff --git a/macros/utils.sql b/macros/utils.sql index 1b1b5fd..5d742de 100644 --- a/macros/utils.sql +++ b/macros/utils.sql @@ -68,6 +68,7 @@ {% macro model_node (node) %} {% set graph_nodes = graph.nodes.values() | + selectattr('resource_type', 'in', ['model', 'snapshot', 'seed']) | selectattr('package_name', 'equalto', node.package_name) | selectattr('name', 'equalto', node.name) | list %} From c997099c3c747115e8ec6a2a8d658b2d33e7cb85 Mon Sep 17 00:00:00 2001 From: Pedro Sousa Date: Thu, 3 Aug 2023 16:47:51 +0100 Subject: [PATCH 5/5] Cleanup --- .../models/model_references_model_with_version.sql | 2 +- .../models/model_references_model_with_version_1.sql | 2 +- .../models/model_references_model_with_version_2.sql | 2 +- integration-tests/tests/unit/model_with_version.sql | 2 +- macros/utils.sql | 11 ++++++++++- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/integration-tests/models/model_references_model_with_version.sql b/integration-tests/models/model_references_model_with_version.sql index 8efd59b..a91313d 100644 --- a/integration-tests/models/model_references_model_with_version.sql +++ b/integration-tests/models/model_references_model_with_version.sql @@ -1,3 +1,3 @@ -{% if dbt_unit_testing.version_bigger_or_equal_to("1.5") %} +{% if dbt_unit_testing.version_bigger_or_equal_to("1.5.0") %} select * from {{ ref('model_with_version') }} where a > 0 {% endif %} diff --git a/integration-tests/models/model_references_model_with_version_1.sql b/integration-tests/models/model_references_model_with_version_1.sql index cdd2f0e..66b0574 100644 --- a/integration-tests/models/model_references_model_with_version_1.sql +++ b/integration-tests/models/model_references_model_with_version_1.sql @@ -1,3 +1,3 @@ -{% if dbt_unit_testing.version_bigger_or_equal_to("1.5") %} +{% if dbt_unit_testing.version_bigger_or_equal_to("1.5.0") %} select * from {{ ref('model_with_version', v=1) }} where a > 1 {% endif %} diff --git a/integration-tests/models/model_references_model_with_version_2.sql b/integration-tests/models/model_references_model_with_version_2.sql index 2e7514a..4e03973 100644 --- a/integration-tests/models/model_references_model_with_version_2.sql +++ b/integration-tests/models/model_references_model_with_version_2.sql @@ -1,4 +1,4 @@ -{% if dbt_unit_testing.version_bigger_or_equal_to("1.5") %} +{% if dbt_unit_testing.version_bigger_or_equal_to("1.5.0") %} select * from {{ ref('model_with_version', version=2) }} where a > 2 {% endif %} diff --git a/integration-tests/tests/unit/model_with_version.sql b/integration-tests/tests/unit/model_with_version.sql index bf78d53..9a9b975 100644 --- a/integration-tests/tests/unit/model_with_version.sql +++ b/integration-tests/tests/unit/model_with_version.sql @@ -4,7 +4,7 @@ ) }} -{% call dbt_unit_testing.test('model_with_version', 'version1 ', version=1) %} +{% call dbt_unit_testing.test('model_with_version', 'version 1', version=1) %} {% call dbt_unit_testing.expect() %} select 1 as a {% endcall %} diff --git a/macros/utils.sql b/macros/utils.sql index 5d742de..90a277b 100644 --- a/macros/utils.sql +++ b/macros/utils.sql @@ -199,7 +199,16 @@ {% do context.update({test_key: {}}) %} {% endmacro %} +{% macro split_and_pad_and_join(s, pad) %} + {% set parts = s.split('.') %} + {% set parts_2 = [] %} + {% for p in parts %} + {% do parts_2.append(dbt_unit_testing.pad(parts[loop.index-1], pad, pad_right=true)) %} + {% endfor %} + {{ return (parts_2 | join('.')) }} +{% endmacro %} + {% macro version_bigger_or_equal_to(v) %} - {{ return (dbt_version >= v )}} + {{ return (dbt_unit_testing.split_and_pad_and_join(dbt_version, 5) >= dbt_unit_testing.split_and_pad_and_join(v, 5)) }} {% endmacro %}