From 9524679c9a8e0cebcd9435e28ea2b37281cc81da Mon Sep 17 00:00:00 2001 From: Luiz Eduardo Date: Fri, 22 Nov 2024 13:06:33 -0300 Subject: [PATCH] =?UTF-8?q?Remo=C3=A7=C3=A3o=20antigo=20custom=5Fnull=5Fpr?= =?UTF-8?q?oportion=5Fmultiple=5Fcolumns=20e=20mudan=C3=A7a=20texto=20de?= =?UTF-8?q?=20saida?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...ustom_null_proportion_multiple_columns.sql | 71 ------------------- .../not_null_proportion_multiple_columns.sql | 14 +--- 2 files changed, 2 insertions(+), 83 deletions(-) delete mode 100644 tests/generic/custom_null_proportion_multiple_columns.sql diff --git a/tests/generic/custom_null_proportion_multiple_columns.sql b/tests/generic/custom_null_proportion_multiple_columns.sql deleted file mode 100644 index e3a3bde2..00000000 --- a/tests/generic/custom_null_proportion_multiple_columns.sql +++ /dev/null @@ -1,71 +0,0 @@ -{% test custom_not_null_proportion_multiple_columns( - model, ignore_values, at_least=0.05 -) %} - - {%- set columns = adapter.get_columns_in_relation(model) -%} - {% set suffix = "_nulls" %} - {% set pivot_columns_query %} - - with null_counts as( - - select - {% for column in columns -%} - {% if column.name not in ignore_values %} - SUM(CASE WHEN {{ column.name }} IS NULL THEN 1 ELSE 0 END) AS {{ column.name }}{{ suffix }}, - {% endif %} - {%- endfor %} - count(*) as total_records - from {{ model }} - ), - - pivot_columns as ( - - {% for column in columns -%} - {% if column.name not in ignore_values %} - select '{{ column.name }}' as column_name, {{ column.name }}{{ suffix }} as quantity, total_records - from null_counts - {% if not loop.last %}union all {% endif %} - {% endif %} - {%- endfor %} - ), - - faulty_columns as ( - select - * - from pivot_columns - where - quantity / total_records > (1 - {{ at_least }}) - - - ) - select * from faulty_columns - {% endset %} - with - validation_errors as ( - {%- set errors = dbt_utils.get_query_results_as_dict( - pivot_columns_query - ) -%} - {% if errors["column_name"] != () %} - {% for e in errors["column_name"] | unique %} - {{ - log( - "LOG: Coluna com preenchimento menor que " - ~ at_least * 100 - ~ "% ---> " - ~ e - ~ " [FAIL]", - info=True, - ) - }} - select '{{e}}' as column - {% if not loop.last %} - union all - {% endif %} - {% endfor %} - ) - select * - from validation_errors - {% else %}select 1 as column) select * from validation_errors where column != 1 - {% endif %} - -{% endtest %} diff --git a/tests/generic/not_null_proportion_multiple_columns.sql b/tests/generic/not_null_proportion_multiple_columns.sql index 57d4c149..6ba15fbc 100644 --- a/tests/generic/not_null_proportion_multiple_columns.sql +++ b/tests/generic/not_null_proportion_multiple_columns.sql @@ -48,18 +48,8 @@ {{ log( - "Coluna: " - ~ e - ~ " - Quantidade: " - ~ errors["quantity"][loop.index0] - ~ " - Total: " - ~ errors["total_records"][loop.index0] - ~ " - Preenchimento: " - ~ "%0.2f" - | format(proc_err | float) - ~ " - Preenchimento ideal: " - ~ at_least * 100 - ~ " - Resultado: FAIL", + "Coluna: " ~ e ~ " - Preenchimento de: " ~ "%0.2f" + | format(proc_err | float) ~ "% - Resultado: FAIL", info=True, ) }}