-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'br_ibge_ppm' of https://github.com/basedosdados/queries…
…-basedosdados into br_ibge_ppm
- Loading branch information
Showing
7 changed files
with
305 additions
and
41 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
{{ config(alias="dicionario", schema="br_ms_sih") }} | ||
select | ||
safe_cast(id_tabela as string) id_tabela, | ||
safe_cast(nome_coluna as string) nome_coluna, | ||
safe_cast(chave as string) chave, | ||
safe_cast(cobertura_temporal as string) cobertura_temporal, | ||
safe_cast(valor as string) valor | ||
from `basedosdados-staging.br_ms_sih_staging.dicionario` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
-- https://github.com/basedosdados/pipelines/wiki/Incluindo-testes-no-seu-modelo#dicionários | ||
{% test custom_dictionaries( | ||
model, dictionary_model_name, table_id, columns_covered_by_dictionary | ||
) %} | ||
{{ config(severity="error") }} | ||
|
||
{%- set combined_query_parts = [] -%} | ||
{%- set union_parts = [] -%} | ||
|
||
{%- for column_name in columns_covered_by_dictionary %} | ||
{% set subquery_name = "exceptions_" ~ loop.index %} | ||
{% set left_table_name = "data_table_" ~ loop.index %} | ||
{% set right_table_name = "dictionary_table_" ~ loop.index %} | ||
|
||
{% set subquery %} | ||
{{ left_table_name }} as ( | ||
select {{ column_name }} as id | ||
from {{ model }} | ||
where {{ column_name }} is not null | ||
), | ||
{{ right_table_name }} as ( | ||
select chave | ||
from {{ dictionary_model_name }} | ||
where valor is not null | ||
and id_tabela = '{{ table_id }}' | ||
and nome_coluna = '{{ column_name }}' | ||
), | ||
{{ subquery_name }} as ( | ||
select '{{ column_name }}' as failed_column, id as missing_value | ||
from {{ left_table_name }} | ||
left join {{ right_table_name }} on {{ left_table_name }}.id = {{ right_table_name }}.chave | ||
where {{ right_table_name }}.chave is null | ||
) | ||
{% endset %} | ||
|
||
{%- do combined_query_parts.append(subquery) -%} | ||
{%- do union_parts.append(subquery_name) -%} | ||
{%- endfor %} | ||
|
||
{# Combine all CTEs into a single WITH clause and then union all results #} | ||
{% set final_query %} | ||
with | ||
{{ combined_query_parts | join(', ') }} | ||
|
||
select distinct failed_column, missing_value from {{ union_parts | join(' union all select distinct failed_column, missing_value from ') }} | ||
{% endset %} | ||
|
||
{{ return(final_query) }} | ||
|
||
{% endtest %} |
Oops, something went wrong.