From 5cb6a2d8b1eaff1d5b4dddc2a597ca14406244bc Mon Sep 17 00:00:00 2001 From: angelica-lastra Date: Mon, 24 Oct 2022 09:29:41 -0700 Subject: [PATCH 01/12] generate_model_yaml for multiple models --- macros/generate_model_yaml.sql | 38 +++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/macros/generate_model_yaml.sql b/macros/generate_model_yaml.sql index f00f910..a029983 100644 --- a/macros/generate_model_yaml.sql +++ b/macros/generate_model_yaml.sql @@ -17,24 +17,34 @@ {% do return(model_yaml) %} {% endmacro %} -{% macro generate_model_yaml(model_name, upstream_descriptions=False) %} +{% macro generate_model_yaml(model_name=[], upstream_descriptions=False) %} -{% set model_yaml=[] %} -{% set column_desc_dict = codegen.build_dict_column_descriptions(model_name) if upstream_descriptions else {} %} + {% set model_yaml=[] %} + {% set column_desc_dict = codegen.build_dict_column_descriptions(model_name) if upstream_descriptions else {} %} -{% do model_yaml.append('version: 2') %} -{% do model_yaml.append('') %} -{% do model_yaml.append('models:') %} -{% do model_yaml.append(' - name: ' ~ model_name | lower) %} -{% do model_yaml.append(' description: ""') %} -{% do model_yaml.append(' columns:') %} + {% do model_yaml.append('version: 2') %} + {% do model_yaml.append('') %} + {% do model_yaml.append('models:') %} + + {% if model_name is string %} + {% set model_name_list=[] %} + {% do model_name_list.append(model_name) %} + {% else %} + {% set model_name_list=model_name %} + {% endif %} + + {% for model in model_name_list %} + {% do model_yaml.append(' - name: ' ~ model | lower) %} + {% do model_yaml.append(' description: ""') %} + {% do model_yaml.append(' columns:') %} -{% set relation=ref(model_name) %} -{%- set columns = adapter.get_columns_in_relation(relation) -%} + {% set relation=ref(model) %} + {%- set columns = adapter.get_columns_in_relation(relation) -%} -{% for column in columns %} - {% set model_yaml = codegen.generate_column_yaml(column, model_yaml, column_desc_dict) %} -{% endfor %} + {% for column in columns %} + {% set model_yaml = codegen.generate_column_yaml(column, model_yaml, column_desc_dict) %} + {% endfor %} + {% endfor %} {% if execute %} From b2d919786a20919d87d778335f222a0f670517d2 Mon Sep 17 00:00:00 2001 From: angelica-lastra Date: Mon, 24 Oct 2022 09:53:51 -0700 Subject: [PATCH 02/12] updated README for generate_model_yaml --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d31f437..f538a6d 100644 --- a/README.md +++ b/README.md @@ -126,11 +126,11 @@ select * from renamed 4. Paste the output in to a model, and refactor as required. ## generate_model_yaml ([source](macros/generate_model_yaml.sql)) -This macro generates the YAML for a model, which you can then paste into a +This macro generates the YAML for a model or list of models, which you can then paste into a schema.yml file. ### Arguments: -* `model_name` (required): The model you wish to generate YAML for. +* `model_name` (required): The model or list of models you wish to generate YAML for. * `upstream_descriptions` (optional, default=False): Whether you want to include descriptions for identical column names from upstream models. ### Usage: @@ -146,7 +146,7 @@ schema.yml file. Alternatively, call the macro as an [operation](https://docs.getdbt.com/docs/using-operations): ``` -$ dbt run-operation generate_model_yaml --args '{"model_name": "customers"}' +$ dbt run-operation generate_model_yaml --args '{"model_name": ["customers", "orders"]}' ``` 3. The YAML for a base model will be logged to the command line From 370f6cd71c284fa603f7d29be0c9fd52d009a695 Mon Sep 17 00:00:00 2001 From: angelica-lastra Date: Mon, 24 Oct 2022 13:21:47 -0700 Subject: [PATCH 03/12] Updated README to include dbt runs w/ string/list --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index f538a6d..9609177 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,12 @@ schema.yml file. Alternatively, call the macro as an [operation](https://docs.getdbt.com/docs/using-operations): +``` +$ dbt run-operation generate_model_yaml --args '{"model_name": "customers"}' +``` + +or + ``` $ dbt run-operation generate_model_yaml --args '{"model_name": ["customers", "orders"]}' ``` From c24912ad6df411e2e0f3f462386657128e03b16d Mon Sep 17 00:00:00 2001 From: angelica-lastra Date: Tue, 25 Oct 2022 10:44:03 -0700 Subject: [PATCH 04/12] Added generate_model_yaml_multiple_models test --- ...st_generate_model_yaml_multiple_models.sql | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 integration_tests/tests/test_generate_model_yaml_multiple_models.sql diff --git a/integration_tests/tests/test_generate_model_yaml_multiple_models.sql b/integration_tests/tests/test_generate_model_yaml_multiple_models.sql new file mode 100644 index 0000000..c7df29f --- /dev/null +++ b/integration_tests/tests/test_generate_model_yaml_multiple_models.sql @@ -0,0 +1,30 @@ +{% set actual_model_yaml = codegen.generate_model_yaml( + model_name=['data__a_relation','data__b_relation'] + ) +%} + +{% set expected_model_yaml %} +version: 2 + +models: + - name: data__a_relation + description: "" + columns: + - name: col_a + description: "" + + - name: col_b + description: "" + + - name: data__b_relation + description: "" + columns: + - name: col_a + description: "" + + - name: col_b + description: "" + +{% endset %} + +{{ assert_equal (actual_model_yaml | trim, expected_model_yaml | trim) }} From 1ebff62f8feb82cb4b6338eb97641c82f46bf23e Mon Sep 17 00:00:00 2001 From: angelica-lastra <111460964+angelica-lastra@users.noreply.github.com> Date: Wed, 26 Oct 2022 12:46:42 -0700 Subject: [PATCH 05/12] Update README.md Co-authored-by: Grace Goheen <53586774+graciegoheen@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9609177..e935865 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ or $ dbt run-operation generate_model_yaml --args '{"model_name": ["customers", "orders"]}' ``` -3. The YAML for a base model will be logged to the command line +3. The YAML for the model(s) will be logged to the command line ``` version: 2 From af741ce2b9d76ec2a8c86e242ba758e4cbe1bb89 Mon Sep 17 00:00:00 2001 From: angelica-lastra Date: Thu, 3 Nov 2022 12:54:12 -0700 Subject: [PATCH 06/12] updated - allow list as inputs and error on string --- README.md | 12 ++++++------ macros/generate_model_yaml.sql | 29 +++++++++++++---------------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index e935865..3a3d803 100644 --- a/README.md +++ b/README.md @@ -126,11 +126,11 @@ select * from renamed 4. Paste the output in to a model, and refactor as required. ## generate_model_yaml ([source](macros/generate_model_yaml.sql)) -This macro generates the YAML for a model or list of models, which you can then paste into a +This macro generates the YAML for a list of model(s), which you can then paste into a schema.yml file. ### Arguments: -* `model_name` (required): The model or list of models you wish to generate YAML for. +* `model_names` (required): The model(s) you wish to generate YAML for. * `upstream_descriptions` (optional, default=False): Whether you want to include descriptions for identical column names from upstream models. ### Usage: @@ -139,23 +139,23 @@ schema.yml file. ``` {{ codegen.generate_model_yaml( - model_name='customers' + model_names=['customers'] ) }} ``` Alternatively, call the macro as an [operation](https://docs.getdbt.com/docs/using-operations): ``` -$ dbt run-operation generate_model_yaml --args '{"model_name": "customers"}' +$ dbt run-operation generate_model_yaml --args '{"model_names": ["customers"]}' ``` or ``` -$ dbt run-operation generate_model_yaml --args '{"model_name": ["customers", "orders"]}' +$ dbt run-operation generate_model_yaml --args '{"model_names": ["customers", "orders"]}' ``` -3. The YAML for the model(s) will be logged to the command line +3. The YAML for a base model(s) will be logged to the command line ``` version: 2 diff --git a/macros/generate_model_yaml.sql b/macros/generate_model_yaml.sql index a029983..881f879 100644 --- a/macros/generate_model_yaml.sql +++ b/macros/generate_model_yaml.sql @@ -17,7 +17,7 @@ {% do return(model_yaml) %} {% endmacro %} -{% macro generate_model_yaml(model_name=[], upstream_descriptions=False) %} +{% macro generate_model_yaml(model_names=[], upstream_descriptions=False) %} {% set model_yaml=[] %} {% set column_desc_dict = codegen.build_dict_column_descriptions(model_name) if upstream_descriptions else {} %} @@ -26,25 +26,22 @@ {% do model_yaml.append('') %} {% do model_yaml.append('models:') %} - {% if model_name is string %} - {% set model_name_list=[] %} - {% do model_name_list.append(model_name) %} + {% if model_names is string %} + {{ exceptions.raise_compiler_error("The name argument to ref() must be a list, got : " ~ number) }} {% else %} - {% set model_name_list=model_name %} - {% endif %} - - {% for model in model_name_list %} - {% do model_yaml.append(' - name: ' ~ model | lower) %} - {% do model_yaml.append(' description: ""') %} - {% do model_yaml.append(' columns:') %} + {% for model in model_names %} + {% do model_yaml.append(' - name: ' ~ model | lower) %} + {% do model_yaml.append(' description: ""') %} + {% do model_yaml.append(' columns:') %} - {% set relation=ref(model) %} - {%- set columns = adapter.get_columns_in_relation(relation) -%} + {% set relation=ref(model) %} + {%- set columns = adapter.get_columns_in_relation(relation) -%} - {% for column in columns %} - {% set model_yaml = codegen.generate_column_yaml(column, model_yaml, column_desc_dict) %} + {% for column in columns %} + {% set model_yaml = codegen.generate_column_yaml(column, model_yaml, column_desc_dict) %} + {% endfor %} {% endfor %} - {% endfor %} + {% endif %} {% if execute %} From 3669971646938112561e7eee41497a034394746c Mon Sep 17 00:00:00 2001 From: angelica-lastra Date: Thu, 3 Nov 2022 12:58:37 -0700 Subject: [PATCH 07/12] updated - integration tests as lists for gmy --- integration_tests/tests/test_generate_model_struct_yaml.sql | 2 +- integration_tests/tests/test_generate_model_yaml.sql | 2 +- .../tests/test_generate_model_yaml_multiple_models.sql | 2 +- .../tests/test_generate_model_yaml_upstream_descriptions.sql | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integration_tests/tests/test_generate_model_struct_yaml.sql b/integration_tests/tests/test_generate_model_struct_yaml.sql index 9670f11..44c1d92 100644 --- a/integration_tests/tests/test_generate_model_struct_yaml.sql +++ b/integration_tests/tests/test_generate_model_struct_yaml.sql @@ -10,7 +10,7 @@ ) %} {% set actual_source_yaml = codegen.generate_model_yaml( - model_name='model_struct' + model_names=['model_struct'] ) %} diff --git a/integration_tests/tests/test_generate_model_yaml.sql b/integration_tests/tests/test_generate_model_yaml.sql index 2f865a8..dbb9747 100644 --- a/integration_tests/tests/test_generate_model_yaml.sql +++ b/integration_tests/tests/test_generate_model_yaml.sql @@ -1,5 +1,5 @@ {% set actual_model_yaml = codegen.generate_model_yaml( - model_name='data__a_relation' + model_names=['data__a_relation'] ) %} diff --git a/integration_tests/tests/test_generate_model_yaml_multiple_models.sql b/integration_tests/tests/test_generate_model_yaml_multiple_models.sql index c7df29f..843959f 100644 --- a/integration_tests/tests/test_generate_model_yaml_multiple_models.sql +++ b/integration_tests/tests/test_generate_model_yaml_multiple_models.sql @@ -1,5 +1,5 @@ {% set actual_model_yaml = codegen.generate_model_yaml( - model_name=['data__a_relation','data__b_relation'] + model_names=['data__a_relation','data__b_relation'] ) %} diff --git a/integration_tests/tests/test_generate_model_yaml_upstream_descriptions.sql b/integration_tests/tests/test_generate_model_yaml_upstream_descriptions.sql index 3bc9804..71ecd89 100644 --- a/integration_tests/tests/test_generate_model_yaml_upstream_descriptions.sql +++ b/integration_tests/tests/test_generate_model_yaml_upstream_descriptions.sql @@ -1,5 +1,5 @@ {% set actual_model_yaml = codegen.generate_model_yaml( - model_name='child_model', + model_names=['child_model'], upstream_descriptions=True ) %} From 456015bc8e06cbace7f15c483d72f184427ba9c2 Mon Sep 17 00:00:00 2001 From: angelica-lastra Date: Thu, 3 Nov 2022 14:17:44 -0700 Subject: [PATCH 08/12] updated upstream descriptions --- macros/generate_model_yaml.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/generate_model_yaml.sql b/macros/generate_model_yaml.sql index 881f879..6882d10 100644 --- a/macros/generate_model_yaml.sql +++ b/macros/generate_model_yaml.sql @@ -20,7 +20,6 @@ {% macro generate_model_yaml(model_names=[], upstream_descriptions=False) %} {% set model_yaml=[] %} - {% set column_desc_dict = codegen.build_dict_column_descriptions(model_name) if upstream_descriptions else {} %} {% do model_yaml.append('version: 2') %} {% do model_yaml.append('') %} @@ -30,6 +29,7 @@ {{ exceptions.raise_compiler_error("The name argument to ref() must be a list, got : " ~ number) }} {% else %} {% for model in model_names %} + {% set column_desc_dict = codegen.build_dict_column_descriptions(model) if upstream_descriptions else {} %} {% do model_yaml.append(' - name: ' ~ model | lower) %} {% do model_yaml.append(' description: ""') %} {% do model_yaml.append(' columns:') %} From f9ea1949bc0a489ac20a21d3f9d9e07f701d15d4 Mon Sep 17 00:00:00 2001 From: angelica-lastra Date: Thu, 3 Nov 2022 14:22:52 -0700 Subject: [PATCH 09/12] cleaned code for readability --- macros/generate_model_yaml.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/generate_model_yaml.sql b/macros/generate_model_yaml.sql index 6882d10..8256492 100644 --- a/macros/generate_model_yaml.sql +++ b/macros/generate_model_yaml.sql @@ -29,13 +29,13 @@ {{ exceptions.raise_compiler_error("The name argument to ref() must be a list, got : " ~ number) }} {% else %} {% for model in model_names %} - {% set column_desc_dict = codegen.build_dict_column_descriptions(model) if upstream_descriptions else {} %} {% do model_yaml.append(' - name: ' ~ model | lower) %} {% do model_yaml.append(' description: ""') %} {% do model_yaml.append(' columns:') %} {% set relation=ref(model) %} {%- set columns = adapter.get_columns_in_relation(relation) -%} + {% set column_desc_dict = codegen.build_dict_column_descriptions(model) if upstream_descriptions else {} %} {% for column in columns %} {% set model_yaml = codegen.generate_column_yaml(column, model_yaml, column_desc_dict) %} From 62c2c1a1ca3468f1d5ef3c81e3e0a7456c7f5977 Mon Sep 17 00:00:00 2001 From: angelica-lastra Date: Thu, 3 Nov 2022 15:56:05 -0700 Subject: [PATCH 10/12] updated README with consistent examples --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index 3a3d803..193b6e3 100644 --- a/README.md +++ b/README.md @@ -149,12 +149,6 @@ Alternatively, call the macro as an [operation](https://docs.getdbt.com/docs/usi $ dbt run-operation generate_model_yaml --args '{"model_names": ["customers"]}' ``` -or - -``` -$ dbt run-operation generate_model_yaml --args '{"model_names": ["customers", "orders"]}' -``` - 3. The YAML for a base model(s) will be logged to the command line ``` From 3e9650fe90c01ec4d5d41ef20b402262939f3bcc Mon Sep 17 00:00:00 2001 From: angelica-lastra Date: Wed, 23 Nov 2022 14:22:12 -0700 Subject: [PATCH 11/12] fixed error message wording --- macros/generate_model_yaml.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/generate_model_yaml.sql b/macros/generate_model_yaml.sql index 8256492..5a3dada 100644 --- a/macros/generate_model_yaml.sql +++ b/macros/generate_model_yaml.sql @@ -26,7 +26,7 @@ {% do model_yaml.append('models:') %} {% if model_names is string %} - {{ exceptions.raise_compiler_error("The name argument to ref() must be a list, got : " ~ number) }} + {{ exceptions.raise_compiler_error(("The `model_names` argument must always be a list, even if there is only one model.") }} {% else %} {% for model in model_names %} {% do model_yaml.append(' - name: ' ~ model | lower) %} From 09d7b526bea3d7b6fb42a82d6260cd49b956cca5 Mon Sep 17 00:00:00 2001 From: angelica-lastra Date: Wed, 23 Nov 2022 14:43:31 -0700 Subject: [PATCH 12/12] fixed message error issue --- macros/generate_model_yaml.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/generate_model_yaml.sql b/macros/generate_model_yaml.sql index 5a3dada..c36b01d 100644 --- a/macros/generate_model_yaml.sql +++ b/macros/generate_model_yaml.sql @@ -26,7 +26,7 @@ {% do model_yaml.append('models:') %} {% if model_names is string %} - {{ exceptions.raise_compiler_error(("The `model_names` argument must always be a list, even if there is only one model.") }} + {{ exceptions.raise_compiler_error("The `model_names` argument must always be a list, even if there is only one model.") }} {% else %} {% for model in model_names %} {% do model_yaml.append(' - name: ' ~ model | lower) %}