From 3571c0f0ec44aac230cbd7680962bc8be8db163d Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Wed, 6 Nov 2024 12:43:24 +0000 Subject: [PATCH 1/9] add meta --- website/docs/docs/build/dimensions.md | 7 +++ .../docs/docs/dbt-versions/release-notes.md | 1 + .../docs/reference/resource-configs/meta.md | 52 +++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/website/docs/docs/build/dimensions.md b/website/docs/docs/build/dimensions.md index 5026f4c45cd..eb1809ea72b 100644 --- a/website/docs/docs/build/dimensions.md +++ b/website/docs/docs/build/dimensions.md @@ -22,6 +22,7 @@ All dimensions require a `name`, `type`, and can optionally include an `expr` pa | `description` | A clear description of the dimension | Optional | | `expr` | Defines the underlying column or SQL query for a dimension. If no `expr` is specified, MetricFlow will use the column with the same name as the group. You can use the column name itself to input a SQL expression. | Optional | | `label` | A recommended string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | +| `meta` | Use the [`meta` field](/reference/resource-configs/meta) to set metadata for a resource and organize resources. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | Refer to the following for the complete specification for dimensions: @@ -59,6 +60,8 @@ semantic_models: type_params: time_granularity: day label: "Date of transaction" # Recommend adding a label to provide more context to users consuming the data + meta: + data_owner: "Finance team" expr: ts - name: is_bulk type: categorical @@ -106,6 +109,8 @@ dimensions: - name: is_bulk_transaction type: categorical expr: case when quantity > 10 then true else false end + meta: + usage: "Filter to identify bulk transactions, like where quantity > 10." ``` ## Time @@ -136,6 +141,8 @@ dimensions: type: time label: "Date of creation" expr: ts_created # ts_created is the underlying column name from the table + meta: + notes: "Only valid for orders from 2022 onward" is_partition: True type_params: time_granularity: day diff --git a/website/docs/docs/dbt-versions/release-notes.md b/website/docs/docs/dbt-versions/release-notes.md index c3f0bbfbe06..8dd16e99bbb 100644 --- a/website/docs/docs/dbt-versions/release-notes.md +++ b/website/docs/docs/dbt-versions/release-notes.md @@ -24,6 +24,7 @@ Release notes are grouped by month for both multi-tenant and virtual private clo - Improved handling of queries when multiple tables are selected in a data source. - Fixed a bug when an IN filter contained a lot of values. - Better error messaging for queries that can't be parsed correctly. +- **New**: [Dimensions](/reference/resource-configs/meta) now support the meta config property in dbt Cloud Versionless ??? or v1.9+??. You can add metadata to your dimensions to provide additional context and information about the dimension. Refer to [meta](/reference/resource-configs/meta) for more information. ## October 2024 diff --git a/website/docs/reference/resource-configs/meta.md b/website/docs/reference/resource-configs/meta.md index 53a4f77184e..ce33feff62e 100644 --- a/website/docs/reference/resource-configs/meta.md +++ b/website/docs/reference/resource-configs/meta.md @@ -19,6 +19,7 @@ hide_table_of_contents: true { label: 'Semantic Models', value: 'semantic models', }, { label: 'Metrics', value: 'metrics', }, { label: 'Saved queries', value: 'saved queries', }, + { label: 'Dimensions', value: 'dimensions', }, ] }> @@ -250,6 +251,22 @@ saved_queries: + + + + +```yml +semantic_models: + - name: semantic_model + ... + dimensions: + - name: order_date + config: + meta: {} +``` + + + @@ -343,3 +360,38 @@ models: +### Assign owner and additional metadata to dimensions + +The following example shows how to assign a `data_owner` and additional metadata value to a dimension in a semantic model. + + + +```yml +semantic_models: + - name: semantic_model + ... + dimensions: + - name: order_date + meta: + data_owner: "Finance team" + used_in_reporting: true +``` + + + +This second example shows how to assign a `data_owner` and additional metadata value to a dimension in the `dbt_project.yml` file using the `+meta` syntax. + + + +```yml +semantic-models: + jaffle_shop: + ... + dimensions: + - name: order_date + +meta: + data_owner: "Finance team" + used_in_reporting: true +``` + + From 2d5a67592296524f884b5097f7b2be932fd367b6 Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Wed, 6 Nov 2024 12:52:56 +0000 Subject: [PATCH 2/9] add link --- website/docs/reference/resource-configs/meta.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/docs/reference/resource-configs/meta.md b/website/docs/reference/resource-configs/meta.md index ce33feff62e..fa0d48c299e 100644 --- a/website/docs/reference/resource-configs/meta.md +++ b/website/docs/reference/resource-configs/meta.md @@ -259,7 +259,7 @@ saved_queries: semantic_models: - name: semantic_model ... - dimensions: + [dimensions](/docs/build/dimensions): - name: order_date config: meta: {} @@ -370,7 +370,7 @@ The following example shows how to assign a `data_owner` and additional metadata semantic_models: - name: semantic_model ... - dimensions: + [dimensions](/docs/build/dimensions): - name: order_date meta: data_owner: "Finance team" @@ -387,7 +387,7 @@ This second example shows how to assign a `data_owner` and additional metadata v semantic-models: jaffle_shop: ... - dimensions: + [dimensions](/docs/build/dimensions): - name: order_date +meta: data_owner: "Finance team" From c34e68e443bf544bd6b0fd1565eb9612d7ddd2fe Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Mon, 18 Nov 2024 15:24:53 -0500 Subject: [PATCH 3/9] update meta --- website/docs/docs/build/conversion-metrics.md | 12 +- website/docs/docs/build/dimensions.md | 107 +++++++++++++- website/docs/docs/build/entities.md | 69 ++++++++- website/docs/docs/build/measures.md | 136 ++++++++++++++++- website/docs/docs/build/simple.md | 4 +- .../docs/reference/resource-configs/meta.md | 138 ++++++++++++++---- website/snippets/_sl-measures-parameters.md | 1 + 7 files changed, 416 insertions(+), 51 deletions(-) diff --git a/website/docs/docs/build/conversion-metrics.md b/website/docs/docs/build/conversion-metrics.md index 2ef2c3910b9..979ee03e7b9 100644 --- a/website/docs/docs/build/conversion-metrics.md +++ b/website/docs/docs/build/conversion-metrics.md @@ -31,13 +31,13 @@ Note that we use the double colon (::) to indicate whether a parameter is nested | `entity` | The entity for each conversion event. | Required | | `calculation` | Method of calculation. Either `conversion_rate` or `conversions`. Defaults to `conversion_rate`. | Optional | | `base_measure` | A list of base measure inputs | Required | -| `base_measure:name` | The base conversion event measure. | Required | -| `base_measure:fill_nulls_with` | Set the value in your metric definition instead of null (such as zero). | Optional | -| `base_measure:join_to_timespine` | Boolean that indicates if the aggregated measure should be joined to the time spine table to fill in missing dates. Default `false`. | Optional | +| `base_measure::name` | The base conversion event measure. | Required | +| `base_measure::fill_nulls_with` | Set the value in your metric definition instead of null (such as zero). | Optional | +| `base_measure::join_to_timespine` | Boolean that indicates if the aggregated measure should be joined to the time spine table to fill in missing dates. Default `false`. | Optional | | `conversion_measure` | A list of conversion measure inputs. | Required | -| `conversion_measure:name` | The base conversion event measure.| Required | -| `conversion_measure:fill_nulls_with` | Set the value in your metric definition instead of null (such as zero). | Optional | -| `conversion_measure:join_to_timespine` | Boolean that indicates if the aggregated measure should be joined to the time spine table to fill in missing dates. Default `false`. | Optional | +| `conversion_measure::name` | The base conversion event measure.| Required | +| `conversion_measure::fill_nulls_with` | Set the value in your metric definition instead of null (such as zero). | Optional | +| `conversion_measure::join_to_timespine` | Boolean that indicates if the aggregated measure should be joined to the time spine table to fill in missing dates. Default `false`. | Optional | | `window` | The time window for the conversion event, such as 7 days, 1 week, 3 months. Defaults to infinity. | Optional | | `constant_properties` | List of constant properties. | Optional | | `base_property` | The property from the base semantic model that you want to hold constant. | Optional | diff --git a/website/docs/docs/build/dimensions.md b/website/docs/docs/build/dimensions.md index eb1809ea72b..424fd72e222 100644 --- a/website/docs/docs/build/dimensions.md +++ b/website/docs/docs/build/dimensions.md @@ -14,6 +14,26 @@ Groups are defined within semantic models, alongside entities and measures, and All dimensions require a `name`, `type`, and can optionally include an `expr` parameter. The `name` for your Dimension must be unique within the same semantic model. +:::tip +Note that we use the double colon (::) to indicate whether a parameter is nested within another parameter. So for example,` config::config_name` means the `config_name` parameter is nested under `config`. +::: + + + +| Parameter | Description | Type | +| --------- | ----------- | ---- | +| `name` | Refers to the name of the group that will be visible to the user in downstream tools. It can also serve as an alias if the column name or SQL query reference is different and provided in the `expr` parameter.

Dimension names should be unique within a semantic model, but they can be non-unique across different models as MetricFlow uses [joins](/docs/build/join-logic) to identify the right dimension. | Required | +| `type` | Specifies the type of group created in the semantic model. There are two types:

- **Categorical**: Describe attributes or features like geography or sales region.
- **Time**: Time-based dimensions like timestamps or dates. | Required | +| `type_params` | Specific type params such as if the time is primary or used as a partition | Required | +| `description` | A clear description of the dimension | Optional | +| `expr` | Defines the underlying column or SQL query for a dimension. If no `expr` is specified, MetricFlow will use the column with the same name as the group. You can use the column name itself to input a SQL expression. | Optional | +| `label` | A recommended string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | +| `config` | Use the [`config`](/reference/resource-properties/config) property to specify configurations for your metric. Supports [`meta`](/reference/resource-configs/meta), [`group`](/reference/resource-configs/group), and [`enabled`](/reference/resource-configs/enabled) configs. | Optional | + + + + + | Parameter | Description | Type | | --------- | ----------- | ---- | | `name` | Refers to the name of the group that will be visible to the user in downstream tools. It can also serve as an alias if the column name or SQL query reference is different and provided in the `expr` parameter.

Dimension names should be unique within a semantic model, but they can be non-unique across different models as MetricFlow uses [joins](/docs/build/join-logic) to identify the right dimension. | Required | @@ -22,7 +42,10 @@ All dimensions require a `name`, `type`, and can optionally include an `expr` pa | `description` | A clear description of the dimension | Optional | | `expr` | Defines the underlying column or SQL query for a dimension. If no `expr` is specified, MetricFlow will use the column with the same name as the group. You can use the column name itself to input a SQL expression. | Optional | | `label` | A recommended string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | -| `meta` | Use the [`meta` field](/reference/resource-configs/meta) to set metadata for a resource and organize resources. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | +| `config` | Use the [`config`](/reference/resource-properties/config) property to specify configurations for your metric. Supports [`meta`](/reference/resource-configs/meta), [`group`](/reference/resource-configs/group), and [`enabled`](/reference/resource-configs/enabled) configs. | Optional | +| `config::meta` | Use [`meta`](/reference/resource-configs/meta), nested under `config`, to set metadata for a resource and organize resources. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | + + Refer to the following for the complete specification for dimensions: @@ -38,6 +61,8 @@ dimensions: Refer to the following example to see how dimensions are used in a semantic model: + + ```yaml semantic_models: - name: transactions @@ -69,6 +94,40 @@ semantic_models: - name: type type: categorical ``` + + + + +```yaml +semantic_models: + - name: transactions + description: A record for every transaction that takes place. Carts are considered multiple transactions for each SKU. + model: {{ ref('fact_transactions') }} + defaults: + agg_time_dimension: order_date +# --- entities --- + entities: + - name: transaction + type: primary + ... +# --- measures --- + measures: + ... +# --- dimensions --- + dimensions: + - name: order_date + type: time + type_params: + time_granularity: day + label: "Date of transaction" # Recommend adding a label to provide more context to users consuming the data + expr: ts + - name: is_bulk + type: categorical + expr: case when quantity > 10 then true else false end + - name: type + type: categorical +``` + Dimensions are bound to the primary entity of the semantic model they are defined in. For example the dimension `type` is defined in a model that has `transaction` as a primary entity. `type` is scoped to the `transaction` entity, and to reference this dimension you would use the fully qualified dimension name i.e `transaction__type`. @@ -104,6 +163,8 @@ This section further explains the dimension definitions, along with examples. Di Categorical dimensions are used to group metrics by different attributes, features, or characteristics such as product type. They can refer to existing columns in your dbt model or be calculated using a SQL expression with the `expr` parameter. An example of a categorical dimension is `is_bulk_transaction`, which is a group created by applying a case statement to the underlying column `quantity`. This allows users to group or filter the data based on bulk transactions. + + ```yaml dimensions: - name: is_bulk_transaction @@ -112,6 +173,17 @@ dimensions: meta: usage: "Filter to identify bulk transactions, like where quantity > 10." ``` + + + + +```yaml +dimensions: + - name: is_bulk_transaction + type: categorical + expr: case when quantity > 10 then true else false end +``` + ## Time @@ -135,6 +207,8 @@ You can set `is_partition` for time to define specific time spans. Additionally, Use `is_partition: True` to show that a dimension exists over a specific time window. For example, a date-partitioned dimensional table. When you query metrics from different tables, the dbt Semantic Layer uses this parameter to ensure that the correct dimensional values are joined to measures. + + ```yaml dimensions: - name: created_at @@ -163,6 +237,37 @@ measures: expr: 1 agg: sum ``` + + + + +```yaml +dimensions: + - name: created_at + type: time + label: "Date of creation" + expr: ts_created # ts_created is the underlying column name from the table + is_partition: True + type_params: + time_granularity: day + - name: deleted_at + type: time + label: "Date of deletion" + expr: ts_deleted # ts_deleted is the underlying column name from the table + is_partition: True + type_params: + time_granularity: day + +measures: + - name: users_deleted + expr: 1 + agg: sum + agg_time_dimension: deleted_at + - name: users_created + expr: 1 + agg: sum +``` + diff --git a/website/docs/docs/build/entities.md b/website/docs/docs/build/entities.md index e4ed0773c3c..558dfd3aea4 100644 --- a/website/docs/docs/build/entities.md +++ b/website/docs/docs/build/entities.md @@ -95,17 +95,67 @@ Natural keys are columns or combinations of columns in a table that uniquely ide The following is the complete spec for entities: + + +```yaml +semantic_models: + - name: semantic_model_name + ..rest of the semantic model config + entities: + - name: entity_name ## Required + type: Primary, natural, foreign, or unique ## Required + description: A description of the field or role the entity takes in this table ## Optional + expr: The field that denotes that entity (transaction_id). ## Optional + Defaults to name if unspecified. + [config](/reference/resource-properties/config): Specify configurations for entity. ## Optional + [meta](/reference/resource-configs/meta): {} Set metadata for a resource and organize resources. Accepts plain text, spaces, and quotes. ## Optional +``` + + + + +```yaml +semantic_models: + - name: semantic_model_name + ..rest of the semantic model config + entities: + - name: entity_name ## Required + type: Primary, or natural, or foreign, or unique ## Required + description: A description of the field or role the entity takes in this table ## Optional + expr: The field that denotes that entity (transaction_id). ## Optional + Defaults to name if unspecified. +``` + + +Here's an example of how to define entities in a semantic model: + + + ```yaml entities: - - name: transaction ## Required - type: Primary or natural or foreign or unique ## Required + - name: transaction + type: primary + expr: id_transaction + - name: order + type: foreign + expr: id_order + - name: user + type: foreign + expr: substring(id_order from 2) + entities: + - name: transaction + type: description: A description of the field or role the entity takes in this table ## Optional - expr: The field that denotes that entity (transaction_id). ## Optional + expr: The field that denotes that entity (transaction_id). Defaults to name if unspecified. + [config](/reference/resource-properties/config): + [meta](/reference/resource-configs/meta): + data_owner: "Finance team" ``` + + + -Here's an example of how to define entities in a semantic model: - ```yaml entities: - name: transaction @@ -117,11 +167,18 @@ entities: - name: user type: foreign expr: substring(id_order from 2) + entities: + - name: transaction + type: + description: A description of the field or role the entity takes in this table ## Optional + expr: The field that denotes that entity (transaction_id). + Defaults to name if unspecified. ``` + ## Combine columns with a key -If a table doesn't have any key (like a primary key), use _surrogate combination_ to form a key that will help you identify a record by combining two columns. This applies to any [entity type](/docs/build/entities#entity-types). For example, you can combine `date_key` and `brand_code` from the `raw_brand_target_weekly` table to form a _surrogate key_. The following example creates a surrogate key by joining `date_key` and `brand_code` using a pipe (`|`) as a separator. +If a table doesn't have any key (like a primary key), use _surrogate combination_ to form a key that will help you identify a record by combining two columns. This applies to any [entity type](/docs/build/entities#entity-types). For example, you can combine `date_key` and `brand_code` from the `raw_brand_target_weekly` table to form a _surrogate key_. The following example creates a surrogate key by joining `date_key` and `brand_code` using a pipe (`|`) as a separator. ```yaml diff --git a/website/docs/docs/build/measures.md b/website/docs/docs/build/measures.md index 977b630fada..a1470e72be9 100644 --- a/website/docs/docs/build/measures.md +++ b/website/docs/docs/build/measures.md @@ -10,6 +10,10 @@ Measures are aggregations performed on columns in your model. They can be used a Measures have several inputs, which are described in the following table along with their field types. +:::tip +Note that we use the double colon (::) to indicate whether a parameter is nested within another parameter. So for example,` config::config_name` means the `config_name` parameter is nested under `config`. +::: + import MeasuresParameters from '/snippets/_sl-measures-parameters.md'; @@ -18,16 +22,41 @@ import MeasuresParameters from '/snippets/_sl-measures-parameters.md'; An example of the complete YAML measures spec is below. The actual configuration of your measures will depend on the aggregation you're using. + + +```yaml +semantic_models: + - name: semantic_model_name + ..rest of the semantic model config + measures: + - name: The name of the measure + description: 'same as always' ## Optional + agg: the aggregation type. + expr: the field + agg_params: 'specific aggregation properties such as a percentile' ## Optional + agg_time_dimension: The time field. Defaults to the default agg time dimension for the semantic model. ## Optional + non_additive_dimension: 'Use these configs when you need non-additive dimensions.' ## Optional + [config](/reference/resource-properties/config): Use the config property to specify configurations for your measure. ## Optional + [meta](/reference/resource-configs/meta): {} Set metadata for a resource and organize resources. Accepts plain text, spaces, and quotes. ## Optional +``` + + + + ```yaml -measures: - - name: The name of the measure - description: 'same as always' ## Optional - agg: the aggregation type. - expr: the field - agg_params: 'specific aggregation properties such as a percentile' ## Optional - agg_time_dimension: The time field. Defaults to the default agg time dimension for the semantic model. ## Optional - non_additive_dimension: 'Use these configs when you need non-additive dimensions.' ## Optional +semantic_models: + - name: semantic_model_name + ..rest of the semantic model config + measures: + - name: The name of the measure + description: 'same as always' ## Optional + agg: the aggregation type. + expr: the field + agg_params: 'specific aggregation properties such as a percentile' ## Optional + agg_time_dimension: The time field. Defaults to the default agg time dimension for the semantic model. ## Optional + non_additive_dimension: 'Use these configs when you need non-additive dimensions.' ## Optional ``` + ### Name @@ -96,6 +125,96 @@ If you use the `dayofweek` function in the `expr` parameter with the legacy Snow ### Model with different aggregations + + +```yaml +semantic_models: + - name: transactions + description: A record of every transaction that takes place. Carts are considered multiple transactions for each SKU. + model: ref('schema.transactions') + defaults: + agg_time_dimension: transaction_date + +# --- entities --- + entities: + - name: transaction_id + type: primary + - name: customer_id + type: foreign + - name: store_id + type: foreign + - name: product_id + type: foreign + +# --- measures --- + measures: + - name: transaction_amount_usd + description: Total USD value of transactions + expr: transaction_amount_usd + agg: sum + config: + meta: + used_in_reporting: true + - name: transaction_amount_usd_avg + description: Average USD value of transactions + expr: transaction_amount_usd + agg: average + - name: transaction_amount_usd_max + description: Maximum USD value of transactions + expr: transaction_amount_usd + agg: max + - name: transaction_amount_usd_min + description: Minimum USD value of transactions + expr: transaction_amount_usd + agg: min + - name: quick_buy_transactions + description: The total transactions bought as quick buy + expr: quick_buy_flag + agg: sum_boolean + - name: distinct_transactions_count + description: Distinct count of transactions + expr: transaction_id + agg: count_distinct + - name: transaction_amount_avg + description: The average value of transactions + expr: transaction_amount_usd + agg: average + - name: transactions_amount_usd_valid # Notice here how we use expr to compute the aggregation based on a condition + description: The total USD value of valid transactions only + expr: CASE WHEN is_valid = True then transaction_amount_usd else 0 end + agg: sum + - name: transactions + description: The average value of transactions. + expr: transaction_amount_usd + agg: average + - name: p99_transaction_value + description: The 99th percentile transaction value + expr: transaction_amount_usd + agg: percentile + agg_params: + percentile: .99 + use_discrete_percentile: False # False calculates the continuous percentile, True calculates the discrete percentile. + - name: median_transaction_value + description: The median transaction value + expr: transaction_amount_usd + agg: median + +# --- dimensions --- + dimensions: + - name: transaction_date + type: time + expr: date_trunc('day', ts) # expr refers to underlying column ts + type_params: + time_granularity: day + - name: is_bulk_transaction + type: categorical + expr: case when quantity > 10 then true else false end + +``` + + + + ```yaml semantic_models: - name: transactions @@ -177,6 +296,7 @@ semantic_models: expr: case when quantity > 10 then true else false end ``` + ### Non-additive dimensions diff --git a/website/docs/docs/build/simple.md b/website/docs/docs/build/simple.md index f57d498d290..4a3aac32708 100644 --- a/website/docs/docs/build/simple.md +++ b/website/docs/docs/build/simple.md @@ -24,8 +24,8 @@ Note that we use the double colon (::) to indicate whether a parameter is nested | `type_params` | The type parameters of the metric. | Required | | `measure` | A list of measure inputs | Required | | `measure:name` | The measure you're referencing. | Required | -| `measure:fill_nulls_with` | Set the value in your metric definition instead of null (such as zero). | Optional | -| `measure:join_to_timespine` | Boolean that indicates if the aggregated measure should be joined to the time spine table to fill in missing dates. Default `false`. | Optional | +| `measure::fill_nulls_with` | Set the value in your metric definition instead of null (such as zero). | Optional | +| `measure::join_to_timespine` | Boolean that indicates if the aggregated measure should be joined to the time spine table to fill in missing dates. Default `false`. | Optional | The following displays the complete specification for simple metrics, along with an example. diff --git a/website/docs/reference/resource-configs/meta.md b/website/docs/reference/resource-configs/meta.md index fa0d48c299e..3d861fc5360 100644 --- a/website/docs/reference/resource-configs/meta.md +++ b/website/docs/reference/resource-configs/meta.md @@ -16,10 +16,9 @@ hide_table_of_contents: true { label: 'Analyses', value: 'analyses', }, { label: 'Macros', value: 'macros', }, { label: 'Exposures', value: 'exposures', }, - { label: 'Semantic Models', value: 'semantic models', }, + { label: 'Semantic models', value: 'semantic models', }, { label: 'Metrics', value: 'metrics', }, { label: 'Saved queries', value: 'saved queries', }, - { label: 'Dimensions', value: 'dimensions', }, ] }> @@ -180,6 +179,10 @@ exposures: +Configure `meta` in the your semantic models YAML file or under the `semantic-models` config block in the `dbt_project.yml` file. + + + ```yml @@ -191,6 +194,39 @@ semantic_models: ``` + + + + +[Dimensions](/docs/build/dimensions), [entities](/docs/build/entities), and [measures](/docs/build/measures) can also have their own `meta` configurations. + + + +```yml +semantic_models: + - name: semantic_model_name + config: + meta: {} + + dimensions: + - name: dimension_name + config: + meta: {} + + entities: + - name: entity_name + config: + meta: {} + + measures: + - name: measure_name + config: + meta: {} + +``` + + + The `meta` config can also be defined under the `semantic-models` config block in `dbt_project.yml`. See [configs and properties](/reference/configs-and-properties) for details. @@ -250,25 +286,7 @@ saved_queries: ``` - - - - - -```yml -semantic_models: - - name: semantic_model - ... - [dimensions](/docs/build/dimensions): - - name: order_date - config: - meta: {} -``` - - - - ## Definition @@ -360,9 +378,54 @@ models: -### Assign owner and additional metadata to dimensions +### Assign meta to semantic model + -The following example shows how to assign a `data_owner` and additional metadata value to a dimension in a semantic model. +The following example shows how to assign a `meta` value to a semantic model in the `semantic_model.yml` file and `dbt_project.yml` file: + + + + +```yaml +semantic_models: + - name: transaction + model: ref('fact_transactions') + description: "Transaction fact table at the transaction level. This table contains one row per transaction and includes the transaction timestamp." + defaults: + agg_time_dimension: transaction_date + config: + meta: + data_owner: "Finance team" + used_in_reporting: true +``` + + + + + +```yaml +semantic-models: + jaffle_shop: + +meta: + used_in_reporting: true +``` + + + +### Assign meta to dimensions, measures, entities + + + +Available in dbt version 1.9 and later. + + + + + + + + +The following example shows how to assign a `meta` value to a [dimension](/docs/build/dimensions), [entity](/docs/build/entities), and [measure](/docs/build/measures) in a semantic model: @@ -370,16 +433,32 @@ The following example shows how to assign a `data_owner` and additional metadata semantic_models: - name: semantic_model ... - [dimensions](/docs/build/dimensions): - - name: order_date - meta: - data_owner: "Finance team" - used_in_reporting: true + dimensions: + - name: order_date + type: time + meta: + data_owner: "Finance team" + used_in_reporting: true + entities: + - name: customer_id + type: primary + meta: + description: "Unique identifier for customers" + data_owner: "Sales team" + used_in_reporting: false + measures: + - name: count_of_users + expr: user_id + meta: + used_in_reporting: true ``` + + + -This second example shows how to assign a `data_owner` and additional metadata value to a dimension in the `dbt_project.yml` file using the `+meta` syntax. +This second example shows how to assign a `data_owner` and additional metadata value to a dimension in the `dbt_project.yml` file using the `+meta` syntax. The similar syntax can be used for entities and measures. @@ -395,3 +474,6 @@ semantic-models: ``` + + + diff --git a/website/snippets/_sl-measures-parameters.md b/website/snippets/_sl-measures-parameters.md index 728d63c6b4f..09916714b96 100644 --- a/website/snippets/_sl-measures-parameters.md +++ b/website/snippets/_sl-measures-parameters.md @@ -9,3 +9,4 @@ | `agg_time_dimension` | The time field. Defaults to the default agg time dimension for the semantic model. | Optional | 1.6 and higher | | `label` | String that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as orders_total or "orders_total"). Available in dbt version 1.7 or higher. | Optional | `create_metric` | Create a `simple` metric from a measure by setting `create_metric: True`. The `label` and `description` attributes will be automatically propagated to the created metric. Available in dbt version 1.7 or higher. | Optional | +| `config::meta` | Use [`meta`](/reference/resource-configs/meta), nested under [`config`](/reference/resource-properties/config), to set metadata for a resource and organize resources. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | From 7824b08ee66b24685b0caa9550fdf17ebf6bff83 Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Mon, 18 Nov 2024 16:32:51 -0500 Subject: [PATCH 4/9] update --- website/docs/docs/build/dimensions.md | 9 ++------- website/docs/docs/build/measures.md | 4 ---- website/snippets/_sl-measures-parameters.md | 2 +- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/website/docs/docs/build/dimensions.md b/website/docs/docs/build/dimensions.md index 424fd72e222..a53869bef3c 100644 --- a/website/docs/docs/build/dimensions.md +++ b/website/docs/docs/build/dimensions.md @@ -14,10 +14,6 @@ Groups are defined within semantic models, alongside entities and measures, and All dimensions require a `name`, `type`, and can optionally include an `expr` parameter. The `name` for your Dimension must be unique within the same semantic model. -:::tip -Note that we use the double colon (::) to indicate whether a parameter is nested within another parameter. So for example,` config::config_name` means the `config_name` parameter is nested under `config`. -::: - | Parameter | Description | Type | @@ -28,7 +24,7 @@ Note that we use the double colon (::) to indicate whether a parameter is nested | `description` | A clear description of the dimension | Optional | | `expr` | Defines the underlying column or SQL query for a dimension. If no `expr` is specified, MetricFlow will use the column with the same name as the group. You can use the column name itself to input a SQL expression. | Optional | | `label` | A recommended string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | -| `config` | Use the [`config`](/reference/resource-properties/config) property to specify configurations for your metric. Supports [`meta`](/reference/resource-configs/meta), [`group`](/reference/resource-configs/group), and [`enabled`](/reference/resource-configs/enabled) configs. | Optional | +| `config` | Use the [`config`](/reference/resource-properties/config) property to specify configurations for your metric. Supports the [`meta`](/reference/resource-configs/meta) property, nested under `config`. | Optional | @@ -42,8 +38,7 @@ Note that we use the double colon (::) to indicate whether a parameter is nested | `description` | A clear description of the dimension | Optional | | `expr` | Defines the underlying column or SQL query for a dimension. If no `expr` is specified, MetricFlow will use the column with the same name as the group. You can use the column name itself to input a SQL expression. | Optional | | `label` | A recommended string that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | -| `config` | Use the [`config`](/reference/resource-properties/config) property to specify configurations for your metric. Supports [`meta`](/reference/resource-configs/meta), [`group`](/reference/resource-configs/group), and [`enabled`](/reference/resource-configs/enabled) configs. | Optional | -| `config::meta` | Use [`meta`](/reference/resource-configs/meta), nested under `config`, to set metadata for a resource and organize resources. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | +| `config` | Use the [`config`](/reference/resource-properties/config) property to specify configurations for your metric. Supports the [`meta`](/reference/resource-configs/meta) configs. | Optional | diff --git a/website/docs/docs/build/measures.md b/website/docs/docs/build/measures.md index a1470e72be9..c12cf67fdc3 100644 --- a/website/docs/docs/build/measures.md +++ b/website/docs/docs/build/measures.md @@ -10,10 +10,6 @@ Measures are aggregations performed on columns in your model. They can be used a Measures have several inputs, which are described in the following table along with their field types. -:::tip -Note that we use the double colon (::) to indicate whether a parameter is nested within another parameter. So for example,` config::config_name` means the `config_name` parameter is nested under `config`. -::: - import MeasuresParameters from '/snippets/_sl-measures-parameters.md'; diff --git a/website/snippets/_sl-measures-parameters.md b/website/snippets/_sl-measures-parameters.md index 09916714b96..8058474d1ef 100644 --- a/website/snippets/_sl-measures-parameters.md +++ b/website/snippets/_sl-measures-parameters.md @@ -9,4 +9,4 @@ | `agg_time_dimension` | The time field. Defaults to the default agg time dimension for the semantic model. | Optional | 1.6 and higher | | `label` | String that defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as orders_total or "orders_total"). Available in dbt version 1.7 or higher. | Optional | `create_metric` | Create a `simple` metric from a measure by setting `create_metric: True`. The `label` and `description` attributes will be automatically propagated to the created metric. Available in dbt version 1.7 or higher. | Optional | -| `config::meta` | Use [`meta`](/reference/resource-configs/meta), nested under [`config`](/reference/resource-properties/config), to set metadata for a resource and organize resources. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | +| `config` | Use the [`config`](/reference/resource-properties/config) property to specify configurations for your metric. Supports the [`meta`](/reference/resource-configs/meta) property, nested under `config`. | Optional | From 7118aa5b95586e7c6beacc67617868aa867e2314 Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Tue, 10 Dec 2024 10:54:15 +0000 Subject: [PATCH 5/9] Update dimensions.md --- website/docs/docs/build/dimensions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/website/docs/docs/build/dimensions.md b/website/docs/docs/build/dimensions.md index 70715537f9a..f1a2f10bf89 100644 --- a/website/docs/docs/build/dimensions.md +++ b/website/docs/docs/build/dimensions.md @@ -22,6 +22,7 @@ All dimensions require a `name`, `type`, and can optionally include an `expr` pa | `description` | A clear description of the dimension. | Optional | String | | `expr` | Defines the underlying column or SQL query for a dimension. If no `expr` is specified, MetricFlow will use the column with the same name as the group. You can use the column name itself to input a SQL expression. | Optional | String | | `label` | Defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | String | +| [`meta`](/reference/resource-configs/meta) | Set metadata for a resource and organize resources. Accepts plain text, spaces, and quotes. | Optional | {} | Refer to the following for the complete specification for dimensions: From e7f407f013642f639012c34a7560a6941c6d0906 Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Tue, 10 Dec 2024 10:59:47 +0000 Subject: [PATCH 6/9] Update website/docs/docs/build/dimensions.md --- website/docs/docs/build/dimensions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/build/dimensions.md b/website/docs/docs/build/dimensions.md index f1a2f10bf89..609b2168906 100644 --- a/website/docs/docs/build/dimensions.md +++ b/website/docs/docs/build/dimensions.md @@ -22,7 +22,7 @@ All dimensions require a `name`, `type`, and can optionally include an `expr` pa | `description` | A clear description of the dimension. | Optional | String | | `expr` | Defines the underlying column or SQL query for a dimension. If no `expr` is specified, MetricFlow will use the column with the same name as the group. You can use the column name itself to input a SQL expression. | Optional | String | | `label` | Defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | String | -| [`meta`](/reference/resource-configs/meta) | Set metadata for a resource and organize resources. Accepts plain text, spaces, and quotes. | Optional | {} | +| [`meta`](/reference/resource-configs/meta) | Set metadata for a resource and organize resources. Accepts plain text, spaces, and quotes. | Optional | | Refer to the following for the complete specification for dimensions: From 5617cf82b9666c01087d5a0c7d3afa1f63d60ded Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:18:14 +0000 Subject: [PATCH 7/9] Update dimensions.md --- website/docs/docs/build/dimensions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/build/dimensions.md b/website/docs/docs/build/dimensions.md index 609b2168906..4ad06e5626e 100644 --- a/website/docs/docs/build/dimensions.md +++ b/website/docs/docs/build/dimensions.md @@ -22,7 +22,7 @@ All dimensions require a `name`, `type`, and can optionally include an `expr` pa | `description` | A clear description of the dimension. | Optional | String | | `expr` | Defines the underlying column or SQL query for a dimension. If no `expr` is specified, MetricFlow will use the column with the same name as the group. You can use the column name itself to input a SQL expression. | Optional | String | | `label` | Defines the display value in downstream tools. Accepts plain text, spaces, and quotes (such as `orders_total` or `"orders_total"`). | Optional | String | -| [`meta`](/reference/resource-configs/meta) | Set metadata for a resource and organize resources. Accepts plain text, spaces, and quotes. | Optional | | +| [`meta`](/reference/resource-configs/meta) | Set metadata for a resource and organize resources. Accepts plain text, spaces, and quotes. | Optional | Dictionary | Refer to the following for the complete specification for dimensions: From af4165628afc27b9323cf788d9e059ba0abdccc9 Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Fri, 13 Dec 2024 11:39:50 +0000 Subject: [PATCH 8/9] add config --- website/docs/docs/build/dimensions.md | 15 ++++++---- .../docs/reference/resource-configs/meta.md | 29 +++++++++++-------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/website/docs/docs/build/dimensions.md b/website/docs/docs/build/dimensions.md index 4ad06e5626e..25a1c729a7a 100644 --- a/website/docs/docs/build/dimensions.md +++ b/website/docs/docs/build/dimensions.md @@ -62,8 +62,9 @@ semantic_models: type_params: time_granularity: day label: "Date of transaction" # Recommend adding a label to provide more context to users consuming the data - meta: - data_owner: "Finance team" + config: + meta: + data_owner: "Finance team" expr: ts - name: is_bulk type: categorical @@ -147,8 +148,9 @@ dimensions: - name: is_bulk_transaction type: categorical expr: case when quantity > 10 then true else false end - meta: - usage: "Filter to identify bulk transactions, like where quantity > 10." + config: + meta: + usage: "Filter to identify bulk transactions, like where quantity > 10." ``` @@ -192,8 +194,9 @@ dimensions: type: time label: "Date of creation" expr: ts_created # ts_created is the underlying column name from the table - meta: - notes: "Only valid for orders from 2022 onward" + config: + meta: + notes: "Only valid for orders from 2022 onward" is_partition: True type_params: time_granularity: day diff --git a/website/docs/reference/resource-configs/meta.md b/website/docs/reference/resource-configs/meta.md index dcf04354dd6..7214dac560b 100644 --- a/website/docs/reference/resource-configs/meta.md +++ b/website/docs/reference/resource-configs/meta.md @@ -436,21 +436,24 @@ semantic_models: dimensions: - name: order_date type: time - meta: - data_owner: "Finance team" - used_in_reporting: true + config: + meta: + data_owner: "Finance team" + used_in_reporting: true entities: - name: customer_id type: primary - meta: - description: "Unique identifier for customers" - data_owner: "Sales team" - used_in_reporting: false + config: + meta: + description: "Unique identifier for customers" + data_owner: "Sales team" + used_in_reporting: false measures: - name: count_of_users expr: user_id - meta: - used_in_reporting: true + config: + meta: + used_in_reporting: true ``` @@ -468,11 +471,13 @@ semantic-models: ... [dimensions](/docs/build/dimensions): - name: order_date - +meta: - data_owner: "Finance team" - used_in_reporting: true + config: + meta: + data_owner: "Finance team" + used_in_reporting: true ``` + From 5fc9357903e14a525edc9c1a4f076f5090203f33 Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Tue, 24 Dec 2024 10:42:29 +0000 Subject: [PATCH 9/9] clarify meta acceptance --- website/docs/reference/resource-configs/meta.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/resource-configs/meta.md b/website/docs/reference/resource-configs/meta.md index 7214dac560b..313595ea497 100644 --- a/website/docs/reference/resource-configs/meta.md +++ b/website/docs/reference/resource-configs/meta.md @@ -290,7 +290,7 @@ saved_queries: ## Definition -The `meta` field can be used to set metadata for a resource. This metadata is compiled into the `manifest.json` file generated by dbt, and is viewable in the auto-generated documentation. +The `meta` field can be used to set metadata for a resource and accepts any key-value pairs. This metadata is compiled into the `manifest.json` file generated by dbt, and is viewable in the auto-generated documentation. Depending on the resource you're configuring, `meta` may be available within the `config` property, and/or as a top-level key. (For backwards compatibility, `meta` is often (but not always) supported as a top-level key, though without the capabilities of config inheritance.)