Merge branch 'main' into unit_testing_feature_branch

.changes/unreleased/Features-20231206-181458.yaml

@@ -0,0 +1,7 @@
+kind: Features
+body: Adds support for parsing conversion metric related properties for the semantic
+  layer.
+time: 2023-12-06T18:14:58.688221-05:00
+custom:
+  Author: WilliamDee
+  Issue: "9203"

.changes/unreleased/Fixes-20231127-165244.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Ensure we produce valid jsonschema schemas for manifest, catalog, run-results,
+  and sources
+time: 2023-11-27T16:52:44.590313-08:00
+custom:
+  Author: QMalcolm
+  Issue: "8991"

core/dbt/contracts/graph/manifest.py

@@ -1581,7 +1581,7 @@ def __init__(self, macros) -> None:
 
 
 @dataclass
-@schema_version("manifest", 11)
+@schema_version("manifest", 12)
 class WritableManifest(ArtifactMixin):
     nodes: Mapping[UniqueID, ManifestNode] = field(
         metadata=dict(description=("The nodes defined in the dbt project and its dependencies"))
@@ -1654,6 +1654,7 @@ def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]:
             ("manifest", 8),
             ("manifest", 9),
             ("manifest", 10),
+            ("manifest", 11),
         ]
 
     @classmethod

core/dbt/contracts/graph/metrics.py

@@ -5,7 +5,7 @@
 
 
 DERIVED_METRICS = [MetricType.DERIVED, MetricType.RATIO]
-BASE_METRICS = [MetricType.SIMPLE, MetricType.CUMULATIVE]
+BASE_METRICS = [MetricType.SIMPLE, MetricType.CUMULATIVE, MetricType.CONVERSION]
 
 
 class MetricReference(object):

core/dbt/contracts/graph/nodes.py

@@ -22,6 +22,7 @@
     SourceFileMetadata,
 )
 from dbt.contracts.graph.unparsed import (
+    ConstantPropertyInput,
     Docs,
     ExposureType,
     ExternalTable,
@@ -63,7 +64,11 @@
     TimeDimensionReference,
 )
 from dbt_semantic_interfaces.references import MetricReference as DSIMetricReference
-from dbt_semantic_interfaces.type_enums import MetricType, TimeGranularity
+from dbt_semantic_interfaces.type_enums import (
+    ConversionCalculationType,
+    MetricType,
+    TimeGranularity,
+)
 
 from .model_config import (
     NodeConfig,
@@ -1550,6 +1555,16 @@ def post_aggregation_reference(self) -> DSIMetricReference:
         return DSIMetricReference(element_name=self.alias or self.name)
 
 
+@dataclass
+class ConversionTypeParams(dbtClassMixin):
+    base_measure: MetricInputMeasure
+    conversion_measure: MetricInputMeasure
+    entity: str
+    calculation: ConversionCalculationType = ConversionCalculationType.CONVERSION_RATE
+    window: Optional[MetricTimeWindow] = None
+    constant_properties: Optional[List[ConstantPropertyInput]] = None
+
+
 @dataclass
 class MetricTypeParams(dbtClassMixin):
     measure: Optional[MetricInputMeasure] = None
@@ -1560,6 +1575,7 @@ class MetricTypeParams(dbtClassMixin):
     window: Optional[MetricTimeWindow] = None
     grain_to_date: Optional[TimeGranularity] = None
     metrics: Optional[List[MetricInput]] = None
+    conversion_type_params: Optional[ConversionTypeParams] = None
 
 
 @dataclass

core/dbt/contracts/graph/unparsed.py

@@ -19,6 +19,7 @@
 from dbt.exceptions import CompilationError, ParsingError, DbtInternalError
 
 from dbt.dataclass_schema import dbtClassMixin, StrEnum, ExtensibleDbtClassMixin, ValidationError
+from dbt_semantic_interfaces.type_enums import ConversionCalculationType
 
 from dataclasses import dataclass, field
 from datetime import timedelta
@@ -602,6 +603,24 @@ class UnparsedMetricInput(dbtClassMixin):
     offset_to_grain: Optional[str] = None  # str is really a TimeGranularity Enum
 
 
+@dataclass
+class ConstantPropertyInput(dbtClassMixin):
+    base_property: str
+    conversion_property: str
+
+
+@dataclass
+class UnparsedConversionTypeParams(dbtClassMixin):
+    base_measure: Union[UnparsedMetricInputMeasure, str]
+    conversion_measure: Union[UnparsedMetricInputMeasure, str]
+    entity: str
+    calculation: str = (
+        ConversionCalculationType.CONVERSION_RATE.value
+    )  # ConversionCalculationType Enum
+    window: Optional[str] = None
+    constant_properties: Optional[List[ConstantPropertyInput]] = None
+
+
 @dataclass
 class UnparsedMetricTypeParams(dbtClassMixin):
     measure: Optional[Union[UnparsedMetricInputMeasure, str]] = None
@@ -611,6 +630,7 @@ class UnparsedMetricTypeParams(dbtClassMixin):
     window: Optional[str] = None
     grain_to_date: Optional[str] = None  # str is really a TimeGranularity Enum
     metrics: Optional[List[Union[UnparsedMetricInput, str]]] = None
+    conversion_type_params: Optional[UnparsedConversionTypeParams] = None
 
 
 @dataclass

core/dbt/contracts/results.py

@@ -254,7 +254,7 @@ def write(self, path: str):
 
 
 @dataclass
-@schema_version("run-results", 5)
+@schema_version("run-results", 6)
 class RunResultsArtifact(ExecutionResult, ArtifactMixin):
     results: Sequence[RunResultOutput]
     args: Dict[str, Any] = field(default_factory=dict)
@@ -280,6 +280,7 @@ def from_execution_results(
     def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]:
         return [
             ("run-results", 4),
+            ("run-results", 5),
         ]
 
     @classmethod

core/dbt/contracts/util.py

@@ -18,6 +18,7 @@
     ValidationError,
 )
 from mashumaro.jsonschema import build_json_schema
+from mashumaro.jsonschema.dialects import DRAFT_2020_12
 import functools
 
 
@@ -197,7 +198,7 @@ class VersionedSchema(dbtClassMixin):
     @classmethod
     @functools.lru_cache
     def json_schema(cls) -> Dict[str, Any]:
-        json_schema_obj = build_json_schema(cls, all_refs=True)
+        json_schema_obj = build_json_schema(cls, dialect=DRAFT_2020_12, with_dialect_uri=True)
         json_schema = json_schema_obj.to_dict()
         json_schema["$id"] = str(cls.dbt_schema_version)
         return json_schema

core/dbt/parser/manifest.py

@@ -1529,43 +1529,66 @@ def _process_refs(
         node.depends_on.add_node(target_model_id)
 
 
-def _process_metric_node(
+def _process_metric_depends_on(
     manifest: Manifest,
     current_project: str,
     metric: Metric,
 ) -> None:
-    """Sets a metric's `input_measures` and `depends_on` properties"""
-
-    # This ensures that if this metrics input_measures have already been set
-    # we skip the work. This could happen either due to recursion or if multiple
-    # metrics derive from another given metric.
-    # NOTE: This does not protect against infinite loops
-    if len(metric.type_params.input_measures) > 0:
-        return
+    """For a given metric, set the `depends_on` property"""
 
-    if metric.type is MetricType.SIMPLE or metric.type is MetricType.CUMULATIVE:
-        assert (
-            metric.type_params.measure is not None
-        ), f"{metric} should have a measure defined, but it does not."
-        metric.type_params.input_measures.append(metric.type_params.measure)
+    assert len(metric.type_params.input_measures) > 0
+    for input_measure in metric.type_params.input_measures:
         target_semantic_model = manifest.resolve_semantic_model_for_measure(
-            target_measure_name=metric.type_params.measure.name,
+            target_measure_name=input_measure.name,
             current_project=current_project,
             node_package=metric.package_name,
         )
         if target_semantic_model is None:
             raise dbt.exceptions.ParsingError(
-                f"A semantic model having a measure `{metric.type_params.measure.name}` does not exist but was referenced.",
+                f"A semantic model having a measure `{input_measure.name}` does not exist but was referenced.",
                 node=metric,
             )
         if target_semantic_model.config.enabled is False:
             raise dbt.exceptions.ParsingError(
-                f"The measure `{metric.type_params.measure.name}` is referenced on disabled semantic model `{target_semantic_model.name}`.",
+                f"The measure `{input_measure.name}` is referenced on disabled semantic model `{target_semantic_model.name}`.",
                 node=metric,
             )
 
         metric.depends_on.add_node(target_semantic_model.unique_id)
 
+
+def _process_metric_node(
+    manifest: Manifest,
+    current_project: str,
+    metric: Metric,
+) -> None:
+    """Sets a metric's `input_measures` and `depends_on` properties"""
+
+    # This ensures that if this metrics input_measures have already been set
+    # we skip the work. This could happen either due to recursion or if multiple
+    # metrics derive from another given metric.
+    # NOTE: This does not protect against infinite loops
+    if len(metric.type_params.input_measures) > 0:
+        return
+
+    if metric.type is MetricType.SIMPLE or metric.type is MetricType.CUMULATIVE:
+        assert (
+            metric.type_params.measure is not None
+        ), f"{metric} should have a measure defined, but it does not."
+        metric.type_params.input_measures.append(metric.type_params.measure)
+        _process_metric_depends_on(
+            manifest=manifest, current_project=current_project, metric=metric
+        )
+    elif metric.type is MetricType.CONVERSION:
+        conversion_type_params = metric.type_params.conversion_type_params
+        assert (
+            conversion_type_params
+        ), f"{metric.name} is a conversion metric and must have conversion_type_params defined."
+        metric.type_params.input_measures.append(conversion_type_params.base_measure)
+        metric.type_params.input_measures.append(conversion_type_params.conversion_measure)
+        _process_metric_depends_on(
+            manifest=manifest, current_project=current_project, metric=metric
+        )
     elif metric.type is MetricType.DERIVED or metric.type is MetricType.RATIO:
         input_metrics = metric.input_metrics
         if metric.type is MetricType.RATIO:

core/dbt/parser/schema_yaml_readers.py

@@ -17,6 +17,7 @@
     UnparsedQueryParams,
     UnparsedSavedQuery,
     UnparsedSemanticModel,
+    UnparsedConversionTypeParams,
 )
 from dbt.contracts.graph.model_config import SavedQueryConfig
 from dbt.contracts.graph.nodes import (
@@ -29,6 +30,7 @@
     MetricTypeParams,
     SemanticModel,
     SavedQuery,
+    ConversionTypeParams,
 )
 from dbt.contracts.graph.saved_queries import Export, ExportConfig, QueryParams
 from dbt.contracts.graph.semantic_layer_common import WhereFilter, WhereFilterIntersection
@@ -52,6 +54,7 @@
 from dbt.dataclass_schema import ValidationError
 from dbt_semantic_interfaces.type_enums import (
     AggregationType,
+    ConversionCalculationType,
     DimensionType,
     EntityType,
     MetricType,
@@ -226,7 +229,7 @@ def _get_time_window(
                     self.yaml.path,
                     "window",
                     {"window": unparsed_window},
-                    f"Invalid window ({unparsed_window}) in cumulative metric. Should be of the form `<count> <granularity>`, "
+                    f"Invalid window ({unparsed_window}) in cumulative/conversion metric. Should be of the form `<count> <granularity>`, "
                     "e.g., `28 days`",
                 )
 
@@ -240,7 +243,7 @@ def _get_time_window(
                     self.yaml.path,
                     "window",
                     {"window": unparsed_window},
-                    f"Invalid time granularity {granularity} in cumulative metric window string: ({unparsed_window})",
+                    f"Invalid time granularity {granularity} in cumulative/conversion metric window string: ({unparsed_window})",
                 )
 
             count = parts[0]
@@ -249,7 +252,7 @@ def _get_time_window(
                     self.yaml.path,
                     "window",
                     {"window": unparsed_window},
-                    f"Invalid count ({count}) in cumulative metric window string: ({unparsed_window})",
+                    f"Invalid count ({count}) in cumulative/conversion metric window string: ({unparsed_window})",
                 )
 
             return MetricTimeWindow(
@@ -295,6 +298,20 @@ def _get_metric_inputs(
 
         return metric_inputs
 
+    def _get_optional_conversion_type_params(
+        self, unparsed: Optional[UnparsedConversionTypeParams]
+    ) -> Optional[ConversionTypeParams]:
+        if unparsed is None:
+            return None
+        return ConversionTypeParams(
+            base_measure=self._get_input_measure(unparsed.base_measure),
+            conversion_measure=self._get_input_measure(unparsed.conversion_measure),
+            entity=unparsed.entity,
+            calculation=ConversionCalculationType(unparsed.calculation),
+            window=self._get_time_window(unparsed.window),
+            constant_properties=unparsed.constant_properties,
+        )
+
     def _get_metric_type_params(self, type_params: UnparsedMetricTypeParams) -> MetricTypeParams:
         grain_to_date: Optional[TimeGranularity] = None
         if type_params.grain_to_date is not None:
@@ -308,6 +325,9 @@ def _get_metric_type_params(self, type_params: UnparsedMetricTypeParams) -> Metr
             window=self._get_time_window(type_params.window),
             grain_to_date=grain_to_date,
             metrics=self._get_metric_inputs(type_params.metrics),
+            conversion_type_params=self._get_optional_conversion_type_params(
+                type_params.conversion_type_params
+            )
             # input measures are calculated via metric processing post parsing
             # input_measures=?,
         )

core/setup.py

@@ -73,7 +73,7 @@
         # These are major-version-0 packages also maintained by dbt-labs. Accept patches.
         "dbt-extractor~=0.5.0",
         "minimal-snowplow-tracker~=0.0.2",
-        "dbt-semantic-interfaces~=0.4.0",
+        "dbt-semantic-interfaces~=0.5.0a2",
         # ----
         # Expect compatibility with all new versions of these packages, so lower bounds only.
         "jsonschema>=3.0",