Add exports config to YAML spec for saved queries (#190)

.changes/unreleased/Features-20231024-162842.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add exports configuration to YAML spec.
+time: 2023-10-24T16:28:42.013032-07:00
+custom:
+  Author: courtneyholcomb
+  Issue: "189"

.github/workflows/ci-schema-consistency.yaml

@@ -26,7 +26,7 @@ jobs:
           python-version: "3.9"
 
       - name: Generate JSON Schema
-        run: hatch run dev-env:python dbt_semantic_interfaces/parsing/generate_json_schema_file.py
+        run: make json_schema
 
       - name: Schema Consistency Check
         run: |

Makefile

@@ -1,4 +1,4 @@
-.PHONY: run install-hatch overwrite-pre-commit install test
+.PHONY: run install-hatch overwrite-pre-commit install test lint json_schema
 
 run:
 	export FORMAT_JSON_LOGS="1"
@@ -18,3 +18,6 @@ test:
 
 lint:
 	hatch run dev-env:pre-commit run --show-diff-on-failure --color=always --all-files
+
+json_schema:
+	hatch run dev-env:python dbt_semantic_interfaces/parsing/generate_json_schema_file.py

dbt_semantic_interfaces/implementations/export.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import Field
+from typing_extensions import override
+
+from dbt_semantic_interfaces.implementations.base import HashableBaseModel
+from dbt_semantic_interfaces.protocols import ProtocolHint
+from dbt_semantic_interfaces.protocols.export import Export, ExportConfig
+from dbt_semantic_interfaces.type_enums.export_destination_type import (
+    ExportDestinationType,
+)
+
+
+class PydanticExportConfig(HashableBaseModel, ProtocolHint[ExportConfig]):
+    """Pydantic implementation of ExportConfig.
+
+    Note on `schema_name`: `schema` is an existing BaseModel attribute, so we need to alias it here.
+    `Field.alias="schema"` enables using the `schema` key in YAML. `Config.allow_population_by_field_name`
+    enables parsing for both `schema` and `schema_name` when deserializing from JSON.
+    """
+
+    class Config:  # noqa: D
+        allow_population_by_field_name = True
+
+    @override
+    def _implements_protocol(self) -> ExportConfig:
+        return self
+
+    export_as: ExportDestinationType
+    schema_name: Optional[str] = Field(alias="schema", default=None)
+    alias: Optional[str] = None
+
+
+class PydanticExport(HashableBaseModel, ProtocolHint[Export]):
+    """Pydantic implementation of Export."""
+
+    @override
+    def _implements_protocol(self) -> Export:
+        return self
+
+    name: str
+    config: PydanticExportConfig

dbt_semantic_interfaces/implementations/saved_query.py

@@ -8,6 +8,7 @@
     HashableBaseModel,
     ModelWithMetadataParsing,
 )
+from dbt_semantic_interfaces.implementations.export import PydanticExport
 from dbt_semantic_interfaces.implementations.filters.where_filter import (
     PydanticWhereFilterIntersection,
 )
@@ -31,3 +32,4 @@ def _implements_protocol(self) -> SavedQuery:
     description: Optional[str] = None
     metadata: Optional[PydanticMetadata] = None
     label: Optional[str] = None
+    exports: Optional[List[PydanticExport]] = None

dbt_semantic_interfaces/parsing/generated_json_schemas/default_explicit_schema.json

@@ -139,6 +139,47 @@
             ],
             "type": "object"
         },
+        "export_config_schema": {
+            "$id": "export_config_schema",
+            "additionalProperties": false,
+            "properties": {
+                "alias": {
+                    "type": "string"
+                },
+                "export_as": {
+                    "enum": [
+                        "TABLE",
+                        "VIEW",
+                        "table",
+                        "view"
+                    ]
+                },
+                "schema": {
+                    "type": "string"
+                }
+            },
+            "required": [
+                "export_as"
+            ],
+            "type": "object"
+        },
+        "export_schema": {
+            "$id": "export_schema",
+            "additionalProperties": false,
+            "properties": {
+                "config": {
+                    "$ref": "#/definitions/export_config_schema"
+                },
+                "name": {
+                    "type": "string"
+                }
+            },
+            "required": [
+                "name",
+                "config"
+            ],
+            "type": "object"
+        },
         "filter_schema": {
             "$id": "filter_schema",
             "oneOf": [
@@ -430,6 +471,12 @@
                 "description": {
                     "type": "string"
                 },
+                "exports": {
+                    "items": {
+                        "$ref": "#/definitions/export_schema"
+                    },
+                    "type": "array"
+                },
                 "group_bys": {
                     "items": {
                         "type": "string"

dbt_semantic_interfaces/parsing/schemas.py

@@ -39,6 +39,10 @@
 
 time_dimension_type_values = ["TIME", "time"]
 
+export_destination_type_values = ["TABLE", "VIEW"]
+export_destination_type_values += [x.lower() for x in export_destination_type_values]
+
+
 filter_schema = {
     "$id": "filter_schema",
     "oneOf": [
@@ -288,6 +292,29 @@
     "required": ["time_spine_table_configurations"],
 }
 
+export_config_schema = {
+    "$id": "export_config_schema",
+    "type": "object",
+    "properties": {
+        "export_as": {"enum": export_destination_type_values},
+        "schema": {"type": "string"},
+        "alias": {"type": "string"},
+    },
+    "required": ["export_as"],
+    "additionalProperties": False,
+}
+
+
+export_schema = {
+    "$id": "export_schema",
+    "type": "object",
+    "properties": {
+        "name": {"type": "string"},
+        "config": {"$ref": "export_config_schema"},
+    },
+    "required": ["name", "config"],
+    "additionalProperties": False,
+}
 
 saved_query_schema = {
     "$id": "saved_query_schema",
@@ -305,6 +332,7 @@
         },
         "where": {"$ref": "filter_schema"},
         "label": {"type": "string"},
+        "exports": {"type": "array", "items": {"$ref": "export_schema"}},
     },
     "required": ["name", "metrics"],
     "additionalProperties": False,
@@ -355,6 +383,8 @@
     node_relation_schema["$id"]: node_relation_schema,
     semantic_model_defaults_schema["$id"]: semantic_model_defaults_schema,
     time_spine_table_configuration_schema["$id"]: time_spine_table_configuration_schema,
+    export_schema["$id"]: export_schema,
+    export_config_schema["$id"]: export_config_schema,
 }
 
 resources: List[Tuple[str, Resource]] = [(str(k), DRAFT7.create_resource(v)) for k, v in schema_store.items()]

dbt_semantic_interfaces/protocols/export.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from abc import abstractmethod
+from typing import Optional, Protocol
+
+from dbt_semantic_interfaces.type_enums.export_destination_type import (
+    ExportDestinationType,
+)
+
+
+class Export(Protocol):
+    """Configuration for writing query results to a table."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:  # noqa: D
+        pass
+
+    @property
+    @abstractmethod
+    def config(self) -> ExportConfig:  # noqa: D
+        pass
+
+
+class ExportConfig(Protocol):
+    """Nested configuration attributes for exports."""
+
+    @property
+    @abstractmethod
+    def export_as(self) -> ExportDestinationType:
+        """Type of destination to write export to."""
+        pass
+
+    @property
+    @abstractmethod
+    def schema_name(self) -> Optional[str]:
+        """Schema to write export to. Defaults to deployment schema."""
+        pass
+
+    @property
+    @abstractmethod
+    def alias(self) -> Optional[str]:
+        """Name for table/filte export is written to. Defaults to export name."""
+        pass

dbt_semantic_interfaces/protocols/saved_query.py

@@ -1,6 +1,7 @@
 from abc import abstractmethod
 from typing import Optional, Protocol, Sequence
 
+from dbt_semantic_interfaces.protocols.export import Export
 from dbt_semantic_interfaces.protocols.metadata import Metadata
 from dbt_semantic_interfaces.protocols.where_filter import WhereFilterIntersection
 
@@ -44,3 +45,9 @@ def where(self) -> Optional[WhereFilterIntersection]:
     def label(self) -> Optional[str]:
         """Returns a string representing a human readable label for the saved query."""
         pass
+
+    @property
+    @abstractmethod
+    def exports(self) -> Optional[Sequence[Export]]:
+        """Exports that can run using this saved query."""
+        pass

dbt_semantic_interfaces/type_enums/export_destination_type.py

@@ -0,0 +1,8 @@
+from dbt_semantic_interfaces.enum_extension import ExtendedEnum
+
+
+class ExportDestinationType(ExtendedEnum):
+    """Types of destinations that exports can be written to."""
+
+    TABLE = "table"
+    VIEW = "view"

tests/fixtures/semantic_manifest_yamls/simple_semantic_manifest/saved_queries.yaml

@@ -10,3 +10,9 @@ saved_query:
     - Dimension('listing__capacity_latest')
   where:
     - "{{ Dimension('listing__capacity_latest') }} > 3"
+  exports:
+    - name: bookings
+      config:
+        export_as: table
+        schema: exports_schema
+        alias: bookings_export_table

tests/parsing/test_saved_query_parsing.py

@@ -4,6 +4,9 @@
     parse_yaml_files_to_semantic_manifest,
 )
 from dbt_semantic_interfaces.parsing.objects import YamlConfigFile
+from dbt_semantic_interfaces.type_enums.export_destination_type import (
+    ExportDestinationType,
+)
 from tests.example_project_configuration import (
     EXAMPLE_PROJECT_CONFIGURATION_YAML_CONFIG_FILE,
 )
@@ -134,3 +137,43 @@ def test_saved_query_where() -> None:
     assert saved_query.where is not None
     assert len(saved_query.where.where_filters) == 1
     assert where == saved_query.where.where_filters[0].where_sql_template
+
+
+def test_saved_query_exports() -> None:
+    """Test for parsing exports referenced in a saved query."""
+    yaml_contents = textwrap.dedent(
+        """\
+        saved_query:
+          name: test_exports
+          metrics:
+            - test_metric_a
+          exports:
+            - name: test_exports1
+              config:
+                export_as: VIEW
+                schema: my_schema
+                alias: my_view_name
+            - name: test_exports2
+              config:
+                export_as: table
+        """
+    )
+    file = YamlConfigFile(filepath="inline_for_test", contents=yaml_contents)
+
+    build_result = parse_yaml_files_to_semantic_manifest(files=[file, EXAMPLE_PROJECT_CONFIGURATION_YAML_CONFIG_FILE])
+
+    assert len(build_result.semantic_manifest.saved_queries) == 1
+    saved_query = build_result.semantic_manifest.saved_queries[0]
+    assert saved_query.exports and len(saved_query.exports) == 2
+    names_to_exports = {export.name: export for export in saved_query.exports}
+    assert set(names_to_exports.keys()) == {"test_exports1", "test_exports2"}
+
+    export1_config = names_to_exports["test_exports1"].config
+    assert export1_config.export_as == ExportDestinationType.VIEW
+    assert export1_config.schema_name == "my_schema"
+    assert export1_config.alias == "my_view_name"
+
+    export2_config = names_to_exports["test_exports2"].config
+    assert export2_config.export_as == ExportDestinationType.TABLE
+    assert export2_config.schema_name is None
+    assert export2_config.alias is None

tests/test_implements_satisfy_protocols.py

@@ -14,6 +14,7 @@
     PydanticMeasureAggregationParameters,
     PydanticNonAdditiveDimensionParameters,
 )
+from dbt_semantic_interfaces.implementations.export import PydanticExport
 from dbt_semantic_interfaces.implementations.filters.where_filter import (
     PydanticWhereFilter,
 )
@@ -124,6 +125,7 @@
     description=OPTIONAL_STR_STRATEGY,
     metadata=OPTIONAL_METADATA_STRATEGY,
     label=OPTIONAL_STR_STRATEGY,
+    exports=from_type(List[PydanticExport]),
 )