Skip to content

Commit

Permalink
feat(ingest/lookml): ingest field tags (datahub-project#10792)
Browse files Browse the repository at this point in the history 
Co-authored-by: Harshal Sheth <[email protected]>
  • Loading branch information
@sid-acryl @hsheth2
sid-acryl and hsheth2 authored Jul 1, 2024
1 parent 8b4e302 commit 3e47496
Show file tree
Hide file tree
Showing 23 changed files with 641 additions and 146 deletions.
26 changes: 18 additions & 8 deletions metadata-ingestion/src/datahub/ingestion/source/looker/looker_common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ class ViewField:
view_name: Optional[str] = None
is_primary_key: bool = False
upstream_fields: List[str] = dataclasses_field(default_factory=list)
tags: List[str] = dataclasses_field(default_factory=list)


@dataclass
Expand Down Expand Up @@ -561,21 +562,30 @@ def _get_tag_mce_for_urn(tag_urn: str) -> MetadataChangeEvent:

@staticmethod
def _get_tags_from_field_type(
field_type: ViewFieldType, reporter: SourceReport
field: ViewField, reporter: SourceReport
) -> Optional[GlobalTagsClass]:
if field_type in LookerUtil.type_to_tag_map:
return GlobalTagsClass(
tags=[
schema_field_tags: List[TagAssociationClass] = [
TagAssociationClass(tag=builder.make_tag_urn(tag_name))
for tag_name in field.tags
]

if field.field_type in LookerUtil.type_to_tag_map:
schema_field_tags.extend(
[
TagAssociationClass(tag=tag_name)
for tag_name in LookerUtil.type_to_tag_map[field_type]
for tag_name in LookerUtil.type_to_tag_map[field.field_type]
]
)
else:
reporter.report_warning(
"lookml",
f"Failed to map view field type {field_type}. Won't emit tags for it",
f"Failed to map view field type {field.field_type}. Won't emit tags for measure and dimension",
)
return None

if schema_field_tags:
return GlobalTagsClass(tags=schema_field_tags)

return None

@staticmethod
def get_tag_mces() -> Iterable[MetadataChangeEvent]:
Expand All @@ -602,7 +612,7 @@ def view_field_to_schema_field(
else f"{field.field_type.value}. {field.description}"
),
globalTags=(
LookerUtil._get_tags_from_field_type(field.field_type, reporter)
LookerUtil._get_tags_from_field_type(field, reporter)
if tag_measures_and_dimensions is True
else None
),
Expand Down
1 change: 1 addition & 0 deletions metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1090,6 +1090,7 @@ def _get_fields(
is_primary_key=is_primary_key,
field_type=type_cls,
upstream_fields=upstream_fields,
tags=field_dict.get("tags") or [],
)
fields.append(field)
return fields
Expand Down
12 changes: 0 additions & 12 deletions metadata-ingestion/tests/integration/looker/golden_looker_mces.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -140,9 +137,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
}
Expand Down Expand Up @@ -178,9 +172,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -222,9 +213,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
}
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_allow_ingest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -137,9 +134,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_external_project_view_mces.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -373,9 +370,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_file_path_ingest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -373,9 +370,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_independent_look_ingest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,9 +324,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -386,9 +383,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_ingest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -373,9 +370,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_ingest_joins.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -373,9 +370,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/golden_test_ingest_unaliased_joins.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -137,9 +134,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down
22 changes: 8 additions & 14 deletions metadata-ingestion/tests/integration/looker/looker_mces_golden_deleted_stateful.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,9 +324,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -386,9 +383,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
},
Expand Down Expand Up @@ -813,8 +807,8 @@
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7",
"entityType": "chart",
"entityUrn": "urn:li:chart:(looker,dashboard_elements.10)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand All @@ -830,8 +824,8 @@
}
},
{
"entityType": "chart",
"entityUrn": "urn:li:chart:(looker,dashboard_elements.10)",
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(looker,dashboards.11)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand All @@ -847,8 +841,8 @@
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)",
"entityType": "container",
"entityUrn": "urn:li:container:621eb6e00da9abece0f64522f81be0e7",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand All @@ -864,8 +858,8 @@
}
},
{
"entityType": "dashboard",
"entityUrn": "urn:li:dashboard:(looker,dashboards.11)",
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:looker,bogus data.explore.my_view,PROD)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
Expand Down
6 changes: 0 additions & 6 deletions metadata-ingestion/tests/integration/looker/looker_mces_usage_history.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
}
Expand Down Expand Up @@ -113,9 +110,6 @@
},
"nativeDataType": "string",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
}
Expand Down
21 changes: 20 additions & 1 deletion metadata-ingestion/tests/integration/lookml/duplicate_field_ingestion_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@
"aspect": {
"json": {
"materialized": false,
"viewLogic": "# File was added to check duplicate field issue\n\nview: dataset_lineages {\n sql_table_name: \"PUBLIC\".\"DATASET_LINEAGES\"\n ;;\n\n dimension: createdon {\n type: date\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension_group: createdon {\n type: time\n timeframes: [\n raw,\n time,\n date,\n week,\n month,\n quarter,\n year\n ]\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension: entity {\n type: string\n sql: ${TABLE}.\"ENTITY\" ;;\n }\n\n dimension: metadata {\n type: string\n sql: ${TABLE}.\"METADATA\" ;;\n }\n\n dimension: urn {\n type: string\n sql: ${TABLE}.\"URN\" ;;\n }\n\n dimension: version {\n type: number\n sql: ${TABLE}.\"VERSION\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n",
"viewLogic": "# File was added to check duplicate field issue\n\nview: dataset_lineages {\n sql_table_name: \"PUBLIC\".\"DATASET_LINEAGES\"\n ;;\n\n dimension: createdon {\n type: date\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension_group: createdon {\n type: time\n timeframes: [\n raw,\n time,\n date,\n week,\n month,\n quarter,\n year\n ]\n sql: ${TABLE}.\"CREATEDON\" ;;\n }\n\n dimension: entity {\n type: string\n sql: ${TABLE}.\"ENTITY\" ;;\n }\n\n dimension: metadata {\n type: string\n sql: ${TABLE}.\"METADATA\" ;;\n }\n\n dimension: urn {\n type: string\n sql: ${TABLE}.\"URN\" ;;\n }\n\n dimension: version {\n type: number\n tags: [\"softVersion\"]\n sql: ${TABLE}.\"VERSION\" ;;\n }\n\n measure: count {\n type: count\n drill_fields: []\n }\n}\n",
"viewLanguage": "lookml"
}
},
Expand Down Expand Up @@ -338,6 +338,9 @@
"recursive": false,
"globalTags": {
"tags": [
{
"tag": "urn:li:tag:softVersion"
},
{
"tag": "urn:li:tag:Dimension"
}
Expand Down Expand Up @@ -484,5 +487,21 @@
"runId": "lookml-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "tag",
"entityUrn": "urn:li:tag:softVersion",
"changeType": "UPSERT",
"aspectName": "tagKey",
"aspect": {
"json": {
"name": "softVersion"
}
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "lookml-test",
"lastRunId": "no-run-id-provided"
}
}
]
9 changes: 0 additions & 9 deletions metadata-ingestion/tests/integration/lookml/expected_output.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1388,9 +1388,6 @@
},
"nativeDataType": "unknown",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
},
{
Expand All @@ -1405,9 +1402,6 @@
},
"nativeDataType": "unknown",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
},
{
Expand All @@ -1422,9 +1416,6 @@
},
"nativeDataType": "unknown",
"recursive": false,
"globalTags": {
"tags": []
},
"isPartOfKey": false
}
],
Expand Down
Loading

0 comments on commit 3e47496

Please sign in to comment.