From 7c1d3b09eddfba779784d603d6639f00b9a69db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20G=C3=B3mez=20Villamor?= Date: Fri, 13 Dec 2024 10:25:05 +0100 Subject: [PATCH 1/2] fix(datahub-client): avoid parallel execution of publish and publish-java8 (#12120) --- .github/workflows/publish-datahub-jars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml index 5aec66bc33bb6..393f9d993e2a2 100644 --- a/.github/workflows/publish-datahub-jars.yml +++ b/.github/workflows/publish-datahub-jars.yml @@ -201,7 +201,7 @@ jobs: permissions: id-token: write contents: read - needs: ["check-secret", "setup"] + needs: ["check-secret", "setup", "publish"] if: ${{ needs.check-secret.outputs.publish-enabled == 'true' }} steps: - uses: acryldata/sane-checkout-action@v3 From 06edf23a33d9580e78ef8f81d3fbe620162c1916 Mon Sep 17 00:00:00 2001 From: Jonny Dixon <45681293+acrylJonny@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:25:31 +0000 Subject: [PATCH 2/2] fix(ingestion/dremio): Ignore filtered containers in schema allowdeny pattern (#11959) Co-authored-by: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> --- .../ingestion/source/dremio/dremio_api.py | 279 +- .../dremio/dremio_datahub_source_mapping.py | 2 + .../source/dremio/dremio_reporting.py | 15 + .../dremio/dremio_mces_golden.json | 2414 +++-- .../dremio_platform_instance_mces_golden.json | 8002 +++++++++++++++++ .../dremio_platform_instance_to_file.yml | 26 + .../dremio_schema_filter_mces_golden.json | 2327 +++++ .../dremio/dremio_schema_filter_to_file.yml | 28 + .../tests/integration/dremio/test_dremio.py | 272 +- .../unit/dremio/test_dremio_schema_filter.py | 123 + 10 files changed, 12457 insertions(+), 1031 deletions(-) create mode 100644 metadata-ingestion/tests/integration/dremio/dremio_platform_instance_mces_golden.json create mode 100644 metadata-ingestion/tests/integration/dremio/dremio_platform_instance_to_file.yml create mode 100644 metadata-ingestion/tests/integration/dremio/dremio_schema_filter_mces_golden.json create mode 100644 metadata-ingestion/tests/integration/dremio/dremio_schema_filter_to_file.yml create mode 100644 metadata-ingestion/tests/unit/dremio/test_dremio_schema_filter.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py index 7f4e0f520b7a5..d913b7e42065d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py @@ -1,6 +1,7 @@ import concurrent.futures import json import logging +import re import warnings from collections import defaultdict from enum import Enum @@ -609,32 +610,6 @@ def extract_all_queries(self) -> List[Dict[str, Any]]: return self.execute_query(query=jobs_query) - def get_source_by_id(self, source_id: str) -> Optional[Dict]: - """ - Fetch source details by ID. - """ - response = self.get( - url=f"/source/{source_id}", - ) - return response if response else None - - def get_source_for_dataset(self, schema: str, dataset: str) -> Optional[Dict]: - """ - Get source information for a dataset given its schema and name. - """ - dataset_id = self.get_dataset_id(schema, dataset) - if not dataset_id: - return None - - catalog_entry = self.get( - url=f"/catalog/{dataset_id}", - ) - if not catalog_entry or "path" not in catalog_entry: - return None - - source_id = catalog_entry["path"][0] - return self.get_source_by_id(source_id) - def get_tags_for_resource(self, resource_id: str) -> Optional[List[str]]: """ Get Dremio tags for a given resource_id. @@ -673,55 +648,119 @@ def get_description_for_resource(self, resource_id: str) -> Optional[str]: ) return None - def get_containers_for_location( - self, resource_id: str, path: List[str] - ) -> List[Dict[str, str]]: - containers = [] + def _check_pattern_match( + self, + pattern: str, + paths: List[str], + allow_prefix: bool = True, + ) -> bool: + """ + Helper method to check if a pattern matches any of the paths. + Handles hierarchical matching where each level is matched independently. + Also handles prefix matching for partial paths. + """ + if pattern == ".*": + return True - def traverse_path(location_id: str, entity_path: List[str]) -> List: - nonlocal containers - try: - response = self.get(url=f"/catalog/{location_id}") - if ( - response.get("entityType") - == DremioEntityContainerType.FOLDER.value.lower() - ): - containers.append( - { - "id": location_id, - "name": entity_path[-1], - "path": entity_path[:-1], - "container_type": DremioEntityContainerType.FOLDER, - } - ) + # Convert the pattern to regex with proper anchoring + regex_pattern = pattern + if pattern.startswith("^"): + # Already has start anchor + regex_pattern = pattern.replace(".", r"\.") # Escape dots + regex_pattern = regex_pattern.replace( + r"\.*", ".*" + ) # Convert .* to wildcard + else: + # Add start anchor and handle dots + regex_pattern = "^" + pattern.replace(".", r"\.").replace(r"\.*", ".*") + + # Handle end matching + if not pattern.endswith(".*"): + if pattern.endswith("$"): + # Keep explicit end anchor + pass + elif not allow_prefix: + # Add end anchor for exact matching + regex_pattern = regex_pattern + "$" + + for path in paths: + if re.match(regex_pattern, path, re.IGNORECASE): + return True - for container in response.get("children", []): - if ( - container.get("type") - == DremioEntityContainerType.CONTAINER.value - ): - traverse_path(container.get("id"), container.get("path")) + return False - except Exception as exc: - logging.info( - "Location {} contains no tables or views. Skipping...".format(id) - ) - self.report.warning( - message="Failed to get tables or views", - context=f"{id}", - exc=exc, - ) + def should_include_container(self, path: List[str], name: str) -> bool: + """ + Helper method to check if a container should be included based on schema patterns. + Used by both get_all_containers and get_containers_for_location. + """ + path_components = path + [name] if path else [name] + full_path = ".".join(path_components) - return containers + # Default allow everything case + if self.allow_schema_pattern == [".*"] and not self.deny_schema_pattern: + self.report.report_container_scanned(full_path) + return True - return traverse_path(location_id=resource_id, entity_path=path) + # Check deny patterns first + if self.deny_schema_pattern: + for pattern in self.deny_schema_pattern: + if self._check_pattern_match( + pattern=pattern, + paths=[full_path], + allow_prefix=False, + ): + self.report.report_container_filtered(full_path) + return False + + # Check allow patterns + for pattern in self.allow_schema_pattern: + # For patterns with wildcards, check if this path is a parent of the pattern + if "*" in pattern: + pattern_parts = pattern.split(".") + path_parts = path_components + + # If pattern has exact same number of parts, check each component + if len(pattern_parts) == len(path_parts): + matches = True + for p_part, c_part in zip(pattern_parts, path_parts): + if p_part != "*" and p_part.lower() != c_part.lower(): + matches = False + break + if matches: + self.report.report_container_scanned(full_path) + return True + # Otherwise check if current path is prefix match + else: + # Remove the trailing wildcard if present + if pattern_parts[-1] == "*": + pattern_parts = pattern_parts[:-1] + + for i in range(len(path_parts)): + current_path = ".".join(path_parts[: i + 1]) + pattern_prefix = ".".join(pattern_parts[: i + 1]) + + if pattern_prefix.startswith(current_path): + self.report.report_container_scanned(full_path) + return True + + # Direct pattern matching + if self._check_pattern_match( + pattern=pattern, + paths=[full_path], + allow_prefix=True, + ): + self.report.report_container_scanned(full_path) + return True + + self.report.report_container_filtered(full_path) + return False def get_all_containers(self): """ - Query the Dremio sources API and return source information. + Query the Dremio sources API and return filtered source information. """ containers = [] - response = self.get(url="/catalog") def process_source(source): @@ -731,34 +770,41 @@ def process_source(source): ) source_config = source_resp.get("config", {}) - if source_config.get("database"): - db = source_config.get("database") - else: - db = source_config.get("databaseName", "") - - return { - "id": source.get("id"), - "name": source.get("path")[0], - "path": [], - "container_type": DremioEntityContainerType.SOURCE, - "source_type": source_resp.get("type"), - "root_path": source_config.get("rootPath"), - "database_name": db, - } + db = source_config.get( + "database", source_config.get("databaseName", "") + ) + + if self.should_include_container([], source.get("path")[0]): + return { + "id": source.get("id"), + "name": source.get("path")[0], + "path": [], + "container_type": DremioEntityContainerType.SOURCE, + "source_type": source_resp.get("type"), + "root_path": source_config.get("rootPath"), + "database_name": db, + } else: - return { - "id": source.get("id"), - "name": source.get("path")[0], - "path": [], - "container_type": DremioEntityContainerType.SPACE, - } + if self.should_include_container([], source.get("path")[0]): + return { + "id": source.get("id"), + "name": source.get("path")[0], + "path": [], + "container_type": DremioEntityContainerType.SPACE, + } + return None def process_source_and_containers(source): container = process_source(source) + if not container: + return [] + + # Get sub-containers sub_containers = self.get_containers_for_location( resource_id=container.get("id"), path=[container.get("name")], ) + return [container] + sub_containers # Use ThreadPoolExecutor to parallelize the processing of sources @@ -771,7 +817,16 @@ def process_source_and_containers(source): } for future in concurrent.futures.as_completed(future_to_source): - containers.extend(future.result()) + source = future_to_source[future] + try: + containers.extend(future.result()) + except Exception as exc: + logger.error(f"Error processing source: {exc}") + self.report.warning( + message="Failed to process source", + context=f"{source}", + exc=exc, + ) return containers @@ -785,3 +840,55 @@ def get_context_for_vds(self, resource_id: str) -> str: ) else: return "" + + def get_containers_for_location( + self, resource_id: str, path: List[str] + ) -> List[Dict[str, str]]: + containers = [] + + def traverse_path(location_id: str, entity_path: List[str]) -> List: + nonlocal containers + try: + response = self.get(url=f"/catalog/{location_id}") + + # Check if current folder should be included + if ( + response.get("entityType") + == DremioEntityContainerType.FOLDER.value.lower() + ): + folder_name = entity_path[-1] + folder_path = entity_path[:-1] + + if self.should_include_container(folder_path, folder_name): + containers.append( + { + "id": location_id, + "name": folder_name, + "path": folder_path, + "container_type": DremioEntityContainerType.FOLDER, + } + ) + + # Recursively process child containers + for container in response.get("children", []): + if ( + container.get("type") + == DremioEntityContainerType.CONTAINER.value + ): + traverse_path(container.get("id"), container.get("path")) + + except Exception as exc: + logging.info( + "Location {} contains no tables or views. Skipping...".format( + location_id + ) + ) + self.report.warning( + message="Failed to get tables or views", + context=f"{location_id}", + exc=exc, + ) + + return containers + + return traverse_path(location_id=resource_id, entity_path=path) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py index 928c145e5eb50..e5d6b8e40fb3d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py @@ -31,6 +31,7 @@ class DremioToDataHubSourceTypeMapping: "SNOWFLAKE": "snowflake", "SYNAPSE": "mssql", "TERADATA": "teradata", + "VERTICA": "vertica", } DATABASE_SOURCE_TYPES = { @@ -52,6 +53,7 @@ class DremioToDataHubSourceTypeMapping: "SNOWFLAKE", "SYNAPSE", "TERADATA", + "VERTICA", } FILE_OBJECT_STORAGE_TYPES = { diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py index 926dbd42eb267..c8eb035461ca1 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py @@ -14,12 +14,27 @@ class DremioSourceReport( ): num_containers_failed: int = 0 num_datasets_failed: int = 0 + containers_scanned: int = 0 + containers_filtered: int = 0 def report_upstream_latency(self, start_time: datetime, end_time: datetime) -> None: # recording total combined latency is not very useful, keeping this method as a placeholder # for future implementation of min / max / percentiles etc. pass + def report_container_scanned(self, name: str) -> None: + """ + Record that a container was successfully scanned + """ + self.containers_scanned += 1 + + def report_container_filtered(self, container_name: str) -> None: + """ + Record that a container was filtered out + """ + self.containers_filtered += 1 + self.report_dropped(container_name) + def report_entity_scanned(self, name: str, ent_type: str = "View") -> None: """ Entity could be a view or a table diff --git a/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json b/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json index 3a8fce62f4bb3..41d47a574a211 100644 --- a/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json +++ b/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json @@ -15,7 +15,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -31,7 +31,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -49,7 +49,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -65,7 +65,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -85,7 +85,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -105,7 +105,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -121,7 +121,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -139,7 +139,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -155,7 +155,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -175,7 +175,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -195,7 +195,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -211,7 +211,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -229,7 +229,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -245,7 +245,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -265,7 +265,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -285,7 +285,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -301,7 +301,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -319,7 +319,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -335,7 +335,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -355,7 +355,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -375,7 +375,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -391,7 +391,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -409,7 +409,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -425,7 +425,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -445,7 +445,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -465,7 +465,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -481,7 +481,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -497,7 +497,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -515,7 +515,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -531,7 +531,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -555,117 +555,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6d76c46c4893c111f0006794f4a16482", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": {}, - "name": "warehouse", - "qualifiedName": "s3.warehouse", - "description": "", - "env": "PROD" - } - }, - "systemMetadata": { - "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6d76c46c4893c111f0006794f4a16482", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:63a316133b08a091e919dc8c7a828a4d" - } - }, - "systemMetadata": { - "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6d76c46c4893c111f0006794f4a16482", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:dremio" - } - }, - "systemMetadata": { - "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6d76c46c4893c111f0006794f4a16482", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Dremio Folder" - ] - } - }, - "systemMetadata": { - "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6d76c46c4893c111f0006794f4a16482", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:6d76c46c4893c111f0006794f4a16482", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "Sources" - }, - { - "id": "urn:li:container:63a316133b08a091e919dc8c7a828a4d", - "urn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -685,7 +575,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -701,7 +591,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -717,7 +607,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -735,7 +625,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -751,7 +641,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -775,7 +665,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -795,7 +685,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -811,7 +701,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -827,7 +717,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -845,7 +735,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -861,7 +751,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -885,7 +775,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -905,7 +795,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -921,7 +811,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -937,7 +827,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -955,7 +845,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -971,7 +861,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -995,7 +885,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1015,7 +905,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1031,7 +921,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1047,7 +937,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1065,7 +955,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1081,7 +971,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1105,7 +995,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1125,7 +1015,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1141,7 +1031,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1157,7 +1047,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1175,7 +1065,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1191,7 +1081,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1215,7 +1105,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1235,7 +1125,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1251,7 +1141,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1267,7 +1157,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1285,7 +1175,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1301,7 +1191,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1325,7 +1215,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1345,7 +1235,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1361,7 +1251,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1377,7 +1267,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1395,7 +1285,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1411,7 +1301,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1435,7 +1325,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1455,7 +1345,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1471,7 +1361,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1487,7 +1377,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1505,7 +1395,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1521,7 +1411,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1549,7 +1439,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1569,7 +1459,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1585,7 +1475,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1601,7 +1491,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1619,7 +1509,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1635,7 +1525,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1663,7 +1553,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1683,7 +1573,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1699,7 +1589,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1715,7 +1605,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1733,7 +1623,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1749,7 +1639,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1781,7 +1671,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1801,7 +1691,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1817,7 +1707,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1833,7 +1723,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1851,7 +1741,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1867,7 +1757,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1903,7 +1793,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1927,7 +1817,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1945,7 +1835,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1961,7 +1851,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1977,7 +1867,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -1995,7 +1885,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2025,7 +1915,7 @@ }, "fields": [ { - "fieldPath": "F", + "fieldPath": "A", "nullable": true, "type": { "type": { @@ -2037,7 +1927,7 @@ "isPartOfKey": false }, { - "fieldPath": "G", + "fieldPath": "B", "nullable": true, "type": { "type": { @@ -2049,7 +1939,7 @@ "isPartOfKey": false }, { - "fieldPath": "H", + "fieldPath": "C", "nullable": true, "type": { "type": { @@ -2061,7 +1951,7 @@ "isPartOfKey": false }, { - "fieldPath": "I", + "fieldPath": "D", "nullable": true, "type": { "type": { @@ -2073,7 +1963,7 @@ "isPartOfKey": false }, { - "fieldPath": "A", + "fieldPath": "E", "nullable": true, "type": { "type": { @@ -2085,7 +1975,7 @@ "isPartOfKey": false }, { - "fieldPath": "B", + "fieldPath": "F", "nullable": true, "type": { "type": { @@ -2097,7 +1987,7 @@ "isPartOfKey": false }, { - "fieldPath": "C", + "fieldPath": "G", "nullable": true, "type": { "type": { @@ -2109,7 +1999,7 @@ "isPartOfKey": false }, { - "fieldPath": "D", + "fieldPath": "H", "nullable": true, "type": { "type": { @@ -2121,7 +2011,7 @@ "isPartOfKey": false }, { - "fieldPath": "E", + "fieldPath": "I", "nullable": true, "type": { "type": { @@ -2137,7 +2027,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2153,7 +2043,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2177,7 +2067,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2201,7 +2091,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2219,7 +2109,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2235,7 +2125,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2251,7 +2141,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2263,13 +2153,13 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT * FROM \"mysql\".northwind.customers", + "viewLogic": "SELECT * FROM mysql.northwind.customers", "viewLanguage": "SQL" } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2299,31 +2189,31 @@ }, "fields": [ { - "fieldPath": "email_address", + "fieldPath": "priority", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "float(24)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "priority", + "fieldPath": "email_address", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "float(24)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "company", + "fieldPath": "first_name", "nullable": true, "type": { "type": { @@ -2335,19 +2225,19 @@ "isPartOfKey": false }, { - "fieldPath": "id", + "fieldPath": "last_name", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "integer(32)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "first_name", + "fieldPath": "company", "nullable": true, "type": { "type": { @@ -2359,14 +2249,14 @@ "isPartOfKey": false }, { - "fieldPath": "last_name", + "fieldPath": "id", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "integer(32)", "recursive": false, "isPartOfKey": false } @@ -2375,7 +2265,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2391,7 +2281,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2419,7 +2309,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2443,7 +2333,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2461,7 +2351,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2477,7 +2367,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2493,7 +2383,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2505,13 +2395,13 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT * FROM \"mysql\".metagalaxy.\"metadata_aspect\"", + "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_aspect", "viewLanguage": "SQL" } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2541,7 +2431,7 @@ }, "fields": [ { - "fieldPath": "createdby", + "fieldPath": "urn", "nullable": true, "type": { "type": { @@ -2553,7 +2443,7 @@ "isPartOfKey": false }, { - "fieldPath": "createdfor", + "fieldPath": "aspect", "nullable": true, "type": { "type": { @@ -2565,19 +2455,19 @@ "isPartOfKey": false }, { - "fieldPath": "urn", + "fieldPath": "version", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "aspect", + "fieldPath": "metadata", "nullable": true, "type": { "type": { @@ -2589,19 +2479,19 @@ "isPartOfKey": false }, { - "fieldPath": "version", + "fieldPath": "createdon", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.DateType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "timestamp(23)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "metadata", + "fieldPath": "createdby", "nullable": true, "type": { "type": { @@ -2613,14 +2503,14 @@ "isPartOfKey": false }, { - "fieldPath": "createdon", + "fieldPath": "createdfor", "nullable": true, "type": { "type": { - "com.linkedin.schema.DateType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "timestamp(23)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false } @@ -2629,7 +2519,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2645,7 +2535,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2673,7 +2563,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2697,7 +2587,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2715,7 +2605,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2731,7 +2621,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2747,7 +2637,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2759,13 +2649,13 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT * FROM \"mysql\".metagalaxy.\"metadata_index\"", + "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_index", "viewLanguage": "SQL" } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2795,19 +2685,19 @@ }, "fields": [ { - "fieldPath": "id", + "fieldPath": "doubleVal", "nullable": true, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "double(53)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "urn", + "fieldPath": "stringVal", "nullable": true, "type": { "type": { @@ -2831,7 +2721,7 @@ "isPartOfKey": false }, { - "fieldPath": "stringVal", + "fieldPath": "path", "nullable": true, "type": { "type": { @@ -2843,19 +2733,19 @@ "isPartOfKey": false }, { - "fieldPath": "doubleVal", + "fieldPath": "aspect", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "double(53)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "path", + "fieldPath": "urn", "nullable": true, "type": { "type": { @@ -2867,14 +2757,14 @@ "isPartOfKey": false }, { - "fieldPath": "aspect", + "fieldPath": "id", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false } @@ -2883,7 +2773,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2899,7 +2789,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2927,7 +2817,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2951,7 +2841,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2969,7 +2859,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -2985,7 +2875,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3001,7 +2891,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3013,13 +2903,13 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT * FROM \"mysql\".metagalaxy.\"metadata_index_view\"", + "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_index_view", "viewLanguage": "SQL" } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3061,14 +2951,14 @@ "isPartOfKey": false }, { - "fieldPath": "urn", + "fieldPath": "doubleVal", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "double(53)", "recursive": false, "isPartOfKey": false }, @@ -3085,14 +2975,14 @@ "isPartOfKey": false }, { - "fieldPath": "doubleVal", + "fieldPath": "urn", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "double(53)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false } @@ -3101,7 +2991,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3117,7 +3007,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3145,7 +3035,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3169,7 +3059,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3187,7 +3077,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3203,7 +3093,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3219,7 +3109,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3231,13 +3121,13 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT * FROM \"mysql\".northwind.orders", + "viewLogic": "SELECT * FROM mysql.northwind.orders", "viewLanguage": "SQL" } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3279,26 +3169,26 @@ "isPartOfKey": false }, { - "fieldPath": "description", + "fieldPath": "id", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "integer(32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "id", + "fieldPath": "description", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "integer(32)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false } @@ -3307,7 +3197,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3323,7 +3213,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3351,7 +3241,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3375,7 +3265,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3393,7 +3283,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3409,7 +3299,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3425,7 +3315,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3437,13 +3327,13 @@ "aspect": { "json": { "materialized": false, - "viewLogic": "SELECT * FROM s3.warehouse.\"sample.parquet\"", + "viewLogic": "SELECT * FROM s3.warehouse", "viewLanguage": "SQL" } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3473,7 +3363,7 @@ }, "fields": [ { - "fieldPath": "salary", + "fieldPath": "age", "nullable": true, "type": { "type": { @@ -3485,31 +3375,31 @@ "isPartOfKey": false }, { - "fieldPath": "age", + "fieldPath": "name", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "name", + "fieldPath": "id", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "id", + "fieldPath": "salary", "nullable": true, "type": { "type": { @@ -3525,7 +3415,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3541,7 +3431,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3569,21 +3459,21 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)", "changeType": "UPSERT", "aspectName": "datasetProperties", "aspect": { "json": { "customProperties": {}, - "externalUrl": "http://localhost:9047/source/\"s3\"/\"warehouse\".\"sample.parquet\"", - "name": "sample.parquet", - "qualifiedName": "s3.warehouse.sample.parquet", + "externalUrl": "http://localhost:9047/source/\"s3\"/\"warehouse\"", + "name": "warehouse", + "qualifiedName": "s3.warehouse", "description": "", "created": { "time": 0 @@ -3593,13 +3483,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -3611,13 +3501,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -3627,34 +3517,34 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { "json": { - "container": "urn:li:container:6d76c46c4893c111f0006794f4a16482" + "container": "urn:li:container:63a316133b08a091e919dc8c7a828a4d" } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)", "changeType": "UPSERT", "aspectName": "schemaMetadata", "aspect": { "json": { - "schemaName": "s3.warehouse.sample.parquet", + "schemaName": "s3.warehouse", "platform": "urn:li:dataPlatform:dremio", "version": 0, "created": { @@ -3725,13 +3615,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -3741,13 +3631,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -3758,7 +3648,7 @@ "time": 0, "actor": "urn:li:corpuser:unknown" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD)", "type": "COPY" } ] @@ -3766,13 +3656,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { @@ -3784,17 +3674,13 @@ { "id": "urn:li:container:63a316133b08a091e919dc8c7a828a4d", "urn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d" - }, - { - "id": "urn:li:container:6d76c46c4893c111f0006794f4a16482", - "urn": "urn:li:container:6d76c46c4893c111f0006794f4a16482" } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3818,7 +3704,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3836,7 +3722,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3852,7 +3738,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3868,7 +3754,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -3910,19 +3796,19 @@ "isPartOfKey": false }, { - "fieldPath": "createdfor", + "fieldPath": "version", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "createdby", + "fieldPath": "metadata", "nullable": true, "type": { "type": { @@ -3946,7 +3832,7 @@ "isPartOfKey": false }, { - "fieldPath": "metadata", + "fieldPath": "createdby", "nullable": true, "type": { "type": { @@ -3958,14 +3844,14 @@ "isPartOfKey": false }, { - "fieldPath": "version", + "fieldPath": "createdfor", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, @@ -3986,7 +3872,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4002,7 +3888,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4027,7 +3913,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4055,7 +3941,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4079,7 +3965,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4097,7 +3983,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4113,7 +3999,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4129,7 +4015,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4171,38 +4057,38 @@ "isPartOfKey": false }, { - "fieldPath": "id", + "fieldPath": "stringVal", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "urn", + "fieldPath": "longVal", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "aspect", + "fieldPath": "id", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, @@ -4219,19 +4105,19 @@ "isPartOfKey": false }, { - "fieldPath": "longVal", + "fieldPath": "aspect", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "stringVal", + "fieldPath": "urn", "nullable": true, "type": { "type": { @@ -4247,7 +4133,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4263,7 +4149,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4288,7 +4174,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4316,7 +4202,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4340,7 +4226,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4358,7 +4244,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4374,7 +4260,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4390,7 +4276,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4420,31 +4306,31 @@ }, "fields": [ { - "fieldPath": "path", + "fieldPath": "id", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "id", + "fieldPath": "urn", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "urn", + "fieldPath": "path", "nullable": true, "type": { "type": { @@ -4472,7 +4358,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4488,7 +4374,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4513,7 +4399,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4541,7 +4427,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4565,7 +4451,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4583,7 +4469,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4599,7 +4485,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4615,7 +4501,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4645,19 +4531,19 @@ }, "fields": [ { - "fieldPath": "id", + "fieldPath": "priority", "nullable": true, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "integer(32)", + "nativeDataType": "float(24)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "company", + "fieldPath": "email_address", "nullable": true, "type": { "type": { @@ -4669,7 +4555,7 @@ "isPartOfKey": false }, { - "fieldPath": "last_name", + "fieldPath": "first_name", "nullable": true, "type": { "type": { @@ -4681,7 +4567,7 @@ "isPartOfKey": false }, { - "fieldPath": "first_name", + "fieldPath": "last_name", "nullable": true, "type": { "type": { @@ -4693,7 +4579,7 @@ "isPartOfKey": false }, { - "fieldPath": "email_address", + "fieldPath": "company", "nullable": true, "type": { "type": { @@ -4705,14 +4591,14 @@ "isPartOfKey": false }, { - "fieldPath": "priority", + "fieldPath": "id", "nullable": true, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "float(24)", + "nativeDataType": "integer(32)", "recursive": false, "isPartOfKey": false } @@ -4721,7 +4607,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4737,7 +4623,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4762,7 +4648,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4790,7 +4676,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4814,7 +4700,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4832,7 +4718,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4848,7 +4734,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4864,7 +4750,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4894,7 +4780,7 @@ }, "fields": [ { - "fieldPath": "customer_id", + "fieldPath": "id", "nullable": true, "type": { "type": { @@ -4906,26 +4792,26 @@ "isPartOfKey": false }, { - "fieldPath": "id", + "fieldPath": "description", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "integer(32)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "description", + "fieldPath": "customer_id", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "integer(32)", "recursive": false, "isPartOfKey": false } @@ -4934,7 +4820,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4950,7 +4836,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -4975,7 +4861,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5003,7 +4889,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5027,7 +4913,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5045,7 +4931,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5061,7 +4947,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5077,7 +4963,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5107,7 +4993,7 @@ }, "fields": [ { - "fieldPath": "F", + "fieldPath": "D", "nullable": true, "type": { "type": { @@ -5119,7 +5005,7 @@ "isPartOfKey": false }, { - "fieldPath": "G", + "fieldPath": "C", "nullable": true, "type": { "type": { @@ -5131,7 +5017,7 @@ "isPartOfKey": false }, { - "fieldPath": "H", + "fieldPath": "B", "nullable": true, "type": { "type": { @@ -5143,7 +5029,7 @@ "isPartOfKey": false }, { - "fieldPath": "I", + "fieldPath": "A", "nullable": true, "type": { "type": { @@ -5155,7 +5041,7 @@ "isPartOfKey": false }, { - "fieldPath": "A", + "fieldPath": "H", "nullable": true, "type": { "type": { @@ -5167,7 +5053,7 @@ "isPartOfKey": false }, { - "fieldPath": "B", + "fieldPath": "G", "nullable": true, "type": { "type": { @@ -5179,7 +5065,7 @@ "isPartOfKey": false }, { - "fieldPath": "C", + "fieldPath": "F", "nullable": true, "type": { "type": { @@ -5191,7 +5077,7 @@ "isPartOfKey": false }, { - "fieldPath": "D", + "fieldPath": "E", "nullable": true, "type": { "type": { @@ -5203,7 +5089,7 @@ "isPartOfKey": false }, { - "fieldPath": "E", + "fieldPath": "I", "nullable": true, "type": { "type": { @@ -5219,7 +5105,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5235,7 +5121,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5260,7 +5146,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5288,7 +5174,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5312,7 +5198,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5330,7 +5216,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5346,7 +5232,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5362,7 +5248,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5552,7 +5438,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5568,7 +5454,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5593,7 +5479,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5625,7 +5511,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5649,7 +5535,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5667,7 +5553,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5683,7 +5569,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5699,7 +5585,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5781,7 +5667,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5797,7 +5683,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5822,7 +5708,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5854,7 +5740,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5878,7 +5764,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5896,7 +5782,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5912,7 +5798,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5928,7 +5814,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -5958,7 +5844,7 @@ }, "fields": [ { - "fieldPath": "cp_start_date_sk", + "fieldPath": "cp_catalog_page_number", "nullable": true, "type": { "type": { @@ -5970,19 +5856,19 @@ "isPartOfKey": false }, { - "fieldPath": "cp_catalog_page_sk", + "fieldPath": "cp_type", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cp_catalog_page_id", + "fieldPath": "cp_description", "nullable": true, "type": { "type": { @@ -5994,7 +5880,7 @@ "isPartOfKey": false }, { - "fieldPath": "cp_end_date_sk", + "fieldPath": "cp_catalog_number", "nullable": true, "type": { "type": { @@ -6006,19 +5892,19 @@ "isPartOfKey": false }, { - "fieldPath": "cp_department", + "fieldPath": "cp_end_date_sk", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cp_catalog_number", + "fieldPath": "cp_start_date_sk", "nullable": true, "type": { "type": { @@ -6030,31 +5916,31 @@ "isPartOfKey": false }, { - "fieldPath": "cp_catalog_page_number", + "fieldPath": "cp_catalog_page_id", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cp_description", + "fieldPath": "cp_catalog_page_sk", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "cp_type", + "fieldPath": "cp_department", "nullable": true, "type": { "type": { @@ -6070,7 +5956,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -6086,7 +5972,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -6111,7 +5997,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -6151,7 +6037,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -6191,22 +6077,22 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdfor)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),version)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdfor)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),version)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdby)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),metadata)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdby)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),metadata)" ], "confidenceScore": 1.0 }, @@ -6224,22 +6110,22 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),metadata)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdby)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),metadata)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdby)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),version)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdfor)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),version)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdfor)" ], "confidenceScore": 1.0 }, @@ -6259,7 +6145,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -6299,33 +6185,33 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),stringVal)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),stringVal)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),urn)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),longVal)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),urn)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),longVal)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),aspect)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),id)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),aspect)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),id)" ], "confidenceScore": 1.0 }, @@ -6343,22 +6229,22 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),longVal)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),aspect)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),longVal)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),aspect)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),stringVal)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),urn)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),stringVal)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),urn)" ], "confidenceScore": 1.0 } @@ -6367,7 +6253,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -6396,33 +6282,33 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),path)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),id)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),path)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),id)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),urn)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),urn)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),urn)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),path)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),urn)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),path)" ], "confidenceScore": 1.0 }, @@ -6442,7 +6328,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -6471,66 +6357,66 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),priority)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),priority)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),company)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),email_address)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),company)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),email_address)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),last_name)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),first_name)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),last_name)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),first_name)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),first_name)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),last_name)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),first_name)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),last_name)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),email_address)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),company)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),email_address)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),company)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),priority)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),id)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),priority)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),id)" ], "confidenceScore": 1.0 } @@ -6539,7 +6425,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -6568,33 +6454,33 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),customer_id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),id)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),customer_id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),id)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),description)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),description)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),description)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),customer_id)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),description)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),customer_id)" ], "confidenceScore": 1.0 } @@ -6603,13 +6489,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)", "changeType": "UPSERT", "aspectName": "upstreamLineage", "aspect": { @@ -6624,7 +6510,7 @@ "time": 0, "actor": "urn:li:corpuser:_ingestion" }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD)", "type": "COPY" } ], @@ -6632,44 +6518,44 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),id)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),id)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),name)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),name)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),name)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),name)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),age)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),age)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),age)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),age)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),salary)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),salary)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),salary)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),salary)" ], "confidenceScore": 1.0 } @@ -6678,7 +6564,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -6852,7 +6738,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -6927,7 +6813,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -6956,99 +6842,99 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),F)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),D)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),F)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),D)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),G)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),C)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),G)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),C)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),H)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),B)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),H)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),B)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),I)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),A)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),I)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),A)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),A)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),H)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),A)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),H)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),B)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),G)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),B)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),G)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),C)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),F)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),C)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),F)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),D)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),E)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),D)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),E)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),E)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),I)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),E)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),I)" ], "confidenceScore": 1.0 } @@ -7057,7 +6943,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -7086,183 +6972,770 @@ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_start_date_sk)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_number)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_start_date_sk)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_number)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_sk)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_type)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_sk)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_type)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_description)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_id)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_description)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_end_date_sk)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_number)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_end_date_sk)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_number)" ], "confidenceScore": 1.0 }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_department)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_end_date_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_end_date_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_start_date_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_start_date_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_department)" ], "downstreamType": "FIELD", "downstreams": [ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_department)" ], "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.northwind.customers,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n *\nFROM mysql.northwind.customers", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.northwind.customers,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_aspect,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n *\nFROM mysql.metagalaxy.metadata_aspect", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_aspect,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_index,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n *\nFROM mysql.metagalaxy.metadata_index", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_index,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_index_view,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n *\nFROM mysql.metagalaxy.metadata_index_view", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_index_view,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.northwind.orders,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n *\nFROM mysql.northwind.orders", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.northwind.orders,PROD)" }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29" + } + ], + "fineGrainedLineages": [ { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_number)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),id)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_number)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),id)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29" }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_number)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),name)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_number)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),name)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29" }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_description)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),age)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_description)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),age)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29" }, { "upstreamType": "FIELD_SET", "upstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_type)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),salary)" ], "downstreamType": "FIELD", "downstreams": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_type)" + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),salary)" ], - "confidenceScore": 1.0 + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29" } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29", "changeType": "UPSERT", - "aspectName": "datasetProfile", + "aspectName": "queryProperties", "aspect": { "json": { - "timestampMillis": 1697353200000, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" + "statement": { + "value": "SELECT\n *\nFROM s3.warehouse", + "language": "SQL" }, - "rowCount": 3834, - "columnCount": 9, - "fieldProfiles": [ + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ { - "fieldPath": "F", - "uniqueCount": 61, - "nullCount": 0 + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)" }, { - "fieldPath": "G", - "uniqueCount": 40, - "nullCount": 0 + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),age)" }, { - "fieldPath": "H", - "uniqueCount": 91, - "nullCount": 0 + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),id)" }, { - "fieldPath": "I", - "uniqueCount": 85, - "nullCount": 0 + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),name)" }, { - "fieldPath": "A", - "uniqueCount": 2, - "nullCount": 0 + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),salary)" }, { - "fieldPath": "B", - "uniqueCount": 2, - "nullCount": 0 + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)" }, { - "fieldPath": "C", - "uniqueCount": 3834, - "nullCount": 0 + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),id)" }, { - "fieldPath": "D", - "uniqueCount": 76, - "nullCount": 0 + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),name)" }, { - "fieldPath": "E", - "uniqueCount": 192, - "nullCount": 0 + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),age)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),salary)" } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7275,23 +7748,62 @@ "rowCount": 0, "columnCount": 4, "fieldProfiles": [ + { + "fieldPath": "id", + "uniqueCount": 0, + "nullCount": 0 + }, + { + "fieldPath": "doubleVal", + "uniqueCount": 0, + "nullCount": 0 + }, { "fieldPath": "path", "uniqueCount": 0, "nullCount": 0 }, + { + "fieldPath": "urn", + "uniqueCount": 0, + "nullCount": 0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1697353200000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 0, + "columnCount": 3, + "fieldProfiles": [ { "fieldPath": "id", "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "urn", + "fieldPath": "description", "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "doubleVal", + "fieldPath": "customer_id", "uniqueCount": 0, "nullCount": 0 } @@ -7300,13 +7812,63 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProfile", + "aspect": { + "json": { + "timestampMillis": 1697353200000, + "partitionSpec": { + "partition": "FULL_TABLE_SNAPSHOT", + "type": "FULL_TABLE" + }, + "rowCount": 4, + "columnCount": 4, + "fieldProfiles": [ + { + "fieldPath": "age", + "uniqueCount": 4, + "nullCount": 0, + "mean": "32.5", + "stdev": "6.454972243679028" + }, + { + "fieldPath": "name", + "uniqueCount": 4, + "nullCount": 0 + }, + { + "fieldPath": "id", + "uniqueCount": 4, + "nullCount": 0, + "mean": "2.5", + "stdev": "1.2909944487358056" + }, + { + "fieldPath": "salary", + "uniqueCount": 4, + "nullCount": 0, + "mean": "65000.0", + "stdev": "12909.944487358056" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7316,56 +7878,56 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 0, + "rowCount": 2, "columnCount": 7, "fieldProfiles": [ { - "fieldPath": "doubleVal", - "uniqueCount": 0, + "fieldPath": "urn", + "uniqueCount": 1, "nullCount": 0 }, { - "fieldPath": "id", - "uniqueCount": 0, + "fieldPath": "aspect", + "uniqueCount": 2, "nullCount": 0 }, { - "fieldPath": "urn", - "uniqueCount": 0, + "fieldPath": "version", + "uniqueCount": 1, "nullCount": 0 }, { - "fieldPath": "aspect", - "uniqueCount": 0, + "fieldPath": "metadata", + "uniqueCount": 2, "nullCount": 0 }, { - "fieldPath": "path", - "uniqueCount": 0, + "fieldPath": "createdon", + "uniqueCount": 1, "nullCount": 0 }, { - "fieldPath": "longVal", - "uniqueCount": 0, + "fieldPath": "createdby", + "uniqueCount": 1, "nullCount": 0 }, { - "fieldPath": "stringVal", + "fieldPath": "createdfor", "uniqueCount": 0, - "nullCount": 0 + "nullCount": 2 } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7375,86 +7937,55 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 0, - "columnCount": 3, + "rowCount": 5, + "columnCount": 6, "fieldProfiles": [ { - "fieldPath": "customer_id", - "uniqueCount": 0, - "nullCount": 0 + "fieldPath": "priority", + "uniqueCount": 3, + "nullCount": 1, + "mean": "4.175000011920929", + "stdev": "0.4924429489953036" }, { - "fieldPath": "id", - "uniqueCount": 0, + "fieldPath": "email_address", + "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "description", - "uniqueCount": 0, + "fieldPath": "first_name", + "uniqueCount": 5, "nullCount": 0 - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)", - "changeType": "UPSERT", - "aspectName": "datasetProfile", - "aspect": { - "json": { - "timestampMillis": 1697353200000, - "partitionSpec": { - "partition": "FULL_TABLE_SNAPSHOT", - "type": "FULL_TABLE" - }, - "rowCount": 4, - "columnCount": 4, - "fieldProfiles": [ - { - "fieldPath": "salary", - "uniqueCount": 4, - "nullCount": 0, - "mean": "65000.0", - "stdev": "12909.944487358056" }, { - "fieldPath": "age", - "uniqueCount": 4, - "nullCount": 0, - "mean": "32.5", - "stdev": "6.454972243679028" + "fieldPath": "last_name", + "uniqueCount": 5, + "nullCount": 0 }, { - "fieldPath": "name", - "uniqueCount": 4, + "fieldPath": "company", + "uniqueCount": 5, "nullCount": 0 }, { "fieldPath": "id", - "uniqueCount": 4, + "uniqueCount": 5, "nullCount": 0, - "mean": "2.5", - "stdev": "1.2909944487358056" + "mean": "3.0", + "stdev": "1.5811388300841898" } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7492,13 +8023,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7508,41 +8039,22 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 5, - "columnCount": 6, + "rowCount": 0, + "columnCount": 3, "fieldProfiles": [ { - "fieldPath": "email_address", - "uniqueCount": 5, - "nullCount": 0 - }, - { - "fieldPath": "priority", - "uniqueCount": 3, - "nullCount": 1, - "mean": "4.175000011920929", - "stdev": "0.4924429489953036" - }, - { - "fieldPath": "company", - "uniqueCount": 5, + "fieldPath": "customer_id", + "uniqueCount": 0, "nullCount": 0 }, { "fieldPath": "id", - "uniqueCount": 5, - "nullCount": 0, - "mean": "3.0", - "stdev": "1.5811388300841898" - }, - { - "fieldPath": "first_name", - "uniqueCount": 5, + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "last_name", - "uniqueCount": 5, + "fieldPath": "description", + "uniqueCount": 0, "nullCount": 0 } ] @@ -7550,13 +8062,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7566,52 +8078,42 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 3834, - "columnCount": 9, + "rowCount": 0, + "columnCount": 7, "fieldProfiles": [ { - "fieldPath": "F", - "uniqueCount": 61, - "nullCount": 0 - }, - { - "fieldPath": "G", - "uniqueCount": 40, - "nullCount": 0 - }, - { - "fieldPath": "H", - "uniqueCount": 91, + "fieldPath": "doubleVal", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "I", - "uniqueCount": 85, + "fieldPath": "stringVal", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "A", - "uniqueCount": 2, + "fieldPath": "longVal", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "B", - "uniqueCount": 2, + "fieldPath": "id", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "C", - "uniqueCount": 3834, + "fieldPath": "path", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "D", - "uniqueCount": 76, + "fieldPath": "aspect", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "E", - "uniqueCount": 192, + "fieldPath": "urn", + "uniqueCount": 0, "nullCount": 0 } ] @@ -7619,13 +8121,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7635,42 +8137,42 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 2, + "rowCount": 0, "columnCount": 7, "fieldProfiles": [ { - "fieldPath": "createdby", - "uniqueCount": 1, + "fieldPath": "doubleVal", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "createdfor", + "fieldPath": "stringVal", "uniqueCount": 0, - "nullCount": 2 + "nullCount": 0 }, { - "fieldPath": "urn", - "uniqueCount": 1, + "fieldPath": "longVal", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "aspect", - "uniqueCount": 2, + "fieldPath": "path", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "version", - "uniqueCount": 1, + "fieldPath": "aspect", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "metadata", - "uniqueCount": 2, + "fieldPath": "urn", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "createdon", - "uniqueCount": 1, + "fieldPath": "id", + "uniqueCount": 0, "nullCount": 0 } ] @@ -7678,13 +8180,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7694,42 +8196,52 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 2, - "columnCount": 7, + "rowCount": 3834, + "columnCount": 9, "fieldProfiles": [ { - "fieldPath": "urn", - "uniqueCount": 1, + "fieldPath": "D", + "uniqueCount": 76, "nullCount": 0 }, { - "fieldPath": "createdfor", - "uniqueCount": 0, - "nullCount": 2 + "fieldPath": "C", + "uniqueCount": 3834, + "nullCount": 0 }, { - "fieldPath": "createdby", - "uniqueCount": 1, + "fieldPath": "B", + "uniqueCount": 2, "nullCount": 0 }, { - "fieldPath": "createdon", - "uniqueCount": 1, + "fieldPath": "A", + "uniqueCount": 2, "nullCount": 0 }, { - "fieldPath": "metadata", - "uniqueCount": 2, + "fieldPath": "H", + "uniqueCount": 91, "nullCount": 0 }, { - "fieldPath": "version", - "uniqueCount": 1, + "fieldPath": "G", + "uniqueCount": 40, + "nullCount": 0 + }, + { + "fieldPath": "F", + "uniqueCount": 61, + "nullCount": 0 + }, + { + "fieldPath": "E", + "uniqueCount": 192, "nullCount": 0 }, { - "fieldPath": "aspect", - "uniqueCount": 2, + "fieldPath": "I", + "uniqueCount": 85, "nullCount": 0 } ] @@ -7737,13 +8249,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7787,13 +8299,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7803,22 +8315,42 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 0, - "columnCount": 3, + "rowCount": 2, + "columnCount": 7, "fieldProfiles": [ { - "fieldPath": "customer_id", - "uniqueCount": 0, + "fieldPath": "urn", + "uniqueCount": 1, "nullCount": 0 }, { - "fieldPath": "description", - "uniqueCount": 0, + "fieldPath": "version", + "uniqueCount": 1, "nullCount": 0 }, { - "fieldPath": "id", + "fieldPath": "metadata", + "uniqueCount": 2, + "nullCount": 0 + }, + { + "fieldPath": "createdon", + "uniqueCount": 1, + "nullCount": 0 + }, + { + "fieldPath": "createdby", + "uniqueCount": 1, + "nullCount": 0 + }, + { + "fieldPath": "createdfor", "uniqueCount": 0, + "nullCount": 2 + }, + { + "fieldPath": "aspect", + "uniqueCount": 2, "nullCount": 0 } ] @@ -7826,13 +8358,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7842,42 +8374,52 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 0, - "columnCount": 7, + "rowCount": 3834, + "columnCount": 9, "fieldProfiles": [ { - "fieldPath": "id", - "uniqueCount": 0, + "fieldPath": "A", + "uniqueCount": 2, "nullCount": 0 }, { - "fieldPath": "urn", - "uniqueCount": 0, + "fieldPath": "B", + "uniqueCount": 2, "nullCount": 0 }, { - "fieldPath": "longVal", - "uniqueCount": 0, + "fieldPath": "C", + "uniqueCount": 3834, "nullCount": 0 }, { - "fieldPath": "stringVal", - "uniqueCount": 0, + "fieldPath": "D", + "uniqueCount": 76, "nullCount": 0 }, { - "fieldPath": "doubleVal", - "uniqueCount": 0, + "fieldPath": "E", + "uniqueCount": 192, "nullCount": 0 }, { - "fieldPath": "path", - "uniqueCount": 0, + "fieldPath": "F", + "uniqueCount": 61, "nullCount": 0 }, { - "fieldPath": "aspect", - "uniqueCount": 0, + "fieldPath": "G", + "uniqueCount": 40, + "nullCount": 0 + }, + { + "fieldPath": "H", + "uniqueCount": 91, + "nullCount": 0 + }, + { + "fieldPath": "I", + "uniqueCount": 85, "nullCount": 0 } ] @@ -7885,7 +8427,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -7905,45 +8447,45 @@ "columnCount": 6, "fieldProfiles": [ { - "fieldPath": "id", - "uniqueCount": 5, - "nullCount": 0, - "mean": "3.0", - "stdev": "1.5811388300841898" + "fieldPath": "priority", + "uniqueCount": 3, + "nullCount": 1, + "mean": "4.175000011920929", + "stdev": "0.4924429489953036" }, { - "fieldPath": "company", + "fieldPath": "email_address", "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "last_name", + "fieldPath": "first_name", "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "first_name", + "fieldPath": "last_name", "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "email_address", + "fieldPath": "company", "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "priority", - "uniqueCount": 3, - "nullCount": 1, - "mean": "4.175000011920929", - "stdev": "0.4924429489953036" + "fieldPath": "id", + "uniqueCount": 5, + "nullCount": 0, + "mean": "3.0", + "stdev": "1.5811388300841898" } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -8032,7 +8574,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -8082,7 +8624,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } }, @@ -8102,34 +8644,20 @@ "columnCount": 9, "fieldProfiles": [ { - "fieldPath": "cp_start_date_sk", - "uniqueCount": 91, - "nullCount": 286, - "mean": "2451880.7982769064", - "stdev": "634.1320559175408" - }, - { - "fieldPath": "cp_catalog_page_sk", - "uniqueCount": 30000, - "nullCount": 0, - "mean": "15000.5", - "stdev": "8660.398374208891" + "fieldPath": "cp_catalog_page_number", + "uniqueCount": 277, + "nullCount": 294, + "mean": "138.8378442065576", + "stdev": "80.01625232480262" }, { - "fieldPath": "cp_catalog_page_id", - "uniqueCount": 30000, + "fieldPath": "cp_type", + "uniqueCount": 4, "nullCount": 0 }, { - "fieldPath": "cp_end_date_sk", - "uniqueCount": 97, - "nullCount": 302, - "mean": "2451940.632601522", - "stdev": "634.7367986145963" - }, - { - "fieldPath": "cp_department", - "uniqueCount": 2, + "fieldPath": "cp_description", + "uniqueCount": 29718, "nullCount": 0 }, { @@ -8140,20 +8668,34 @@ "stdev": "31.27039684588463" }, { - "fieldPath": "cp_catalog_page_number", - "uniqueCount": 277, - "nullCount": 294, - "mean": "138.8378442065576", - "stdev": "80.01625232480262" + "fieldPath": "cp_end_date_sk", + "uniqueCount": 97, + "nullCount": 302, + "mean": "2451940.632601522", + "stdev": "634.7367986145963" }, { - "fieldPath": "cp_description", - "uniqueCount": 29718, + "fieldPath": "cp_start_date_sk", + "uniqueCount": 91, + "nullCount": 286, + "mean": "2451880.7982769064", + "stdev": "634.1320559175408" + }, + { + "fieldPath": "cp_catalog_page_id", + "uniqueCount": 30000, "nullCount": 0 }, { - "fieldPath": "cp_type", - "uniqueCount": 4, + "fieldPath": "cp_catalog_page_sk", + "uniqueCount": 30000, + "nullCount": 0, + "mean": "15000.5", + "stdev": "8660.398374208891" + }, + { + "fieldPath": "cp_department", + "uniqueCount": 2, "nullCount": 0 } ] @@ -8161,7 +8703,103 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-bo12f3", + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-5p79wf", "lastRunId": "no-run-id-provided" } } diff --git a/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_mces_golden.json new file mode 100644 index 0000000000000..81e9c38d1bbee --- /dev/null +++ b/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_mces_golden.json @@ -0,0 +1,8002 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "s3", + "qualifiedName": "s3", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "mysql", + "qualifiedName": "mysql", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "Samples", + "qualifiedName": "Samples", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "@admin", + "qualifiedName": "@admin", + "description": "# Wikis & Labels\n\n![Gnarly Catalog](https://d33wubrfki0l68.cloudfront.net/c1a54376c45a9276c080f3d10ed25ce61c17bcd2/2b946/img/home/open-source-for-everyone.svg)\n\nYou are reading the wiki for your home space! You can create and edit this information for any source, space, or folder.\n\nThis sidebar always shows the wiki for the current source, space or folder you are browsing.\n\nWhen browsing or previewing datasets, click on the `Open details panel` button to create a wiki or add labels to that dataset.\n\n**Tip:** You can hide the wiki by clicking on the sidebar icon on upper right hand side.", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Space" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Spaces" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "space", + "qualifiedName": "space", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Space" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Spaces" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "test_folder", + "qualifiedName": "space.test_folder", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Spaces" + }, + { + "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7", + "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "dataCharmer", + "qualifiedName": "mysql.dataCharmer", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "metagalaxy", + "qualifiedName": "mysql.metagalaxy", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "mysql", + "qualifiedName": "mysql.mysql", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "northwind", + "qualifiedName": "mysql.northwind", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "performance_schema", + "qualifiedName": "mysql.performance_schema", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "test_cases", + "qualifiedName": "mysql.test_cases", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "samples.dremio.com", + "qualifiedName": "Samples.samples.dremio.com", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:007d12a4a241c87924b54e1e35990234" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:007d12a4a241c87924b54e1e35990234", + "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "Dremio University", + "qualifiedName": "Samples.samples.dremio.com.Dremio University", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:007d12a4a241c87924b54e1e35990234", + "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234" + }, + { + "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715", + "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "tpcds_sf1000", + "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:007d12a4a241c87924b54e1e35990234", + "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234" + }, + { + "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715", + "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "catalog_page", + "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:007d12a4a241c87924b54e1e35990234", + "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234" + }, + { + "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715", + "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715" + }, + { + "id": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554", + "urn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "1ab266d5-18eb-4780-711d-0fa337fa6c00", + "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:007d12a4a241c87924b54e1e35990234", + "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234" + }, + { + "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715", + "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715" + }, + { + "id": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554", + "urn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554" + }, + { + "id": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c", + "urn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/space/\"space\"/\"warehouse\"", + "name": "warehouse", + "qualifiedName": "space.warehouse", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "SELECT * from Samples.\"samples.dremio.com\".\"NYC-weather.csv\"", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "space.warehouse", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "I", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "A", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "B", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "C", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "D", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "E", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "F", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "G", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "H", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Spaces" + }, + { + "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7", + "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"customers\"", + "name": "customers", + "qualifiedName": "space.test_folder.customers", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "SELECT * FROM mysql.northwind.customers", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "space.test_folder.customers", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "first_name", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer(32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "priority", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "float(24)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email_address", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Spaces" + }, + { + "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7", + "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7" + }, + { + "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5", + "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"metadata_aspect\"", + "name": "metadata_aspect", + "qualifiedName": "space.test_folder.metadata_aspect", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_aspect", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "space.test_folder.metadata_aspect", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "urn", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdby", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdon", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.DateType": {} + } + }, + "nativeDataType": "timestamp(23)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "metadata", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "version", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdfor", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "aspect", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Spaces" + }, + { + "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7", + "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7" + }, + { + "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5", + "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"metadata_index\"", + "name": "metadata_index", + "qualifiedName": "space.test_folder.metadata_index", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_index", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "space.test_folder.metadata_index", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urn", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "aspect", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "path", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "longVal", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stringVal", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "doubleVal", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "double(53)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Spaces" + }, + { + "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7", + "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7" + }, + { + "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5", + "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"metadata_index_view\"", + "name": "metadata_index_view", + "qualifiedName": "space.test_folder.metadata_index_view", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_index_view", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "space.test_folder.metadata_index_view", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "doubleVal", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "double(53)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "path", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urn", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Spaces" + }, + { + "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7", + "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7" + }, + { + "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5", + "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"orders\"", + "name": "orders", + "qualifiedName": "space.test_folder.orders", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "SELECT * FROM mysql.northwind.orders", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "space.test_folder.orders", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer(32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "description", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer(32)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Spaces" + }, + { + "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7", + "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7" + }, + { + "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5", + "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"raw\"", + "name": "raw", + "qualifiedName": "space.test_folder.raw", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "viewProperties", + "aspect": { + "json": { + "materialized": false, + "viewLogic": "SELECT * FROM s3.warehouse", + "viewLanguage": "SQL" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "space.test_folder.raw", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "salary", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "age", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Spaces" + }, + { + "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7", + "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7" + }, + { + "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5", + "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/source/\"s3\"/\"warehouse\"", + "name": "warehouse", + "qualifiedName": "s3.warehouse", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "s3.warehouse", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "age", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "salary", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46", + "urn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/source/\"mysql\"/\"metagalaxy\".\"metadata_aspect\"", + "name": "metadata_aspect", + "qualifiedName": "mysql.metagalaxy.metadata_aspect", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:67653e01aa4da4fcfd6675a591700e89" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "mysql.metagalaxy.metadata_aspect", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "metadata", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdon", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.DateType": {} + } + }, + "nativeDataType": "timestamp(23)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdby", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdfor", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urn", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "aspect", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "version", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + }, + { + "id": "urn:li:container:67653e01aa4da4fcfd6675a591700e89", + "urn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/source/\"mysql\"/\"metagalaxy\".\"metadata_index\"", + "name": "metadata_index", + "qualifiedName": "mysql.metagalaxy.metadata_index", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:67653e01aa4da4fcfd6675a591700e89" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "mysql.metagalaxy.metadata_index", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "path", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "aspect", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urn", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "stringVal", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "longVal", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "doubleVal", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "double(53)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + }, + { + "id": "urn:li:container:67653e01aa4da4fcfd6675a591700e89", + "urn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/source/\"mysql\"/\"metagalaxy\".\"metadata_index_view\"", + "name": "metadata_index_view", + "qualifiedName": "mysql.metagalaxy.metadata_index_view", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:67653e01aa4da4fcfd6675a591700e89" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "mysql.metagalaxy.metadata_index_view", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "urn", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "path", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "doubleVal", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "double(53)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + }, + { + "id": "urn:li:container:67653e01aa4da4fcfd6675a591700e89", + "urn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/source/\"mysql\"/\"northwind\".\"customers\"", + "name": "customers", + "qualifiedName": "mysql.northwind.customers", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "mysql.northwind.customers", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "priority", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "float(24)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email_address", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "company", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer(32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + }, + { + "id": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f", + "urn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/source/\"mysql\"/\"northwind\".\"orders\"", + "name": "orders", + "qualifiedName": "mysql.northwind.orders", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "mysql.northwind.orders", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "description", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer(32)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "integer(32)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2", + "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2" + }, + { + "id": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f", + "urn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"NYC-weather.csv\"", + "name": "NYC-weather.csv", + "qualifiedName": "Samples.samples.dremio.com.NYC-weather.csv", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "Samples.samples.dremio.com.NYC-weather.csv", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "E", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "G", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "H", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "I", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "F", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "A", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "B", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "C", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "D", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:007d12a4a241c87924b54e1e35990234", + "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234" + }, + { + "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715", + "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"googleplaystore.csv\"", + "name": "googleplaystore.csv", + "qualifiedName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:55c3b773b40bb744b4e5946db4e13455" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "F", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "G", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "H", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "I", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "J", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "K", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "L", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "M", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "A", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "B", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "C", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "D", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "E", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:007d12a4a241c87924b54e1e35990234", + "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234" + }, + { + "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715", + "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715" + }, + { + "id": "urn:li:container:55c3b773b40bb744b4e5946db4e13455", + "urn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"oracle-departments.xlsx\"", + "name": "oracle-departments.xlsx", + "qualifiedName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:55c3b773b40bb744b4e5946db4e13455" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "DEPARTMENT_ID", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "double(53)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "DEPARTMENT_NAME", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "MANAGER_ID", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "double(53)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "LOCATION_ID", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "double(53)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:007d12a4a241c87924b54e1e35990234", + "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234" + }, + { + "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715", + "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715" + }, + { + "id": "urn:li:container:55c3b773b40bb744b4e5946db4e13455", + "urn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"tpcds_sf1000\".\"catalog_page\".\"1ab266d5-18eb-4780-711d-0fa337fa6c00\".\"0_0_0.parquet\"", + "name": "0_0_0.parquet", + "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "cp_catalog_page_sk", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_catalog_page_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_start_date_sk", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_end_date_sk", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_department", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_catalog_number", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_catalog_page_number", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_description", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_type", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + }, + { + "id": "Sources" + }, + { + "id": "urn:li:container:007d12a4a241c87924b54e1e35990234", + "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234" + }, + { + "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715", + "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715" + }, + { + "id": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554", + "urn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554" + }, + { + "id": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c", + "urn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c" + }, + { + "id": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8", + "urn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),metadata)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),metadata)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),createdon)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),createdon)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),createdby)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),createdby)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),createdfor)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),createdfor)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),urn)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),urn)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),aspect)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),aspect)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),version)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),version)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),path)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),path)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),aspect)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),aspect)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),urn)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),urn)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),stringVal)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),stringVal)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),longVal)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),longVal)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),doubleVal)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),doubleVal)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD),urn)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD),urn)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD),path)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD),path)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD),doubleVal)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD),doubleVal)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),priority)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),priority)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),email_address)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),email_address)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),company)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),company)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),last_name)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD),description)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD),description)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD),id)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),age)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),age)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),salary)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),salary)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),F)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),F)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),G)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),G)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),H)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),H)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),I)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),I)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),J)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),J)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),K)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),K)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),L)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),L)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),M)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),M)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),A)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),A)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),B)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),B)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),C)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),C)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),D)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),D)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),E)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),E)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_NAME)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_NAME)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),MANAGER_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),MANAGER_ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),LOCATION_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),LOCATION_ID)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),E)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),E)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),G)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),G)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),H)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),H)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),I)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),I)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),F)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),F)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),A)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),A)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),B)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),B)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),C)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),C)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),D)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),D)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_start_date_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_start_date_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_end_date_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_end_date_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_department)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_department)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_number)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_number)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_description)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_description)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_type)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_type)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.northwind.customers,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n *\nFROM mysql.northwind.customers", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.northwind.customers,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_aspect,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n *\nFROM mysql.metagalaxy.metadata_aspect", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_aspect,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_index,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n *\nFROM mysql.metagalaxy.metadata_index", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_index,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_index_view,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n *\nFROM mysql.metagalaxy.metadata_index_view", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_index_view,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.northwind.orders,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n *\nFROM mysql.northwind.orders", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.northwind.orders,PROD)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)", + "type": "VIEW", + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),id)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),name)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),age)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),age)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29" + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),salary)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),salary)" + ], + "confidenceScore": 0.9, + "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "queryProperties", + "aspect": { + "json": { + "statement": { + "value": "SELECT\n *\nFROM s3.warehouse", + "language": "SQL" + }, + "source": "SYSTEM", + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "lastModified": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + } + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "querySubjects", + "aspect": { + "json": { + "subjects": [ + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),age)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),name)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),salary)" + }, + { + "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),id)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),name)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),age)" + }, + { + "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),salary)" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-t5gf20", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_to_file.yml b/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_to_file.yml new file mode 100644 index 0000000000000..923dae139681a --- /dev/null +++ b/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_to_file.yml @@ -0,0 +1,26 @@ +source: + type: dremio + config: + # Coordinates + hostname: localhost + port: 9047 + tls: false + + # Credentials + authentication_method: password + username: admin + password: "2310Admin1234!@" + + platform_instance: test-platform + + include_query_lineage: false + + source_mappings: + - platform: s3 + source_name: samples + platform_instance: s3_test_samples + +sink: + type: file + config: + filename: "./dremio_mces.json" diff --git a/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_mces_golden.json b/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_mces_golden.json new file mode 100644 index 0000000000000..f4dfed45ccf0c --- /dev/null +++ b/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_mces_golden.json @@ -0,0 +1,2327 @@ +[ +{ + "entityType": "container", + "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "Samples", + "qualifiedName": "Samples", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Sources" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "samples.dremio.com", + "qualifiedName": "Samples.samples.dremio.com", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Sources" + }, + { + "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "Dremio University", + "qualifiedName": "Samples.samples.dremio.com.Dremio University", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Sources" + }, + { + "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" + }, + { + "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83", + "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "tpcds_sf1000", + "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Sources" + }, + { + "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" + }, + { + "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83", + "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "catalog_page", + "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Sources" + }, + { + "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" + }, + { + "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83", + "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83" + }, + { + "id": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f", + "urn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": {}, + "name": "1ab266d5-18eb-4780-711d-0fa337fa6c00", + "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00", + "description": "", + "env": "PROD" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Dremio Folder" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Sources" + }, + { + "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" + }, + { + "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83", + "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83" + }, + { + "id": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f", + "urn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f" + }, + { + "id": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400", + "urn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"NYC-weather.csv\"", + "name": "NYC-weather.csv", + "qualifiedName": "Samples.samples.dremio.com.NYC-weather.csv", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "Samples.samples.dremio.com.NYC-weather.csv", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "I", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "H", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "G", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "F", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "E", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "D", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "C", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "B", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "A", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Sources" + }, + { + "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" + }, + { + "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83", + "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"googleplaystore.csv\"", + "name": "googleplaystore.csv", + "qualifiedName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:687c0496e464bc4c0de935cb1da1becf" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "E", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "A", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "B", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "C", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "D", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "F", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "G", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "H", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "I", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "J", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "K", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "L", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "M", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Sources" + }, + { + "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" + }, + { + "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83", + "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83" + }, + { + "id": "urn:li:container:687c0496e464bc4c0de935cb1da1becf", + "urn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"oracle-departments.xlsx\"", + "name": "oracle-departments.xlsx", + "qualifiedName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:687c0496e464bc4c0de935cb1da1becf" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "LOCATION_ID", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "double(53)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "MANAGER_ID", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "double(53)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "DEPARTMENT_NAME", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "DEPARTMENT_ID", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "double(53)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Sources" + }, + { + "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" + }, + { + "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83", + "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83" + }, + { + "id": "urn:li:container:687c0496e464bc4c0de935cb1da1becf", + "urn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": {}, + "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"tpcds_sf1000\".\"catalog_page\".\"1ab266d5-18eb-4780-711d-0fa337fa6c00\".\"0_0_0.parquet\"", + "name": "0_0_0.parquet", + "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet", + "description": "", + "created": { + "time": 0 + }, + "tags": [] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Table" + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dremio" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b" + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "schemaMetadata", + "aspect": { + "json": { + "schemaName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet", + "platform": "urn:li:dataPlatform:dremio", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "cp_start_date_sk", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_catalog_page_id", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_catalog_page_sk", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_department", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_catalog_number", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_end_date_sk", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_catalog_page_number", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "bigint(64)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_description", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "cp_type", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "character varying(65536)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)", + "type": "COPY" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "Sources" + }, + { + "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68", + "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68" + }, + { + "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83", + "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83" + }, + { + "id": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f", + "urn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f" + }, + { + "id": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400", + "urn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400" + }, + { + "id": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b", + "urn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),E)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),E)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),A)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),A)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),B)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),B)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),C)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),C)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),D)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),D)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),F)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),F)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),G)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),G)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),H)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),H)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),I)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),I)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),J)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),J)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),K)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),K)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),L)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),L)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),M)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),M)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),LOCATION_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),LOCATION_ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),MANAGER_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),MANAGER_ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_NAME)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_NAME)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_ID)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),I)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),I)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),H)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),H)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),G)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),G)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),F)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),F)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),E)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),E)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),D)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),D)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),C)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),C)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),B)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),B)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),A)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),A)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 1697353200000, + "actor": "urn:li:corpuser:_ingestion" + }, + "created": { + "time": 0, + "actor": "urn:li:corpuser:_ingestion" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_start_date_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_start_date_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_department)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_department)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_number)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_end_date_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_end_date_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_number)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_description)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_description)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_type)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_type)" + ], + "confidenceScore": 1.0 + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1697353200000, + "runId": "dremio-2023_10_15-07_00_00-vftab1", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_to_file.yml b/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_to_file.yml new file mode 100644 index 0000000000000..2b3cea465eebe --- /dev/null +++ b/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_to_file.yml @@ -0,0 +1,28 @@ +source: + type: dremio + config: + # Coordinates + hostname: localhost + port: 9047 + tls: false + + # Credentials + authentication_method: password + username: admin + password: "2310Admin1234!@" + + include_query_lineage: false + + source_mappings: + - platform: s3 + source_name: samples + platform_instance: s3_test_samples + + schema_pattern: + allow: + - "Samples" + +sink: + type: file + config: + filename: "./dremio_mces.json" diff --git a/metadata-ingestion/tests/integration/dremio/test_dremio.py b/metadata-ingestion/tests/integration/dremio/test_dremio.py index cc3a7e19bc93e..401f487d8a14b 100644 --- a/metadata-ingestion/tests/integration/dremio/test_dremio.py +++ b/metadata-ingestion/tests/integration/dremio/test_dremio.py @@ -1,6 +1,7 @@ import json import os import subprocess +from typing import Dict import boto3 import pytest @@ -75,9 +76,10 @@ def create_spaces_and_folders(headers): def create_sample_source(headers): - url = f"{DREMIO_HOST}/apiv2/source/Samples" + url = f"{DREMIO_HOST}/api/v3/catalog" payload = { + "entityType": "source", "config": { "externalBucketList": ["samples.dremio.com"], "credentialType": "NONE", @@ -95,14 +97,15 @@ def create_sample_source(headers): "type": "S3", } - response = requests.put(url, headers=headers, data=json.dumps(payload)) + response = requests.post(url, headers=headers, data=json.dumps(payload)) assert response.status_code == 200, f"Failed to add dataset: {response.text}" def create_s3_source(headers): - url = f"{DREMIO_HOST}/apiv2/source/s3" + url = f"{DREMIO_HOST}/api/v3/catalog" payload = { + "entityType": "source", "name": "s3", "config": { "credentialType": "ACCESS_KEY", @@ -139,24 +142,25 @@ def create_s3_source(headers): "metadataPolicy": { "deleteUnavailableDatasets": True, "autoPromoteDatasets": False, - "namesRefreshMillis": 3600000, - "datasetDefinitionRefreshAfterMillis": 3600000, - "datasetDefinitionExpireAfterMillis": 10800000, - "authTTLMillis": 86400000, - "updateMode": "PREFETCH_QUERIED", + "namesRefreshMs": 3600000, + "datasetRefreshAfterMs": 3600000, + "datasetExpireAfterMs": 10800000, + "authTTLMs": 86400000, + "datasetUpdateMode": "PREFETCH_QUERIED", }, "type": "S3", "accessControlList": {"userControls": [], "roleControls": []}, } - response = requests.put(url, headers=headers, data=json.dumps(payload)) + response = requests.post(url, headers=headers, data=json.dumps(payload)) assert response.status_code == 200, f"Failed to add s3 datasource: {response.text}" def create_mysql_source(headers): - url = f"{DREMIO_HOST}/apiv2/source/mysql" + url = f"{DREMIO_HOST}/api/v3/catalog" payload = { + "entityType": "source", "config": { "username": "root", "password": "rootpwd123", @@ -169,7 +173,7 @@ def create_mysql_source(headers): "maxIdleConns": 8, "idleTimeSec": 60, }, - "name": "mysql-source", + "name": "mysql", "accelerationRefreshPeriod": 3600000, "accelerationGracePeriod": 10800000, "accelerationActivePolicyType": "PERIOD", @@ -177,72 +181,121 @@ def create_mysql_source(headers): "accelerationRefreshOnDataChanges": False, "metadataPolicy": { "deleteUnavailableDatasets": True, - "namesRefreshMillis": 3600000, - "datasetDefinitionRefreshAfterMillis": 3600000, - "datasetDefinitionExpireAfterMillis": 10800000, - "authTTLMillis": 86400000, - "updateMode": "PREFETCH_QUERIED", + "namesRefreshMs": 3600000, + "datasetRefreshAfterMs": 3600000, + "datasetExpireAfterMs": 10800000, + "authTTLMs": 86400000, + "datasetUpdateMode": "PREFETCH_QUERIED", }, "type": "MYSQL", } - response = requests.put(url, headers=headers, data=json.dumps(payload)) + response = requests.post(url, headers=headers, data=json.dumps(payload)) assert ( response.status_code == 200 ), f"Failed to add mysql datasource: {response.text}" def upload_dataset(headers): - url = f"{DREMIO_HOST}/apiv2/source/s3/file_format/warehouse/sample.parquet" - payload = {"ignoreOtherFileFormats": False, "type": "Parquet"} + url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2Fs3%2Fwarehouse" + payload = { + "entityType": "dataset", + "type": "PHYSICAL_DATASET", + "path": [ + "s3", + "warehouse", + ], + "format": {"type": "Parquet"}, + } - response = requests.put(url, headers=headers, data=json.dumps(payload)) + response = requests.post(url, headers=headers, data=json.dumps(payload)) assert response.status_code == 200, f"Failed to add dataset: {response.text}" - url = f"{DREMIO_HOST}/apiv2/source/Samples/file_format/samples.dremio.com/NYC-weather.csv" + url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2FSamples%2Fsamples.dremio.com%2FNYC-weather.csv" payload = { - "fieldDelimiter": ",", - "quote": '"', - "comment": "#", - "lineDelimiter": "\r\n", - "escape": '"', - "extractHeader": False, - "trimHeader": True, - "skipFirstLine": False, - "type": "Text", + "entityType": "dataset", + "type": "PHYSICAL_DATASET", + "path": [ + "Samples", + "samples.dremio.com", + "NYC-weather.csv", + ], + "format": { + "fieldDelimiter": ",", + "quote": '"', + "comment": "#", + "lineDelimiter": "\r\n", + "escape": '"', + "extractHeader": False, + "trimHeader": True, + "skipFirstLine": False, + "type": "Text", + }, } - response = requests.put(url, headers=headers, data=json.dumps(payload)) + response = requests.post(url, headers=headers, data=json.dumps(payload)) assert response.status_code == 200, f"Failed to add dataset: {response.text}" - url = f"{DREMIO_HOST}/apiv2/source/Samples/file_format/samples.dremio.com/Dremio%20University/oracle-departments.xlsx" + url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2FSamples%2Fsamples.dremio.com%2FDremio%20University%2Foracle-departments.xlsx" - payload = {"extractHeader": True, "hasMergedCells": False, "type": "Excel"} + payload = { + "entityType": "dataset", + "type": "PHYSICAL_DATASET", + "path": [ + "Samples", + "samples.dremio.com", + "Dremio University", + "oracle-departments.xlsx", + ], + "format": {"extractHeader": True, "hasMergedCells": False, "type": "Excel"}, + } - response = requests.put(url, headers=headers, data=json.dumps(payload)) + response = requests.post(url, headers=headers, data=json.dumps(payload)) assert response.status_code == 200, f"Failed to add dataset: {response.text}" - url = f"{DREMIO_HOST}/apiv2/source/Samples/file_format/samples.dremio.com/Dremio%20University/googleplaystore.csv" + url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2FSamples%2Fsamples.dremio.com%2FDremio%20University%2Fgoogleplaystore.csv" payload = { - "fieldDelimiter": ",", - "quote": '"', - "comment": "#", - "lineDelimiter": "\r\n", - "escape": '"', - "extractHeader": False, - "trimHeader": True, - "skipFirstLine": False, - "type": "Text", + "entityType": "dataset", + "type": "PHYSICAL_DATASET", + "path": [ + "Samples", + "samples.dremio.com", + "Dremio University", + "googleplaystore.csv", + ], + "format": { + "fieldDelimiter": ",", + "quote": '"', + "comment": "#", + "lineDelimiter": "\r\n", + "escape": '"', + "extractHeader": False, + "trimHeader": True, + "skipFirstLine": False, + "type": "Text", + }, } - response = requests.put(url, headers=headers, data=json.dumps(payload)) + response = requests.post(url, headers=headers, data=json.dumps(payload)) assert response.status_code == 200, f"Failed to add dataset: {response.text}" - url = f"{DREMIO_HOST}/apiv2/source/Samples/file_format/samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet" - payload = {"ignoreOtherFileFormats": False, "type": "Parquet"} + url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2FSamples%2Fsamples.dremio.com%2Ftpcds_sf1000%2Fcatalog_page%2F1ab266d5-18eb-4780-711d-0fa337fa6c00%2F0_0_0.parquet" + payload = { + "entityType": "dataset", + "type": "PHYSICAL_DATASET", + "path": [ + "Samples", + "samples.dremio.com", + "tpcds_sf1000", + "catalog_page", + "1ab266d5-18eb-4780-711d-0fa337fa6c00", + "0_0_0.parquet", + ], + "format": {"type": "Parquet"}, + } - response = requests.put(url, headers=headers, data=json.dumps(payload)) + response = requests.post(url, headers=headers, data=json.dumps(payload)) assert response.status_code == 200, f"Failed to add dataset: {response.text}" @@ -253,7 +306,7 @@ def create_view(headers): "entityType": "dataset", "type": "VIRTUAL_DATASET", "path": ["space", "test_folder", "raw"], - "sql": 'SELECT * FROM s3.warehouse."sample.parquet"', + "sql": "SELECT * FROM s3.warehouse", } response = requests.post(url, headers=headers, data=json.dumps(payload)) assert response.status_code == 200, f"Failed to create view: {response.text}" @@ -273,7 +326,7 @@ def create_view(headers): "entityType": "dataset", "type": "VIRTUAL_DATASET", "path": ["space", "test_folder", "customers"], - "sql": 'SELECT * FROM "mysql".northwind.customers', + "sql": "SELECT * FROM mysql.northwind.customers", "sqlContext": ["mysql", "northwind"], } response = requests.post(url, headers=headers, data=json.dumps(payload)) @@ -283,7 +336,7 @@ def create_view(headers): "entityType": "dataset", "type": "VIRTUAL_DATASET", "path": ["space", "test_folder", "orders"], - "sql": 'SELECT * FROM "mysql".northwind.orders', + "sql": "SELECT * FROM mysql.northwind.orders", "sqlContext": ["mysql", "northwind"], } response = requests.post(url, headers=headers, data=json.dumps(payload)) @@ -293,7 +346,7 @@ def create_view(headers): "entityType": "dataset", "type": "VIRTUAL_DATASET", "path": ["space", "test_folder", "metadata_aspect"], - "sql": 'SELECT * FROM "mysql".metagalaxy."metadata_aspect"', + "sql": "SELECT * FROM mysql.metagalaxy.metadata_aspect", "sqlContext": ["mysql", "metagalaxy"], } response = requests.post(url, headers=headers, data=json.dumps(payload)) @@ -303,7 +356,7 @@ def create_view(headers): "entityType": "dataset", "type": "VIRTUAL_DATASET", "path": ["space", "test_folder", "metadata_index"], - "sql": 'SELECT * FROM "mysql".metagalaxy."metadata_index"', + "sql": "SELECT * FROM mysql.metagalaxy.metadata_index", "sqlContext": ["mysql", "metagalaxy"], } response = requests.post(url, headers=headers, data=json.dumps(payload)) @@ -313,7 +366,7 @@ def create_view(headers): "entityType": "dataset", "type": "VIRTUAL_DATASET", "path": ["space", "test_folder", "metadata_index_view"], - "sql": 'SELECT * FROM "mysql".metagalaxy."metadata_index_view"', + "sql": "SELECT * FROM mysql.metagalaxy.metadata_index_view", "sqlContext": ["mysql", "metagalaxy"], } response = requests.post(url, headers=headers, data=json.dumps(payload)) @@ -422,14 +475,119 @@ def test_dremio_ingest( pytestconfig, tmp_path, ): - # Run the metadata ingestion pipeline. + # Run the metadata ingestion pipeline with specific output file config_file = (test_resources_dir / "dremio_to_file.yml").resolve() + output_path = tmp_path / "dremio_mces.json" + run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path) - # Verify the output. + # Verify the output mce_helpers.check_golden_file( pytestconfig, - output_path=tmp_path / "dremio_mces.json", + output_path=output_path, golden_path=test_resources_dir / "dremio_mces_golden.json", ignore_paths=[], ) + + +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_dremio_platform_instance_urns( + test_resources_dir, + dremio_setup, + pytestconfig, + tmp_path, +): + config_file = ( + test_resources_dir / "dremio_platform_instance_to_file.yml" + ).resolve() + output_path = tmp_path / "dremio_mces.json" + + run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path) + + with output_path.open() as f: + content = f.read() + # Skip if file is empty or just contains brackets + if not content or content.strip() in ("[]", "[", "]"): + pytest.fail(f"Output file is empty or invalid: {content}") + + try: + # Try to load as JSON Lines first + mces = [] + for line in content.splitlines(): + line = line.strip() + if line and line not in ("[", "]"): # Skip empty lines and bare brackets + mce = json.loads(line) + mces.append(mce) + except json.JSONDecodeError: + # If that fails, try loading as a single JSON array + try: + mces = json.loads(content) + except json.JSONDecodeError as e: + print(f"Failed to parse file content: {content}") + raise e + + # Verify MCEs + assert len(mces) > 0, "No MCEs found in output file" + + # Verify the platform instances + for mce in mces: + if "entityType" not in mce: + continue + + # Check dataset URN structure + if mce["entityType"] == "dataset" and "entityUrn" in mce: + assert ( + "test-platform.dremio" in mce["entityUrn"] + ), f"Platform instance missing in dataset URN: {mce['entityUrn']}" + + # Check aspects for both datasets and containers + if "aspectName" in mce: + # Check dataPlatformInstance aspect + if mce["aspectName"] == "dataPlatformInstance": + aspect = mce["aspect"] + if not isinstance(aspect, Dict) or "json" not in aspect: + continue + + aspect_json = aspect["json"] + if not isinstance(aspect_json, Dict): + continue + + if "instance" not in aspect_json: + continue + + instance = aspect_json["instance"] + expected_instance = "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)" + assert ( + instance == expected_instance + ), f"Invalid platform instance format: {instance}" + + # Verify against golden file + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_path, + golden_path=test_resources_dir / "dremio_platform_instance_mces_golden.json", + ignore_paths=[], + ) + + +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_dremio_schema_filter( + test_resources_dir, + dremio_setup, + pytestconfig, + tmp_path, +): + config_file = (test_resources_dir / "dremio_schema_filter_to_file.yml").resolve() + output_path = tmp_path / "dremio_mces.json" + + run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path) + + # Verify against golden file + mce_helpers.check_golden_file( + pytestconfig, + output_path=output_path, + golden_path=test_resources_dir / "dremio_schema_filter_mces_golden.json", + ignore_paths=[], + ) diff --git a/metadata-ingestion/tests/unit/dremio/test_dremio_schema_filter.py b/metadata-ingestion/tests/unit/dremio/test_dremio_schema_filter.py new file mode 100644 index 0000000000000..27df05296b752 --- /dev/null +++ b/metadata-ingestion/tests/unit/dremio/test_dremio_schema_filter.py @@ -0,0 +1,123 @@ +from unittest.mock import Mock + +import pytest + +from datahub.ingestion.source.dremio.dremio_api import DremioAPIOperations +from datahub.ingestion.source.dremio.dremio_config import DremioSourceConfig +from datahub.ingestion.source.dremio.dremio_reporting import DremioSourceReport + + +class TestDremioContainerFiltering: + @pytest.fixture + def dremio_api(self, monkeypatch): + # Mock the requests.Session + mock_session = Mock() + monkeypatch.setattr("requests.Session", Mock(return_value=mock_session)) + + # Mock the authentication response + mock_session.post.return_value.json.return_value = {"token": "dummy-token"} + mock_session.post.return_value.status_code = 200 + + config = DremioSourceConfig( + hostname="dummy-host", + port=9047, + tls=False, + authentication_method="password", + username="dummy-user", + password="dummy-password", + schema_pattern=dict(allow=[".*"], deny=[]), + ) + report = DremioSourceReport() + return DremioAPIOperations(config, report) + + def test_basic_allow_pattern(self, dremio_api): + """Test basic allow pattern matching""" + dremio_api.allow_schema_pattern = ["test"] + dremio_api.deny_schema_pattern = [] + + assert dremio_api.should_include_container([], "test") + assert dremio_api.should_include_container(["test"], "subfolder") + assert not dremio_api.should_include_container([], "prod_space") + + def test_basic_deny_pattern(self, dremio_api): + """Test basic deny pattern matching""" + dremio_api.allow_schema_pattern = [".*"] + dremio_api.deny_schema_pattern = ["test_space.*"] + + assert not dremio_api.should_include_container([], "test_space") + assert not dremio_api.should_include_container(["test_space"], "subfolder") + assert dremio_api.should_include_container([], "prod_space") + + def test_hierarchical_matching(self, dremio_api): + """Test matching with hierarchical paths""" + dremio_api.allow_schema_pattern = ["prod.data.*"] + dremio_api.deny_schema_pattern = [] + + assert dremio_api.should_include_container([], "prod") + assert dremio_api.should_include_container(["prod"], "data") + assert dremio_api.should_include_container(["prod", "data"], "sales") + assert not dremio_api.should_include_container([], "dev") + assert not dremio_api.should_include_container(["dev"], "data") + + def test_allow_and_deny_patterns(self, dremio_api): + """Test combination of allow and deny patterns""" + dremio_api.allow_schema_pattern = ["prod.*"] + dremio_api.deny_schema_pattern = ["prod.internal.*"] + + assert dremio_api.should_include_container([], "prod") + assert dremio_api.should_include_container(["prod"], "public") + assert dremio_api.should_include_container(["prod", "public"], "next") + assert not dremio_api.should_include_container(["prod"], "internal") + assert not dremio_api.should_include_container(["prod", "internal"], "secrets") + + def test_wildcard_patterns(self, dremio_api): + """Test wildcard pattern handling""" + dremio_api.allow_schema_pattern = [".*"] + dremio_api.deny_schema_pattern = [] + + assert dremio_api.should_include_container([], "any_space") + assert dremio_api.should_include_container(["any_space"], "any_folder") + + # Test with specific wildcard in middle + dremio_api.allow_schema_pattern = ["prod.*.public"] + assert dremio_api.should_include_container(["prod", "customer"], "public") + assert not dremio_api.should_include_container(["prod", "customer"], "private") + + def test_case_insensitive_matching(self, dremio_api): + """Test case-insensitive pattern matching""" + dremio_api.allow_schema_pattern = ["PROD.*"] + dremio_api.deny_schema_pattern = [] + + assert dremio_api.should_include_container([], "prod") + assert dremio_api.should_include_container([], "PROD") + assert dremio_api.should_include_container(["prod"], "DATA") + assert dremio_api.should_include_container(["PROD"], "data") + + def test_empty_patterns(self, dremio_api): + """Test behavior with empty patterns""" + dremio_api.allow_schema_pattern = [".*"] + dremio_api.deny_schema_pattern = [] + + # Should allow everything when allow pattern is empty + assert dremio_api.should_include_container([], "any_space") + assert dremio_api.should_include_container(["any_space"], "any_folder") + + def test_partial_path_matching(self, dremio_api): + """Test matching behavior with partial paths""" + dremio_api.allow_schema_pattern = ["^pr.*.data.*"] + dremio_api.deny_schema_pattern = [] + + assert dremio_api.should_include_container(["prod"], "data") + # Should match the partial path even though pattern doesn't have wildcards + assert dremio_api.should_include_container(["prod", "data"], "sales") + assert not dremio_api.should_include_container([], "dev") + assert not dremio_api.should_include_container(["dev", "data"], "sales") + + def test_partial_start_end_chars(self, dremio_api): + """Test matching behavior with partial paths""" + dremio_api.allow_schema_pattern = ["pr.*.data$"] + dremio_api.deny_schema_pattern = [] + + assert dremio_api.should_include_container(["prod"], "data") + # Should match the partial path even though pattern doesn't have wildcards + assert not dremio_api.should_include_container(["prod", "data"], "sales")