diff --git a/datahub-graphql-core/build.gradle b/datahub-graphql-core/build.gradle index 49a7fa7fbfbc2..47ada8e9929dd 100644 --- a/datahub-graphql-core/build.gradle +++ b/datahub-graphql-core/build.gradle @@ -3,6 +3,7 @@ plugins { id "io.github.kobylynskyi.graphql.codegen" version "4.1.1" } +apply from: '../gradle/coverage/java-coverage.gradle' dependencies { implementation project(':metadata-service:restli-client-api') diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle index b783efa09713d..372b0eb0570b9 100644 --- a/datahub-upgrade/build.gradle +++ b/datahub-upgrade/build.gradle @@ -5,6 +5,7 @@ plugins { } apply from: "../gradle/versioning/versioning.gradle" +apply from: "../gradle/coverage/java-coverage.gradle" ext { docker_registry = rootProject.ext.docker_registry == 'linkedin' ? 'acryldata' : docker_registry diff --git a/docs/managed-datahub/release-notes/v_0_3_7.md b/docs/managed-datahub/release-notes/v_0_3_7.md index 94cbdd79dbf5e..be3a2d97514ef 100644 --- a/docs/managed-datahub/release-notes/v_0_3_7.md +++ b/docs/managed-datahub/release-notes/v_0_3_7.md @@ -19,6 +19,13 @@ If you are using an older CLI/SDK version, then please upgrade it. This applies ## Release Changelog --- +### v0.3.7.5 + +- [GMS] Fix upstream lineage patching when path contained encoded slash +- [UI] Fix merging siblings schema with v1 and v2 fields +- [UI] Fix display nullable in schema field drawer +- [Ingestion] Reduce Data Product write volume from unset side-effect + ### v0.3.7.4 - [#11935](https://github.com/datahub-project/datahub/pull/11935) - Added environment variable for enabling stricter URN validation rules `STRICT_URN_VALIDATION_ENABLED` [[1](https://datahubproject.io/docs/what/urn/#restrictions)]. diff --git a/entity-registry/build.gradle b/entity-registry/build.gradle index 22e5b601d39db..e5baa95967f30 100644 --- a/entity-registry/build.gradle +++ b/entity-registry/build.gradle @@ -4,6 +4,8 @@ plugins { id 'java-test-fixtures' } +apply from: "../gradle/coverage/java-coverage.gradle" + dependencies { implementation spec.product.pegasus.data implementation spec.product.pegasus.generator @@ -53,3 +55,4 @@ dependencies { testFixturesAnnotationProcessor externalDependency.lombok } compileTestJava.dependsOn tasks.getByPath(':entity-registry:custom-test-model:modelDeploy') + diff --git a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java index f9af15a3d4cc6..f934dd8961ca3 100644 --- a/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java +++ b/entity-registry/src/test/java/com/linkedin/metadata/aspect/patch/template/UpstreamLineageTemplateTest.java @@ -272,8 +272,8 @@ public void testLargePatchStandard() throws Exception { UpstreamLineage result = upstreamLineageTemplate.applyPatch(upstreamLineage, jsonPatch); long end = System.currentTimeMillis(); assertTrue( - end - start < 10000, - String.format("Expected less then 10 seconds patch actual %s ms", end - start)); + end - start < 20000, + String.format("Expected less then 20 seconds patch actual %s ms", end - start)); assertEquals(result.getUpstreams().size(), 187, "Expected 1 less upstream"); assertEquals(result.getFineGrainedLineages().size(), 607); diff --git a/gradle/coverage/java-coverage.gradle b/gradle/coverage/java-coverage.gradle new file mode 100644 index 0000000000000..17260c1a30978 --- /dev/null +++ b/gradle/coverage/java-coverage.gradle @@ -0,0 +1,31 @@ +apply plugin: "jacoco" + +jacoco { + toolVersion = "0.8.12" +} + +/* +These need to run after evaluation since jacoco plugin alters the test task based on the included test +lib dependencies defined in each subproject (junit or testNG) +*/ +afterEvaluate { + test { + finalizedBy jacocoTestReport + } + + jacocoTestReport { + dependsOn test + reports { + xml { + required = true + /* + Tools that aggregate and analyse coverage tools search for the coverage result files. Keeping them under one + folder will minimize the time spent searching through the full source tree. + */ + outputLocation = rootProject.layout.buildDirectory.file("coverage-reports/jacoco-${project.name}.xml") + } + csv.required = false + html.required = false + } + } +} \ No newline at end of file diff --git a/gradle/coverage/python-coverage.gradle b/gradle/coverage/python-coverage.gradle new file mode 100644 index 0000000000000..0ab921dfb21ff --- /dev/null +++ b/gradle/coverage/python-coverage.gradle @@ -0,0 +1,18 @@ +//coverage related args to be passed to pytest +ext.get_coverage_args = { test_name = "" -> + + def coverage_file_name = "pycov-${project.name}${test_name}.xml" + + /* + Tools that aggregate and analyse coverage tools search for the coverage result files. Keeping them under one folder + will minimize the time spent searching through the full source tree. + */ + def base_path = "${rootProject.buildDir}/coverage-reports" + + /* + --cov=src was added via setup.cfg in many of the python projects but for some reason, was not getting picked up + consistently, so adding it explicitly. Centralizing these params would make it easier to adjust them for all python + projects (with overrides being in the sub-project build script.) + */ + return "--cov-report xml:${base_path}/${coverage_file_name} --cov=src" +} \ No newline at end of file diff --git a/ingestion-scheduler/build.gradle b/ingestion-scheduler/build.gradle index 9505ec57aa858..fb1cfd4e27a4b 100644 --- a/ingestion-scheduler/build.gradle +++ b/ingestion-scheduler/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: "../gradle/coverage/java-coverage.gradle" + dependencies { implementation project(path: ':metadata-models') implementation project(path: ':metadata-io') diff --git a/li-utils/build.gradle b/li-utils/build.gradle index 975cd2bccccf3..e00fcc2854612 100644 --- a/li-utils/build.gradle +++ b/li-utils/build.gradle @@ -1,8 +1,11 @@ + plugins { id 'java-library' id 'pegasus' } +apply from: "../gradle/coverage/java-coverage.gradle" + dependencies { api spec.product.pegasus.data diff --git a/metadata-auth/auth-api/build.gradle b/metadata-auth/auth-api/build.gradle index acc1af7e2e3ad..3aafaf45bc2cb 100644 --- a/metadata-auth/auth-api/build.gradle +++ b/metadata-auth/auth-api/build.gradle @@ -8,6 +8,7 @@ apply plugin: 'signing' apply plugin: 'maven-publish' apply plugin: 'io.codearte.nexus-staging' apply from: '../../metadata-integration/java/versioning.gradle' +apply from: '../../gradle/coverage/java-coverage.gradle' jar { archiveClassifier = "lib" diff --git a/metadata-dao-impl/kafka-producer/build.gradle b/metadata-dao-impl/kafka-producer/build.gradle index 2df15309810db..d3ba031817afe 100644 --- a/metadata-dao-impl/kafka-producer/build.gradle +++ b/metadata-dao-impl/kafka-producer/build.gradle @@ -1,5 +1,7 @@ apply plugin: 'java' +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-events:mxe-avro') implementation project(':metadata-events:mxe-registration') diff --git a/metadata-events/mxe-utils-avro/build.gradle b/metadata-events/mxe-utils-avro/build.gradle index 860ced6af2581..f893fadb8e2df 100644 --- a/metadata-events/mxe-utils-avro/build.gradle +++ b/metadata-events/mxe-utils-avro/build.gradle @@ -3,6 +3,8 @@ plugins { id 'pegasus' } +apply from: "../../gradle/coverage/java-coverage.gradle" + dependencies { api project(':metadata-events:mxe-avro') api project(':metadata-models') diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle index 9506609a10044..f30858ba6a14e 100644 --- a/metadata-ingestion-modules/airflow-plugin/build.gradle +++ b/metadata-ingestion-modules/airflow-plugin/build.gradle @@ -2,6 +2,8 @@ plugins { id 'base' } +apply from: "../../gradle/coverage/python-coverage.gradle" + ext { python_executable = 'python3' venv_name = 'venv' @@ -97,7 +99,7 @@ task testQuick(type: Exec, dependsOn: installTest) { inputs.files(project.fileTree(dir: "tests/")) commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + - "pytest --cov-config=setup.cfg --cov-report xml:coverage_quick.xml -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" + "pytest --cov-config=setup.cfg ${get_coverage_args('quick')} -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" } diff --git a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py index 44efd94f834b1..3becf10703df6 100644 --- a/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py +++ b/metadata-ingestion-modules/airflow-plugin/tests/integration/test_plugin.py @@ -8,6 +8,7 @@ import random import signal import subprocess +import textwrap import time from typing import Any, Iterator, Sequence @@ -110,6 +111,48 @@ def _wait_for_dag_finish( raise NotReadyError(f"DAG has not finished yet: {dag_run['state']}") +def _dump_dag_logs(airflow_instance: AirflowInstance, dag_id: str) -> None: + # Get the dag run info + res = airflow_instance.session.get( + f"{airflow_instance.airflow_url}/api/v1/dags/{dag_id}/dagRuns", timeout=5 + ) + res.raise_for_status() + dag_run = res.json()["dag_runs"][0] + dag_run_id = dag_run["dag_run_id"] + + # List the tasks in the dag run + res = airflow_instance.session.get( + f"{airflow_instance.airflow_url}/api/v1/dags/{dag_id}/dagRuns/{dag_run_id}/taskInstances", + timeout=5, + ) + res.raise_for_status() + task_instances = res.json()["task_instances"] + + # Sort tasks by start_date to maintain execution order + task_instances.sort(key=lambda x: x["start_date"] or "") + + print(f"\nTask execution order for DAG {dag_id}:") + for task in task_instances: + task_id = task["task_id"] + state = task["state"] + try_number = task.get("try_number", 1) + + task_header = f"Task: {task_id} (State: {state}; Try: {try_number})" + + # Get logs for the task's latest try number + try: + res = airflow_instance.session.get( + f"{airflow_instance.airflow_url}/api/v1/dags/{dag_id}/dagRuns/{dag_run_id}" + f"/taskInstances/{task_id}/logs/{try_number}", + params={"full_content": "true"}, + timeout=5, + ) + res.raise_for_status() + print(f"\n=== {task_header} ===\n{textwrap.indent(res.text, ' ')}") + except Exception as e: + print(f"Failed to fetch logs for {task_header}: {e}") + + @contextlib.contextmanager def _run_airflow( tmp_path: pathlib.Path, @@ -377,6 +420,11 @@ def test_airflow_plugin( print("Sleeping for a few seconds to let the plugin finish...") time.sleep(10) + try: + _dump_dag_logs(airflow_instance, dag_id) + except Exception as e: + print(f"Failed to dump DAG logs: {e}") + if dag_id == DAG_TO_SKIP_INGESTION: # Verify that no MCPs were generated. assert not os.path.exists(airflow_instance.metadata_file) diff --git a/metadata-ingestion-modules/airflow-plugin/tox.ini b/metadata-ingestion-modules/airflow-plugin/tox.ini index 2e4596a24c2a6..28c0b9532bcb8 100644 --- a/metadata-ingestion-modules/airflow-plugin/tox.ini +++ b/metadata-ingestion-modules/airflow-plugin/tox.ini @@ -59,6 +59,6 @@ commands = [testenv:py310-airflow24] extras = dev,integration-tests,plugin-v2,test-airflow24 -[testenv:py310-airflow{26,27,28},py311-airflow{29,210}] +[testenv:py3{10,11}-airflow{26,27,28,29,210}] extras = dev,integration-tests,plugin-v2 diff --git a/metadata-ingestion-modules/dagster-plugin/build.gradle b/metadata-ingestion-modules/dagster-plugin/build.gradle index 0f11af9ca83d7..0d57bb5bfdff7 100644 --- a/metadata-ingestion-modules/dagster-plugin/build.gradle +++ b/metadata-ingestion-modules/dagster-plugin/build.gradle @@ -2,6 +2,8 @@ plugins { id 'base' } +apply from: "../../gradle/coverage/python-coverage.gradle" + ext { python_executable = 'python3' venv_name = 'venv' @@ -84,7 +86,7 @@ task testQuick(type: Exec, dependsOn: installDevTest) { outputs.dir("${venv_name}") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + - "pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" + "pytest -vv ${get_coverage_args('quick')} --continue-on-collection-errors --junit-xml=junit.quick.xml" } task buildWheel(type: Exec, dependsOn: [environmentSetup]) { diff --git a/metadata-ingestion-modules/gx-plugin/build.gradle b/metadata-ingestion-modules/gx-plugin/build.gradle index 2288ae6bd8396..70bc3e0360a3a 100644 --- a/metadata-ingestion-modules/gx-plugin/build.gradle +++ b/metadata-ingestion-modules/gx-plugin/build.gradle @@ -2,6 +2,8 @@ plugins { id 'base' } +apply from: "../../gradle/coverage/python-coverage.gradle" + ext { python_executable = 'python3' venv_name = 'venv' @@ -84,7 +86,7 @@ task testQuick(type: Exec, dependsOn: installDevTest) { outputs.dir("${venv_name}") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + - "pytest -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" + "pytest -vv ${get_coverage_args('quick')} --continue-on-collection-errors --junit-xml=junit.quick.xml" } task buildWheel(type: Exec, dependsOn: [environmentSetup]) { diff --git a/metadata-ingestion-modules/prefect-plugin/build.gradle b/metadata-ingestion-modules/prefect-plugin/build.gradle index bc091c6493364..a2b240cc8bf70 100644 --- a/metadata-ingestion-modules/prefect-plugin/build.gradle +++ b/metadata-ingestion-modules/prefect-plugin/build.gradle @@ -2,6 +2,8 @@ plugins { id 'base' } +apply from: "../../gradle/coverage/python-coverage.gradle" + ext { python_executable = 'python3' venv_name = 'venv' @@ -82,14 +84,14 @@ task testQuick(type: Exec, dependsOn: installDevTest) { outputs.dir("${venv_name}") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + - "pytest --cov-config=setup.cfg --cov-report xml:coverage_quick.xml -vv --continue-on-collection-errors --junit-xml=junit.quick.xml -s" + "pytest --cov-config=setup.cfg ${get_coverage_args('quick')} -vv --continue-on-collection-errors --junit-xml=junit.quick.xml -s" } task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) { commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + - "pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml" + "pytest -m 'not slow_integration' -vv ${get_coverage_args('full')} --continue-on-collection-errors --junit-xml=junit.full.xml" } diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle index 4e03dd6e2faaf..2c5d8e6c9646a 100644 --- a/metadata-ingestion/build.gradle +++ b/metadata-ingestion/build.gradle @@ -2,6 +2,8 @@ plugins { id 'base' } +apply from: "../gradle/coverage/python-coverage.gradle" + ext { python_executable = 'python3' venv_name = 'venv' @@ -11,10 +13,6 @@ if (!project.hasProperty("extra_pip_requirements")) { ext.extra_pip_requirements = "" } -def get_coverage_arg(test_name) { - return "--cov-report xml:coverage_${test_name}.xml " -} - task checkPythonVersion(type: Exec) { commandLine python_executable, '-c', 'import sys; sys.version_info >= (3, 8), f"Python version {sys.version_info[:2]} not allowed"' @@ -134,7 +132,7 @@ task testQuick(type: Exec, dependsOn: [installDev, ':metadata-models:generateJso inputs.files(project.fileTree(dir: "src/", include: "**/*.py")) inputs.files(project.fileTree(dir: "tests/")) outputs.dir("${venv_name}") - def cvg_arg = get_coverage_arg("quick") + def cvg_arg = get_coverage_args("quick") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "pytest ${cvg_arg} tests/unit --random-order --durations=20 -m 'not integration' -vv --continue-on-collection-errors --junit-xml=junit.quick.xml" @@ -166,19 +164,19 @@ task testSingle(dependsOn: [installDevTest]) { } task testIntegrationBatch0(type: Exec, dependsOn: [installDevTest]) { - def cvg_arg = get_coverage_arg("intBatch0") + def cvg_arg = get_coverage_args("intBatch0") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "pytest ${cvg_arg} --durations=50 -m 'integration_batch_0' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch0.xml" } task testIntegrationBatch1(type: Exec, dependsOn: [installDevTest]) { - def cvg_arg = get_coverage_arg("intBatch1") + def cvg_arg = get_coverage_args("intBatch1") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "pytest ${cvg_arg} --durations=50 -m 'integration_batch_1' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch1.xml" } task testIntegrationBatch2(type: Exec, dependsOn: [installDevTest]) { - def cvg_arg = get_coverage_arg("intBatch2") + def cvg_arg = get_coverage_args("intBatch2") commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "pytest ${cvg_arg} --durations=20 -m 'integration_batch_2' -vv --continue-on-collection-errors --junit-xml=junit.integrationbatch2.xml" diff --git a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py index 56e02e4329055..181c70adc640a 100644 --- a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py +++ b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py @@ -121,7 +121,7 @@ def fqn(self) -> str: return ( self.qualified_name or self.id - or Urn.create_from_string(self.urn).get_entity_id()[0] + or Urn.from_string(self.urn).get_entity_id()[0] ) @validator("urn", pre=True, always=True) diff --git a/metadata-ingestion/src/datahub/cli/put_cli.py b/metadata-ingestion/src/datahub/cli/put_cli.py index 0a40a9f4ccf92..d3a6fb5caaf19 100644 --- a/metadata-ingestion/src/datahub/cli/put_cli.py +++ b/metadata-ingestion/src/datahub/cli/put_cli.py @@ -105,7 +105,7 @@ def platform( """ if name.startswith(f"urn:li:{DataPlatformUrn.ENTITY_TYPE}"): - platform_urn = DataPlatformUrn.create_from_string(name) + platform_urn = DataPlatformUrn.from_string(name) platform_name = platform_urn.get_entity_id_as_string() else: platform_name = name.lower() diff --git a/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py b/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py index 8ec4d3ad24937..857a6fbb4e18e 100644 --- a/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py +++ b/metadata-ingestion/src/datahub/cli/specific/dataproduct_cli.py @@ -45,7 +45,7 @@ def _get_owner_urn(maybe_urn: str) -> str: def _abort_if_non_existent_urn(graph: DataHubGraph, urn: str, operation: str) -> None: try: - parsed_urn: Urn = Urn.create_from_string(urn) + parsed_urn: Urn = Urn.from_string(urn) entity_type = parsed_urn.get_type() except Exception: click.secho(f"Provided urn {urn} does not seem valid", fg="red") diff --git a/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py index 3790399539437..779b42e1e1ee9 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_patch_builder.py @@ -1,4 +1,5 @@ import json +import time from collections import defaultdict from dataclasses import dataclass from typing import Any, Dict, Iterable, List, Optional, Sequence, Union @@ -6,12 +7,15 @@ from datahub.emitter.aspect import JSON_PATCH_CONTENT_TYPE from datahub.emitter.serialization_helper import pre_json_transform from datahub.metadata.schema_classes import ( + AuditStampClass, ChangeTypeClass, + EdgeClass, GenericAspectClass, KafkaAuditHeaderClass, MetadataChangeProposalClass, SystemMetadataClass, ) +from datahub.metadata.urns import Urn from datahub.utilities.urns.urn import guess_entity_type @@ -89,3 +93,42 @@ def build(self) -> Iterable[MetadataChangeProposalClass]: ) for aspect_name, patches in self.patches.items() ] + + @classmethod + def _mint_auditstamp(cls, message: Optional[str] = None) -> AuditStampClass: + """ + Creates an AuditStampClass instance with the current timestamp and other default values. + + Args: + message: The message associated with the audit stamp (optional). + + Returns: + An instance of AuditStampClass. + """ + return AuditStampClass( + time=int(time.time() * 1000.0), + actor="urn:li:corpuser:datahub", + message=message, + ) + + @classmethod + def _ensure_urn_type( + cls, entity_type: str, edges: List[EdgeClass], context: str + ) -> None: + """ + Ensures that the destination URNs in the given edges have the specified entity type. + + Args: + entity_type: The entity type to check against. + edges: A list of Edge objects. + context: The context or description of the operation. + + Raises: + ValueError: If any of the destination URNs is not of the specified entity type. + """ + for e in edges: + urn = Urn.from_string(e.destinationUrn) + if not urn.entity_type == entity_type: + raise ValueError( + f"{context}: {e.destinationUrn} is not of type {entity_type}" + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py index 319c838d2658a..42f82704c81b9 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_audit.py @@ -190,7 +190,7 @@ def from_string_name(cls, ref: str) -> "BigQueryTableRef": @classmethod def from_urn(cls, urn: str) -> "BigQueryTableRef": """Raises: ValueError if urn is not a valid BigQuery table URN.""" - dataset_urn = DatasetUrn.create_from_string(urn) + dataset_urn = DatasetUrn.from_string(urn) split = dataset_urn.name.rsplit(".", 3) if len(split) == 3: project, dataset, table = split diff --git a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py index e4829f8713cf7..42e025073b534 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py +++ b/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py @@ -653,7 +653,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: is_resource_row: bool = not row["subresource"] entity_urn = row["resource"] - entity_type = Urn.create_from_string(row["resource"]).get_type() + entity_type = Urn.from_string(row["resource"]).get_type() term_associations: List[ GlossaryTermAssociationClass diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py index f814108c37760..319290d25169a 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py @@ -396,10 +396,12 @@ def process_dataset( ): yield dremio_mcp # Check if the emitted aspect is SchemaMetadataClass - if isinstance(dremio_mcp.metadata, SchemaMetadataClass): + if isinstance( + dremio_mcp.metadata, MetadataChangeProposalWrapper + ) and isinstance(dremio_mcp.metadata.aspect, SchemaMetadataClass): self.sql_parsing_aggregator.register_schema( urn=dataset_urn, - schema=dremio_mcp.metadata, + schema=dremio_mcp.metadata.aspect, ) if dataset_info.dataset_type == DremioDatasetType.VIEW: diff --git a/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py b/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py index aa5913f5dc66b..99aa5f54f6a57 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py +++ b/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py @@ -227,7 +227,7 @@ def collapse_name(name: str, collapse_urns: CollapseUrns) -> str: def collapse_urn(urn: str, collapse_urns: CollapseUrns) -> str: if len(collapse_urns.urns_suffix_regex) == 0: return urn - urn_obj = DatasetUrn.create_from_string(urn) + urn_obj = DatasetUrn.from_string(urn) name = collapse_name(name=urn_obj.get_dataset_name(), collapse_urns=collapse_urns) data_platform_urn = urn_obj.get_data_platform_urn() return str( diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py index 0f35e1a67fede..ca67cd6daa045 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py @@ -277,7 +277,12 @@ def delete_dpi_from_datajobs(self, job: DataJobEntity) -> None: assert self.ctx.graph dpis = self.fetch_dpis(job.urn, self.config.batch_size) - dpis.sort(key=lambda x: x["created"]["time"], reverse=True) + dpis.sort( + key=lambda x: x["created"]["time"] + if x["created"] and x["created"]["time"] + else 0, + reverse=True, + ) with ThreadPoolExecutor(max_workers=self.config.max_workers) as executor: if self.config.keep_last_n: diff --git a/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py b/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py index 7ec7bb7e589d6..3b367cdea5813 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py +++ b/metadata-ingestion/src/datahub/ingestion/source/gc/soft_deleted_entity_cleanup.py @@ -104,7 +104,7 @@ def __init__( def delete_entity(self, urn: str) -> None: assert self.ctx.graph - entity_urn = Urn.create_from_string(urn) + entity_urn = Urn.from_string(urn) self.report.num_soft_deleted_entity_removed += 1 self.report.num_soft_deleted_entity_removed_by_type[entity_urn.entity_type] = ( self.report.num_soft_deleted_entity_removed_by_type.get( diff --git a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py index f7d783cd3dec0..c2c43a0e805b2 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ge_data_profiler.py @@ -57,7 +57,11 @@ convert_to_cardinality, ) from datahub.ingestion.source.sql.sql_report import SQLSourceReport -from datahub.metadata.com.linkedin.pegasus2avro.schema import EditableSchemaMetadata +from datahub.ingestion.source.sql.sql_types import resolve_sql_type +from datahub.metadata.com.linkedin.pegasus2avro.schema import ( + EditableSchemaMetadata, + NumberType, +) from datahub.metadata.schema_classes import ( DatasetFieldProfileClass, DatasetProfileClass, @@ -361,6 +365,8 @@ class _SingleDatasetProfiler(BasicDatasetProfilerBase): platform: str env: str + column_types: Dict[str, str] = dataclasses.field(default_factory=dict) + def _get_columns_to_profile(self) -> List[str]: if not self.config.any_field_level_metrics_enabled(): return [] @@ -374,6 +380,7 @@ def _get_columns_to_profile(self) -> List[str]: for col_dict in self.dataset.columns: col = col_dict["name"] + self.column_types[col] = str(col_dict["type"]) # We expect the allow/deny patterns to specify '.' if not self.config._allow_deny_patterns.allowed( f"{self.dataset_name}.{col}" @@ -430,6 +437,21 @@ def _get_column_type(self, column_spec: _SingleColumnSpec, column: str) -> None: self.dataset, column ) + if column_spec.type_ == ProfilerDataType.UNKNOWN: + try: + datahub_field_type = resolve_sql_type( + self.column_types[column], self.dataset.engine.dialect.name.lower() + ) + except Exception as e: + logger.debug( + f"Error resolving sql type {self.column_types[column]}: {e}" + ) + datahub_field_type = None + if datahub_field_type is None: + return + if isinstance(datahub_field_type, NumberType): + column_spec.type_ = ProfilerDataType.NUMERIC + @_run_with_query_combiner def _get_column_cardinality( self, column_spec: _SingleColumnSpec, column: str diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py index 89ca160ba1f48..9ec73a9af96dc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_types.py @@ -276,7 +276,6 @@ def resolve_vertica_modified_type(type_string: str) -> Any: return VERTICA_SQL_TYPES_MAP[type_string] -# see https://docs.snowflake.com/en/sql-reference/intro-summary-data-types.html SNOWFLAKE_TYPES_MAP: Dict[str, Any] = { "NUMBER": NumberType, "DECIMAL": NumberType, @@ -312,6 +311,18 @@ def resolve_vertica_modified_type(type_string: str) -> Any: "GEOGRAPHY": None, } + +def resolve_snowflake_modified_type(type_string: str) -> Any: + # Match types with precision and scale, e.g., 'DECIMAL(38,0)' + match = re.match(r"([a-zA-Z_]+)\(\d+,\s\d+\)", type_string) + if match: + modified_type_base = match.group(1) # Extract the base type + return SNOWFLAKE_TYPES_MAP.get(modified_type_base, None) + + # Fallback for types without precision/scale + return SNOWFLAKE_TYPES_MAP.get(type_string, None) + + # see https://github.com/googleapis/python-bigquery-sqlalchemy/blob/main/sqlalchemy_bigquery/_types.py#L32 BIGQUERY_TYPES_MAP: Dict[str, Any] = { "STRING": StringType, @@ -380,6 +391,7 @@ def resolve_vertica_modified_type(type_string: str) -> Any: "row": RecordType, "map": MapType, "array": ArrayType, + "json": RecordType, } # https://docs.aws.amazon.com/athena/latest/ug/data-types.html @@ -490,7 +502,7 @@ def resolve_sql_type( TypeClass = resolve_vertica_modified_type(column_type) elif platform == "snowflake": # Snowflake types are uppercase, so we check that. - TypeClass = _merged_mapping.get(column_type.upper()) + TypeClass = resolve_snowflake_modified_type(column_type.upper()) if TypeClass: return TypeClass() diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py index c60f4dca28882..355ca7a373653 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/add_dataset_tags.py @@ -74,7 +74,7 @@ def handle_end_of_stream( logger.debug("Generating tags") for tag_association in self.processed_tags.values(): - tag_urn = TagUrn.create_from_string(tag_association.tag) + tag_urn = TagUrn.from_string(tag_association.tag) mcps.append( MetadataChangeProposalWrapper( entityUrn=tag_urn.urn(), diff --git a/metadata-ingestion/src/datahub/ingestion/transformer/generic_aspect_transformer.py b/metadata-ingestion/src/datahub/ingestion/transformer/generic_aspect_transformer.py index 3c0bc00633e7c..5bf70274dce89 100644 --- a/metadata-ingestion/src/datahub/ingestion/transformer/generic_aspect_transformer.py +++ b/metadata-ingestion/src/datahub/ingestion/transformer/generic_aspect_transformer.py @@ -100,7 +100,7 @@ def transform( ) if transformed_aspect: # for end of stream records, we modify the workunit-id - structured_urn = Urn.create_from_string(urn) + structured_urn = Urn.from_string(urn) simple_name = "-".join(structured_urn.get_entity_id()) record_metadata = envelope.metadata.copy() record_metadata.update( diff --git a/metadata-ingestion/src/datahub/integrations/assertion/common.py b/metadata-ingestion/src/datahub/integrations/assertion/common.py index 9ffad5cf66640..f0b7510df14a3 100644 --- a/metadata-ingestion/src/datahub/integrations/assertion/common.py +++ b/metadata-ingestion/src/datahub/integrations/assertion/common.py @@ -42,7 +42,7 @@ def get_entity_name(assertion: BaseEntityAssertion) -> Tuple[str, str, str]: if qualified_name is not None: parts = qualified_name.split(".") else: - urn_id = Urn.create_from_string(assertion.entity).entity_ids[1] + urn_id = Urn.from_string(assertion.entity).entity_ids[1] parts = urn_id.split(".") if len(parts) > 3: parts = parts[-3:] diff --git a/metadata-ingestion/src/datahub/lite/duckdb_lite.py b/metadata-ingestion/src/datahub/lite/duckdb_lite.py index f40a2e9498e62..8931738352092 100644 --- a/metadata-ingestion/src/datahub/lite/duckdb_lite.py +++ b/metadata-ingestion/src/datahub/lite/duckdb_lite.py @@ -609,7 +609,7 @@ def get_typed_aspect( aspect_map, DataPlatformInstanceClass ) # type: ignore - needs_platform = Urn.create_from_string(entity_urn).get_type() in [ + needs_platform = Urn.from_string(entity_urn).get_type() in [ "dataset", "container", "chart", @@ -617,7 +617,7 @@ def get_typed_aspect( "dataFlow", "dataJob", ] - entity_urn_parsed = Urn.create_from_string(entity_urn) + entity_urn_parsed = Urn.from_string(entity_urn) if entity_urn_parsed.get_type() in ["dataFlow", "dataJob"]: self.add_edge( entity_urn, @@ -630,15 +630,12 @@ def get_typed_aspect( # this is a top-level entity if not dpi: logger.debug(f"No data platform instance for {entity_urn}") - maybe_parent_urn = Urn.create_from_string(entity_urn).get_entity_id()[0] + maybe_parent_urn = Urn.from_string(entity_urn).get_entity_id()[0] needs_dpi = False if maybe_parent_urn.startswith(Urn.URN_PREFIX): parent_urn = maybe_parent_urn - if ( - Urn.create_from_string(maybe_parent_urn).get_type() - == "dataPlatform" - ): - data_platform_urn = DataPlatformUrn.create_from_string( + if Urn.from_string(maybe_parent_urn).get_type() == "dataPlatform": + data_platform_urn = DataPlatformUrn.from_string( maybe_parent_urn ) needs_dpi = True @@ -660,7 +657,7 @@ def get_typed_aspect( logger.error(f"Failed to generate edges entity {entity_urn}", e) parent_urn = str(data_platform_instance_urn) else: - data_platform_urn = DataPlatformUrn.create_from_string(dpi.platform) + data_platform_urn = DataPlatformUrn.from_string(dpi.platform) data_platform_instance = dpi.instance or "default" data_platform_instance_urn = Urn( entity_type="dataPlatformInstance", @@ -673,9 +670,7 @@ def get_typed_aspect( parent_urn = "__root__" types = ( - subtypes.typeNames - if subtypes - else [Urn.create_from_string(entity_urn).get_type()] + subtypes.typeNames if subtypes else [Urn.from_string(entity_urn).get_type()] ) for t in types: type_urn = Urn(entity_type="systemNode", entity_id=[parent_urn, t]) @@ -686,7 +681,7 @@ def get_typed_aspect( def _create_edges_from_data_platform_instance( self, data_platform_instance_urn: Urn ) -> None: - data_platform_urn = DataPlatformUrn.create_from_string( + data_platform_urn = DataPlatformUrn.from_string( data_platform_instance_urn.get_entity_id()[0] ) data_platform_instances_urn = Urn( @@ -735,7 +730,7 @@ def post_update_hook( if isinstance(aspect, DatasetPropertiesClass): dp: DatasetPropertiesClass = aspect if dp.name: - specific_urn = DatasetUrn.create_from_string(entity_urn) + specific_urn = DatasetUrn.from_string(entity_urn) if ( specific_urn.get_data_platform_urn().get_entity_id_as_string() == "looker" @@ -755,7 +750,7 @@ def post_update_hook( self.add_edge(entity_urn, "name", cp.name, remove_existing=True) elif isinstance(aspect, DataPlatformInstanceClass): dpi: DataPlatformInstanceClass = aspect - data_platform_urn = DataPlatformUrn.create_from_string(dpi.platform) + data_platform_urn = DataPlatformUrn.from_string(dpi.platform) data_platform_instance = dpi.instance or "default" data_platform_instance_urn = Urn( entity_type="dataPlatformInstance", @@ -763,7 +758,7 @@ def post_update_hook( ) self._create_edges_from_data_platform_instance(data_platform_instance_urn) elif isinstance(aspect, ChartInfoClass): - urn = Urn.create_from_string(entity_urn) + urn = Urn.from_string(entity_urn) self.add_edge( entity_urn, "name", @@ -771,7 +766,7 @@ def post_update_hook( remove_existing=True, ) elif isinstance(aspect, DashboardInfoClass): - urn = Urn.create_from_string(entity_urn) + urn = Urn.from_string(entity_urn) self.add_edge( entity_urn, "name", diff --git a/metadata-ingestion/src/datahub/specific/chart.py b/metadata-ingestion/src/datahub/specific/chart.py index cc68168b68db7..104a7c21a07e2 100644 --- a/metadata-ingestion/src/datahub/specific/chart.py +++ b/metadata-ingestion/src/datahub/specific/chart.py @@ -1,10 +1,8 @@ -import time from typing import Dict, List, Optional, Union from datahub.emitter.mcp_patch_builder import MetadataPatchProposal from datahub.metadata.schema_classes import ( AccessLevelClass, - AuditStampClass, ChangeAuditStampsClass, ChartInfoClass as ChartInfo, ChartTypeClass, @@ -47,43 +45,6 @@ def __init__( ) self.ownership_patch_helper = OwnershipPatchHelper(self) - def _mint_auditstamp(self, message: Optional[str] = None) -> AuditStampClass: - """ - Creates an AuditStampClass instance with the current timestamp and other default values. - - Args: - message: The message associated with the audit stamp (optional). - - Returns: - An instance of AuditStampClass. - """ - return AuditStampClass( - time=int(time.time() * 1000.0), - actor="urn:li:corpuser:datahub", - message=message, - ) - - def _ensure_urn_type( - self, entity_type: str, edges: List[Edge], context: str - ) -> None: - """ - Ensures that the destination URNs in the given edges have the specified entity type. - - Args: - entity_type: The entity type to check against. - edges: A list of Edge objects. - context: The context or description of the operation. - - Raises: - ValueError: If any of the destination URNs is not of the specified entity type. - """ - for e in edges: - urn = Urn.create_from_string(e.destinationUrn) - if not urn.get_type() == entity_type: - raise ValueError( - f"{context}: {e.destinationUrn} is not of type {entity_type}" - ) - def add_owner(self, owner: Owner) -> "ChartPatchBuilder": """ Adds an owner to the ChartPatchBuilder. diff --git a/metadata-ingestion/src/datahub/specific/dashboard.py b/metadata-ingestion/src/datahub/specific/dashboard.py index f57df15914369..da5abbfd1dc12 100644 --- a/metadata-ingestion/src/datahub/specific/dashboard.py +++ b/metadata-ingestion/src/datahub/specific/dashboard.py @@ -1,10 +1,8 @@ -import time from typing import Dict, List, Optional, Union from datahub.emitter.mcp_patch_builder import MetadataPatchProposal from datahub.metadata.schema_classes import ( AccessLevelClass, - AuditStampClass, ChangeAuditStampsClass, DashboardInfoClass as DashboardInfo, EdgeClass as Edge, @@ -46,43 +44,6 @@ def __init__( ) self.ownership_patch_helper = OwnershipPatchHelper(self) - def _mint_auditstamp(self, message: Optional[str] = None) -> AuditStampClass: - """ - Creates an AuditStampClass instance with the current timestamp and other default values. - - Args: - message: The message associated with the audit stamp (optional). - - Returns: - An instance of AuditStampClass. - """ - return AuditStampClass( - time=int(time.time() * 1000.0), - actor="urn:li:corpuser:datahub", - message=message, - ) - - def _ensure_urn_type( - self, entity_type: str, edges: List[Edge], context: str - ) -> None: - """ - Ensures that the destination URNs in the given edges have the specified entity type. - - Args: - entity_type: The entity type to check against. - edges: A list of Edge objects. - context: The context or description of the operation. - - Raises: - ValueError: If any of the destination URNs is not of the specified entity type. - """ - for e in edges: - urn = Urn.create_from_string(e.destinationUrn) - if not urn.get_type() == entity_type: - raise ValueError( - f"{context}: {e.destinationUrn} is not of type {entity_type}" - ) - def add_owner(self, owner: Owner) -> "DashboardPatchBuilder": """ Adds an owner to the DashboardPatchBuilder. diff --git a/metadata-ingestion/src/datahub/specific/datajob.py b/metadata-ingestion/src/datahub/specific/datajob.py index 8da8edc8ef0f2..fb7b0ae7816f1 100644 --- a/metadata-ingestion/src/datahub/specific/datajob.py +++ b/metadata-ingestion/src/datahub/specific/datajob.py @@ -1,9 +1,7 @@ -import time from typing import Dict, List, Optional, Union from datahub.emitter.mcp_patch_builder import MetadataPatchProposal from datahub.metadata.schema_classes import ( - AuditStampClass, DataJobInfoClass as DataJobInfo, DataJobInputOutputClass as DataJobInputOutput, EdgeClass as Edge, @@ -16,10 +14,9 @@ SystemMetadataClass, TagAssociationClass as Tag, ) +from datahub.metadata.urns import SchemaFieldUrn, TagUrn, Urn from datahub.specific.custom_properties import CustomPropertiesPatchHelper from datahub.specific.ownership import OwnershipPatchHelper -from datahub.utilities.urns.tag_urn import TagUrn -from datahub.utilities.urns.urn import Urn class DataJobPatchBuilder(MetadataPatchProposal): @@ -45,43 +42,6 @@ def __init__( ) self.ownership_patch_helper = OwnershipPatchHelper(self) - def _mint_auditstamp(self, message: Optional[str] = None) -> AuditStampClass: - """ - Creates an AuditStampClass instance with the current timestamp and other default values. - - Args: - message: The message associated with the audit stamp (optional). - - Returns: - An instance of AuditStampClass. - """ - return AuditStampClass( - time=int(time.time() * 1000.0), - actor="urn:li:corpuser:datahub", - message=message, - ) - - def _ensure_urn_type( - self, entity_type: str, edges: List[Edge], context: str - ) -> None: - """ - Ensures that the destination URNs in the given edges have the specified entity type. - - Args: - entity_type: The entity type to check against. - edges: A list of Edge objects. - context: The context or description of the operation. - - Raises: - ValueError: If any of the destination URNs is not of the specified entity type. - """ - for e in edges: - urn = Urn.create_from_string(e.destinationUrn) - if not urn.get_type() == entity_type: - raise ValueError( - f"{context}: {e.destinationUrn} is not of type {entity_type}" - ) - def add_owner(self, owner: Owner) -> "DataJobPatchBuilder": """ Adds an owner to the DataJobPatchBuilder. @@ -392,9 +352,7 @@ def add_input_dataset_field(self, input: Union[Urn, str]) -> "DataJobPatchBuilde ValueError: If the input is not a Schema Field urn. """ input_urn = str(input) - urn = Urn.create_from_string(input_urn) - if not urn.get_type() == "schemaField": - raise ValueError(f"Input {input} is not a Schema Field urn") + assert SchemaFieldUrn.from_string(input_urn) self._add_patch( DataJobInputOutput.ASPECT_NAME, @@ -466,9 +424,7 @@ def add_output_dataset_field( ValueError: If the output is not a Schema Field urn. """ output_urn = str(output) - urn = Urn.create_from_string(output_urn) - if not urn.get_type() == "schemaField": - raise ValueError(f"Input {output} is not a Schema Field urn") + assert SchemaFieldUrn.from_string(output_urn) self._add_patch( DataJobInputOutput.ASPECT_NAME, diff --git a/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py b/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py index 1b50d4b2fe810..7dadd16fb7f1c 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py +++ b/metadata-ingestion/src/datahub/utilities/urns/_urn_base.py @@ -200,7 +200,7 @@ def get_entity_id_as_string(self) -> str: @classmethod @deprecated(reason="no longer needed") def validate(cls, urn_str: str) -> None: - Urn.create_from_string(urn_str) + Urn.from_string(urn_str) @staticmethod def url_encode(urn: str) -> str: diff --git a/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py b/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py index 6774978c7a76d..748d5519f1477 100644 --- a/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py +++ b/metadata-ingestion/src/datahub/utilities/urns/structured_properties_urn.py @@ -4,4 +4,4 @@ def make_structured_property_urn(structured_property_id: str) -> str: - return str(StructuredPropertyUrn.create_from_string(structured_property_id)) + return str(StructuredPropertyUrn.from_string(structured_property_id)) diff --git a/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json b/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json index df89255c30048..3a8fce62f4bb3 100644 --- a/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json +++ b/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json @@ -15,7 +15,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -31,7 +31,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -49,7 +49,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -65,7 +65,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -85,7 +85,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -105,7 +105,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -121,7 +121,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -139,7 +139,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -155,7 +155,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -175,7 +175,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -195,7 +195,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -211,7 +211,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -229,7 +229,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -245,7 +245,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -265,7 +265,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -285,7 +285,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -301,7 +301,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -319,7 +319,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -335,7 +335,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -355,7 +355,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -375,7 +375,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -391,7 +391,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -409,7 +409,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -425,7 +425,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -445,7 +445,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -465,7 +465,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -481,7 +481,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -497,7 +497,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -515,7 +515,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -531,7 +531,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -555,7 +555,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -575,7 +575,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -591,7 +591,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -607,7 +607,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -625,7 +625,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -641,7 +641,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -665,7 +665,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -685,7 +685,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -701,7 +701,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -717,7 +717,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -735,7 +735,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -751,7 +751,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -775,7 +775,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -795,7 +795,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -811,7 +811,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -827,7 +827,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -845,7 +845,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -861,7 +861,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -885,7 +885,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -905,7 +905,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -921,7 +921,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -937,7 +937,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -955,7 +955,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -971,7 +971,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -995,7 +995,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1015,7 +1015,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1031,7 +1031,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1047,7 +1047,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1065,7 +1065,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1081,7 +1081,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1105,7 +1105,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1125,7 +1125,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1141,7 +1141,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1157,7 +1157,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1175,7 +1175,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1191,7 +1191,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1215,7 +1215,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1235,7 +1235,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1251,7 +1251,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1267,7 +1267,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1285,7 +1285,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1301,7 +1301,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1325,7 +1325,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1345,7 +1345,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1361,7 +1361,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1377,7 +1377,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1395,7 +1395,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1411,7 +1411,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1435,7 +1435,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1455,7 +1455,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1471,7 +1471,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1487,7 +1487,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1505,7 +1505,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1521,7 +1521,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1549,7 +1549,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1569,7 +1569,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1585,7 +1585,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1601,7 +1601,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1619,7 +1619,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1635,7 +1635,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1663,7 +1663,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1683,7 +1683,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1699,7 +1699,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1715,7 +1715,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1733,7 +1733,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1749,7 +1749,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1781,7 +1781,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1801,7 +1801,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1817,7 +1817,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1833,7 +1833,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1851,7 +1851,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1867,7 +1867,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1903,7 +1903,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1927,7 +1927,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1945,7 +1945,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1961,7 +1961,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1977,7 +1977,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -1995,7 +1995,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2025,7 +2025,7 @@ }, "fields": [ { - "fieldPath": "A", + "fieldPath": "F", "nullable": true, "type": { "type": { @@ -2037,7 +2037,7 @@ "isPartOfKey": false }, { - "fieldPath": "I", + "fieldPath": "G", "nullable": true, "type": { "type": { @@ -2061,7 +2061,7 @@ "isPartOfKey": false }, { - "fieldPath": "G", + "fieldPath": "I", "nullable": true, "type": { "type": { @@ -2073,7 +2073,7 @@ "isPartOfKey": false }, { - "fieldPath": "F", + "fieldPath": "A", "nullable": true, "type": { "type": { @@ -2085,7 +2085,7 @@ "isPartOfKey": false }, { - "fieldPath": "E", + "fieldPath": "B", "nullable": true, "type": { "type": { @@ -2097,7 +2097,7 @@ "isPartOfKey": false }, { - "fieldPath": "D", + "fieldPath": "C", "nullable": true, "type": { "type": { @@ -2109,7 +2109,7 @@ "isPartOfKey": false }, { - "fieldPath": "C", + "fieldPath": "D", "nullable": true, "type": { "type": { @@ -2121,7 +2121,7 @@ "isPartOfKey": false }, { - "fieldPath": "B", + "fieldPath": "E", "nullable": true, "type": { "type": { @@ -2137,7 +2137,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2153,7 +2153,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2177,7 +2177,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2201,7 +2201,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2219,7 +2219,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2235,7 +2235,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2251,7 +2251,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2269,7 +2269,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2299,31 +2299,31 @@ }, "fields": [ { - "fieldPath": "id", + "fieldPath": "email_address", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "integer(32)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "company", + "fieldPath": "priority", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "float(24)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "last_name", + "fieldPath": "company", "nullable": true, "type": { "type": { @@ -2335,31 +2335,31 @@ "isPartOfKey": false }, { - "fieldPath": "first_name", + "fieldPath": "id", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "integer(32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "priority", + "fieldPath": "first_name", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "float(24)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "email_address", + "fieldPath": "last_name", "nullable": true, "type": { "type": { @@ -2375,7 +2375,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2391,7 +2391,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2419,7 +2419,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2443,7 +2443,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2461,7 +2461,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2477,7 +2477,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2493,7 +2493,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2511,7 +2511,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2541,7 +2541,7 @@ }, "fields": [ { - "fieldPath": "urn", + "fieldPath": "createdby", "nullable": true, "type": { "type": { @@ -2553,7 +2553,7 @@ "isPartOfKey": false }, { - "fieldPath": "aspect", + "fieldPath": "createdfor", "nullable": true, "type": { "type": { @@ -2565,19 +2565,19 @@ "isPartOfKey": false }, { - "fieldPath": "version", + "fieldPath": "urn", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "metadata", + "fieldPath": "aspect", "nullable": true, "type": { "type": { @@ -2589,19 +2589,19 @@ "isPartOfKey": false }, { - "fieldPath": "createdon", + "fieldPath": "version", "nullable": true, "type": { "type": { - "com.linkedin.schema.DateType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "timestamp(23)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "createdby", + "fieldPath": "metadata", "nullable": true, "type": { "type": { @@ -2613,14 +2613,14 @@ "isPartOfKey": false }, { - "fieldPath": "createdfor", + "fieldPath": "createdon", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.DateType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "timestamp(23)", "recursive": false, "isPartOfKey": false } @@ -2629,7 +2629,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2645,7 +2645,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2673,7 +2673,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2697,7 +2697,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2715,7 +2715,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2731,7 +2731,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2747,7 +2747,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2765,7 +2765,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2795,19 +2795,19 @@ }, "fields": [ { - "fieldPath": "path", + "fieldPath": "id", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "aspect", + "fieldPath": "urn", "nullable": true, "type": { "type": { @@ -2819,26 +2819,26 @@ "isPartOfKey": false }, { - "fieldPath": "urn", + "fieldPath": "longVal", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "id", + "fieldPath": "stringVal", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, @@ -2855,7 +2855,7 @@ "isPartOfKey": false }, { - "fieldPath": "stringVal", + "fieldPath": "path", "nullable": true, "type": { "type": { @@ -2867,14 +2867,14 @@ "isPartOfKey": false }, { - "fieldPath": "longVal", + "fieldPath": "aspect", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false } @@ -2883,7 +2883,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2899,7 +2899,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2927,7 +2927,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2951,7 +2951,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2969,7 +2969,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -2985,7 +2985,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3001,7 +3001,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3019,7 +3019,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3049,19 +3049,19 @@ }, "fields": [ { - "fieldPath": "doubleVal", + "fieldPath": "id", "nullable": true, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "double(53)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "path", + "fieldPath": "urn", "nullable": true, "type": { "type": { @@ -3073,7 +3073,7 @@ "isPartOfKey": false }, { - "fieldPath": "urn", + "fieldPath": "path", "nullable": true, "type": { "type": { @@ -3085,14 +3085,14 @@ "isPartOfKey": false }, { - "fieldPath": "id", + "fieldPath": "doubleVal", "nullable": true, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "double(53)", "recursive": false, "isPartOfKey": false } @@ -3101,7 +3101,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3117,7 +3117,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3145,7 +3145,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3169,7 +3169,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3187,7 +3187,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3203,7 +3203,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3219,7 +3219,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3237,7 +3237,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3267,7 +3267,7 @@ }, "fields": [ { - "fieldPath": "id", + "fieldPath": "customer_id", "nullable": true, "type": { "type": { @@ -3291,7 +3291,7 @@ "isPartOfKey": false }, { - "fieldPath": "customer_id", + "fieldPath": "id", "nullable": true, "type": { "type": { @@ -3307,7 +3307,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3323,7 +3323,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3351,7 +3351,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3375,7 +3375,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3393,7 +3393,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3409,7 +3409,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3425,7 +3425,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3443,7 +3443,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3473,7 +3473,7 @@ }, "fields": [ { - "fieldPath": "id", + "fieldPath": "salary", "nullable": true, "type": { "type": { @@ -3485,31 +3485,31 @@ "isPartOfKey": false }, { - "fieldPath": "name", + "fieldPath": "age", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "age", + "fieldPath": "name", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "salary", + "fieldPath": "id", "nullable": true, "type": { "type": { @@ -3525,7 +3525,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3541,7 +3541,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3569,7 +3569,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3593,7 +3593,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3611,7 +3611,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3627,7 +3627,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3643,7 +3643,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3725,7 +3725,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3741,7 +3741,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3766,7 +3766,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3794,7 +3794,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3818,7 +3818,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3836,7 +3836,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3852,7 +3852,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3868,7 +3868,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -3910,7 +3910,7 @@ "isPartOfKey": false }, { - "fieldPath": "aspect", + "fieldPath": "createdfor", "nullable": true, "type": { "type": { @@ -3922,55 +3922,55 @@ "isPartOfKey": false }, { - "fieldPath": "version", + "fieldPath": "createdby", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "metadata", + "fieldPath": "createdon", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.DateType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "timestamp(23)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "createdon", + "fieldPath": "metadata", "nullable": true, "type": { "type": { - "com.linkedin.schema.DateType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "timestamp(23)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "createdby", + "fieldPath": "version", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "createdfor", + "fieldPath": "aspect", "nullable": true, "type": { "type": { @@ -3986,7 +3986,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4002,7 +4002,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4027,7 +4027,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4055,7 +4055,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4079,7 +4079,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4097,7 +4097,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4113,7 +4113,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4129,7 +4129,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4247,7 +4247,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4263,7 +4263,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4288,7 +4288,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4316,7 +4316,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4340,7 +4340,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4358,7 +4358,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4374,7 +4374,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4390,7 +4390,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4432,38 +4432,38 @@ "isPartOfKey": false }, { - "fieldPath": "doubleVal", + "fieldPath": "id", "nullable": true, "type": { "type": { "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "double(53)", + "nativeDataType": "bigint(64)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "id", + "fieldPath": "urn", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "bigint(64)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "urn", + "fieldPath": "doubleVal", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "double(53)", "recursive": false, "isPartOfKey": false } @@ -4472,7 +4472,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4488,7 +4488,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4513,7 +4513,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4541,7 +4541,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4565,7 +4565,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4583,7 +4583,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4599,7 +4599,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4615,7 +4615,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4721,7 +4721,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4737,7 +4737,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4762,7 +4762,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4790,7 +4790,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4814,7 +4814,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4832,7 +4832,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4848,7 +4848,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4864,7 +4864,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4894,7 +4894,7 @@ }, "fields": [ { - "fieldPath": "id", + "fieldPath": "customer_id", "nullable": true, "type": { "type": { @@ -4906,26 +4906,26 @@ "isPartOfKey": false }, { - "fieldPath": "description", + "fieldPath": "id", "nullable": true, "type": { "type": { - "com.linkedin.schema.StringType": {} + "com.linkedin.schema.NumberType": {} } }, - "nativeDataType": "character varying(65536)", + "nativeDataType": "integer(32)", "recursive": false, "isPartOfKey": false }, { - "fieldPath": "customer_id", + "fieldPath": "description", "nullable": true, "type": { "type": { - "com.linkedin.schema.NumberType": {} + "com.linkedin.schema.StringType": {} } }, - "nativeDataType": "integer(32)", + "nativeDataType": "character varying(65536)", "recursive": false, "isPartOfKey": false } @@ -4934,7 +4934,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4950,7 +4950,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -4975,7 +4975,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5003,7 +5003,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5027,7 +5027,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5045,7 +5045,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5061,7 +5061,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5077,7 +5077,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5107,7 +5107,7 @@ }, "fields": [ { - "fieldPath": "B", + "fieldPath": "F", "nullable": true, "type": { "type": { @@ -5119,7 +5119,7 @@ "isPartOfKey": false }, { - "fieldPath": "C", + "fieldPath": "G", "nullable": true, "type": { "type": { @@ -5131,7 +5131,7 @@ "isPartOfKey": false }, { - "fieldPath": "D", + "fieldPath": "H", "nullable": true, "type": { "type": { @@ -5143,7 +5143,7 @@ "isPartOfKey": false }, { - "fieldPath": "E", + "fieldPath": "I", "nullable": true, "type": { "type": { @@ -5155,7 +5155,7 @@ "isPartOfKey": false }, { - "fieldPath": "F", + "fieldPath": "A", "nullable": true, "type": { "type": { @@ -5167,7 +5167,7 @@ "isPartOfKey": false }, { - "fieldPath": "G", + "fieldPath": "B", "nullable": true, "type": { "type": { @@ -5179,7 +5179,7 @@ "isPartOfKey": false }, { - "fieldPath": "H", + "fieldPath": "C", "nullable": true, "type": { "type": { @@ -5191,7 +5191,7 @@ "isPartOfKey": false }, { - "fieldPath": "I", + "fieldPath": "D", "nullable": true, "type": { "type": { @@ -5203,7 +5203,7 @@ "isPartOfKey": false }, { - "fieldPath": "A", + "fieldPath": "E", "nullable": true, "type": { "type": { @@ -5219,7 +5219,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5235,7 +5235,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5260,7 +5260,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5288,7 +5288,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5312,7 +5312,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5330,7 +5330,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5346,7 +5346,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5362,7 +5362,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5392,7 +5392,7 @@ }, "fields": [ { - "fieldPath": "I", + "fieldPath": "A", "nullable": true, "type": { "type": { @@ -5404,7 +5404,7 @@ "isPartOfKey": false }, { - "fieldPath": "A", + "fieldPath": "B", "nullable": true, "type": { "type": { @@ -5416,7 +5416,7 @@ "isPartOfKey": false }, { - "fieldPath": "B", + "fieldPath": "C", "nullable": true, "type": { "type": { @@ -5428,7 +5428,7 @@ "isPartOfKey": false }, { - "fieldPath": "C", + "fieldPath": "D", "nullable": true, "type": { "type": { @@ -5440,7 +5440,7 @@ "isPartOfKey": false }, { - "fieldPath": "D", + "fieldPath": "E", "nullable": true, "type": { "type": { @@ -5452,7 +5452,7 @@ "isPartOfKey": false }, { - "fieldPath": "E", + "fieldPath": "F", "nullable": true, "type": { "type": { @@ -5464,7 +5464,7 @@ "isPartOfKey": false }, { - "fieldPath": "F", + "fieldPath": "G", "nullable": true, "type": { "type": { @@ -5476,7 +5476,7 @@ "isPartOfKey": false }, { - "fieldPath": "G", + "fieldPath": "H", "nullable": true, "type": { "type": { @@ -5488,7 +5488,7 @@ "isPartOfKey": false }, { - "fieldPath": "H", + "fieldPath": "I", "nullable": true, "type": { "type": { @@ -5552,7 +5552,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5568,7 +5568,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5593,7 +5593,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5625,7 +5625,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5649,7 +5649,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5667,7 +5667,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5683,7 +5683,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5699,7 +5699,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5781,7 +5781,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5797,7 +5797,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5822,7 +5822,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5854,7 +5854,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5878,7 +5878,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5896,7 +5896,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5912,7 +5912,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -5928,7 +5928,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6070,7 +6070,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6086,7 +6086,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6111,7 +6111,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6151,7 +6151,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6175,12 +6175,91 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),urn)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),urn)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdfor)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdfor)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdby)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdby)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdon)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdon)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),metadata)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),metadata)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),version)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),version)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),aspect)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),aspect)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6204,12 +6283,91 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),doubleVal)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),doubleVal)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),urn)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),urn)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),aspect)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),aspect)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),path)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),path)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),longVal)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),longVal)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),stringVal)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),stringVal)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6233,12 +6391,58 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),path)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),path)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),urn)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),urn)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),doubleVal)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),doubleVal)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6262,12 +6466,80 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),company)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),company)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),last_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),email_address)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),email_address)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),priority)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),priority)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6291,12 +6563,47 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),description)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),description)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6320,12 +6627,58 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),age)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),age)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),salary)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),salary)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6349,12 +6702,157 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),A)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),A)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),B)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),B)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),C)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),C)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),D)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),D)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),E)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),E)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),F)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),F)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),G)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),G)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),H)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),H)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),I)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),I)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),J)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),J)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),K)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),K)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),L)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),L)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),M)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),M)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6378,12 +6876,58 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_NAME)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_NAME)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),MANAGER_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),MANAGER_ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_ID)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),LOCATION_ID)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),LOCATION_ID)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6407,12 +6951,113 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),F)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),F)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),G)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),G)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),H)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),H)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),I)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),I)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),A)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),A)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),B)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),B)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),C)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),C)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),D)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),D)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),E)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),E)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -6436,18 +7081,119 @@ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)", "type": "COPY" } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_start_date_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_start_date_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_end_date_sk)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_end_date_sk)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_department)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_department)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_number)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_number)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_number)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_description)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_description)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_type)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_type)" + ], + "confidenceScore": 1.0 + } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6457,41 +7203,52 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 5, - "columnCount": 6, + "rowCount": 3834, + "columnCount": 9, "fieldProfiles": [ { - "fieldPath": "id", - "uniqueCount": 5, - "nullCount": 0, - "mean": "3.0", - "stdev": "1.5811388300841898" + "fieldPath": "F", + "uniqueCount": 61, + "nullCount": 0 }, { - "fieldPath": "company", - "uniqueCount": 5, + "fieldPath": "G", + "uniqueCount": 40, "nullCount": 0 }, { - "fieldPath": "last_name", - "uniqueCount": 5, + "fieldPath": "H", + "uniqueCount": 91, "nullCount": 0 }, { - "fieldPath": "first_name", - "uniqueCount": 5, + "fieldPath": "I", + "uniqueCount": 85, "nullCount": 0 }, { - "fieldPath": "priority", - "uniqueCount": 3, - "nullCount": 1, - "mean": "4.175000011920929", - "stdev": "0.4924429489953036" + "fieldPath": "A", + "uniqueCount": 2, + "nullCount": 0 }, { - "fieldPath": "email_address", - "uniqueCount": 5, + "fieldPath": "B", + "uniqueCount": 2, + "nullCount": 0 + }, + { + "fieldPath": "C", + "uniqueCount": 3834, + "nullCount": 0 + }, + { + "fieldPath": "D", + "uniqueCount": 76, + "nullCount": 0 + }, + { + "fieldPath": "E", + "uniqueCount": 192, "nullCount": 0 } ] @@ -6499,13 +7256,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6516,7 +7273,7 @@ "type": "FULL_TABLE" }, "rowCount": 0, - "columnCount": 7, + "columnCount": 4, "fieldProfiles": [ { "fieldPath": "path", @@ -6524,7 +7281,7 @@ "nullCount": 0 }, { - "fieldPath": "aspect", + "fieldPath": "id", "uniqueCount": 0, "nullCount": 0 }, @@ -6533,38 +7290,23 @@ "uniqueCount": 0, "nullCount": 0 }, - { - "fieldPath": "id", - "uniqueCount": 0, - "nullCount": 0 - }, { "fieldPath": "doubleVal", "uniqueCount": 0, "nullCount": 0 - }, - { - "fieldPath": "stringVal", - "uniqueCount": 0, - "nullCount": 0 - }, - { - "fieldPath": "longVal", - "uniqueCount": 0, - "nullCount": 0 } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6575,20 +7317,40 @@ "type": "FULL_TABLE" }, "rowCount": 0, - "columnCount": 3, + "columnCount": 7, "fieldProfiles": [ + { + "fieldPath": "doubleVal", + "uniqueCount": 0, + "nullCount": 0 + }, { "fieldPath": "id", "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "description", + "fieldPath": "urn", "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "customer_id", + "fieldPath": "aspect", + "uniqueCount": 0, + "nullCount": 0 + }, + { + "fieldPath": "path", + "uniqueCount": 0, + "nullCount": 0 + }, + { + "fieldPath": "longVal", + "uniqueCount": 0, + "nullCount": 0 + }, + { + "fieldPath": "stringVal", "uniqueCount": 0, "nullCount": 0 } @@ -6597,13 +7359,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6617,17 +7379,17 @@ "columnCount": 3, "fieldProfiles": [ { - "fieldPath": "id", + "fieldPath": "customer_id", "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "description", + "fieldPath": "id", "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "customer_id", + "fieldPath": "description", "uniqueCount": 0, "nullCount": 0 } @@ -6636,13 +7398,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6656,16 +7418,11 @@ "columnCount": 4, "fieldProfiles": [ { - "fieldPath": "id", + "fieldPath": "salary", "uniqueCount": 4, "nullCount": 0, - "mean": "2.5", - "stdev": "1.2909944487358056" - }, - { - "fieldPath": "name", - "uniqueCount": 4, - "nullCount": 0 + "mean": "65000.0", + "stdev": "12909.944487358056" }, { "fieldPath": "age", @@ -6675,24 +7432,29 @@ "stdev": "6.454972243679028" }, { - "fieldPath": "salary", + "fieldPath": "name", + "uniqueCount": 4, + "nullCount": 0 + }, + { + "fieldPath": "id", "uniqueCount": 4, "nullCount": 0, - "mean": "65000.0", - "stdev": "12909.944487358056" + "mean": "2.5", + "stdev": "1.2909944487358056" } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6702,52 +7464,27 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 3834, - "columnCount": 9, + "rowCount": 0, + "columnCount": 4, "fieldProfiles": [ { - "fieldPath": "A", - "uniqueCount": 2, - "nullCount": 0 - }, - { - "fieldPath": "I", - "uniqueCount": 85, - "nullCount": 0 - }, - { - "fieldPath": "H", - "uniqueCount": 91, - "nullCount": 0 - }, - { - "fieldPath": "G", - "uniqueCount": 40, - "nullCount": 0 - }, - { - "fieldPath": "F", - "uniqueCount": 61, - "nullCount": 0 - }, - { - "fieldPath": "E", - "uniqueCount": 192, + "fieldPath": "id", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "D", - "uniqueCount": 76, + "fieldPath": "urn", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "C", - "uniqueCount": 3834, + "fieldPath": "path", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "B", - "uniqueCount": 2, + "fieldPath": "doubleVal", + "uniqueCount": 0, "nullCount": 0 } ] @@ -6755,13 +7492,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6771,27 +7508,41 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 0, - "columnCount": 4, + "rowCount": 5, + "columnCount": 6, "fieldProfiles": [ { - "fieldPath": "path", - "uniqueCount": 0, + "fieldPath": "email_address", + "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "doubleVal", - "uniqueCount": 0, + "fieldPath": "priority", + "uniqueCount": 3, + "nullCount": 1, + "mean": "4.175000011920929", + "stdev": "0.4924429489953036" + }, + { + "fieldPath": "company", + "uniqueCount": 5, "nullCount": 0 }, { "fieldPath": "id", - "uniqueCount": 0, + "uniqueCount": 5, + "nullCount": 0, + "mean": "3.0", + "stdev": "1.5811388300841898" + }, + { + "fieldPath": "first_name", + "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "urn", - "uniqueCount": 0, + "fieldPath": "last_name", + "uniqueCount": 5, "nullCount": 0 } ] @@ -6799,13 +7550,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6815,55 +7566,66 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 5, - "columnCount": 6, + "rowCount": 3834, + "columnCount": 9, "fieldProfiles": [ { - "fieldPath": "id", - "uniqueCount": 5, - "nullCount": 0, - "mean": "3.0", - "stdev": "1.5811388300841898" + "fieldPath": "F", + "uniqueCount": 61, + "nullCount": 0 }, { - "fieldPath": "company", - "uniqueCount": 5, + "fieldPath": "G", + "uniqueCount": 40, "nullCount": 0 }, { - "fieldPath": "last_name", - "uniqueCount": 5, + "fieldPath": "H", + "uniqueCount": 91, "nullCount": 0 }, { - "fieldPath": "first_name", - "uniqueCount": 5, + "fieldPath": "I", + "uniqueCount": 85, "nullCount": 0 }, { - "fieldPath": "email_address", - "uniqueCount": 5, + "fieldPath": "A", + "uniqueCount": 2, "nullCount": 0 }, { - "fieldPath": "priority", - "uniqueCount": 3, - "nullCount": 1, - "mean": "4.175000011920929", - "stdev": "0.4924429489953036" + "fieldPath": "B", + "uniqueCount": 2, + "nullCount": 0 + }, + { + "fieldPath": "C", + "uniqueCount": 3834, + "nullCount": 0 + }, + { + "fieldPath": "D", + "uniqueCount": 76, + "nullCount": 0 + }, + { + "fieldPath": "E", + "uniqueCount": 192, + "nullCount": 0 } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6873,42 +7635,42 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 0, + "rowCount": 2, "columnCount": 7, "fieldProfiles": [ { - "fieldPath": "doubleVal", - "uniqueCount": 0, + "fieldPath": "createdby", + "uniqueCount": 1, "nullCount": 0 }, { - "fieldPath": "id", + "fieldPath": "createdfor", "uniqueCount": 0, - "nullCount": 0 + "nullCount": 2 }, { "fieldPath": "urn", - "uniqueCount": 0, + "uniqueCount": 1, "nullCount": 0 }, { "fieldPath": "aspect", - "uniqueCount": 0, + "uniqueCount": 2, "nullCount": 0 }, { - "fieldPath": "path", - "uniqueCount": 0, + "fieldPath": "version", + "uniqueCount": 1, "nullCount": 0 }, { - "fieldPath": "longVal", - "uniqueCount": 0, + "fieldPath": "metadata", + "uniqueCount": 2, "nullCount": 0 }, { - "fieldPath": "stringVal", - "uniqueCount": 0, + "fieldPath": "createdon", + "uniqueCount": 1, "nullCount": 0 } ] @@ -6916,13 +7678,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6932,47 +7694,56 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 4, - "columnCount": 4, + "rowCount": 2, + "columnCount": 7, "fieldProfiles": [ { - "fieldPath": "id", - "uniqueCount": 4, - "nullCount": 0, - "mean": "2.5", - "stdev": "1.2909944487358056" + "fieldPath": "urn", + "uniqueCount": 1, + "nullCount": 0 }, { - "fieldPath": "name", - "uniqueCount": 4, + "fieldPath": "createdfor", + "uniqueCount": 0, + "nullCount": 2 + }, + { + "fieldPath": "createdby", + "uniqueCount": 1, "nullCount": 0 }, { - "fieldPath": "age", - "uniqueCount": 4, - "nullCount": 0, - "mean": "32.5", - "stdev": "6.454972243679028" + "fieldPath": "createdon", + "uniqueCount": 1, + "nullCount": 0 }, { - "fieldPath": "salary", - "uniqueCount": 4, - "nullCount": 0, - "mean": "65000.0", - "stdev": "12909.944487358056" + "fieldPath": "metadata", + "uniqueCount": 2, + "nullCount": 0 + }, + { + "fieldPath": "version", + "uniqueCount": 1, + "nullCount": 0 + }, + { + "fieldPath": "aspect", + "uniqueCount": 2, + "nullCount": 0 } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -6982,86 +7753,47 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 10842, - "columnCount": 13, + "rowCount": 4, + "columnCount": 4, "fieldProfiles": [ { - "fieldPath": "I", - "uniqueCount": 8, - "nullCount": 0 - }, - { - "fieldPath": "A", - "uniqueCount": 9661, - "nullCount": 0 - }, - { - "fieldPath": "B", - "uniqueCount": 35, - "nullCount": 0 - }, - { - "fieldPath": "C", - "uniqueCount": 42, - "nullCount": 0 - }, - { - "fieldPath": "D", - "uniqueCount": 6003, - "nullCount": 0 - }, - { - "fieldPath": "E", - "uniqueCount": 463, - "nullCount": 0 - }, - { - "fieldPath": "F", - "uniqueCount": 23, - "nullCount": 0 - }, - { - "fieldPath": "G", - "uniqueCount": 5, - "nullCount": 0 - }, - { - "fieldPath": "H", - "uniqueCount": 94, - "nullCount": 0 - }, - { - "fieldPath": "J", - "uniqueCount": 121, - "nullCount": 0 + "fieldPath": "id", + "uniqueCount": 4, + "nullCount": 0, + "mean": "2.5", + "stdev": "1.2909944487358056" }, { - "fieldPath": "K", - "uniqueCount": 1379, + "fieldPath": "name", + "uniqueCount": 4, "nullCount": 0 }, { - "fieldPath": "L", - "uniqueCount": 2835, - "nullCount": 0 + "fieldPath": "age", + "uniqueCount": 4, + "nullCount": 0, + "mean": "32.5", + "stdev": "6.454972243679028" }, { - "fieldPath": "M", - "uniqueCount": 35, - "nullCount": 1 + "fieldPath": "salary", + "uniqueCount": 4, + "nullCount": 0, + "mean": "65000.0", + "stdev": "12909.944487358056" } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7072,20 +7804,15 @@ "type": "FULL_TABLE" }, "rowCount": 0, - "columnCount": 4, + "columnCount": 3, "fieldProfiles": [ { - "fieldPath": "doubleVal", - "uniqueCount": 0, - "nullCount": 0 - }, - { - "fieldPath": "path", + "fieldPath": "customer_id", "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "urn", + "fieldPath": "description", "uniqueCount": 0, "nullCount": 0 }, @@ -7099,13 +7826,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7115,52 +7842,42 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 3834, - "columnCount": 9, + "rowCount": 0, + "columnCount": 7, "fieldProfiles": [ { - "fieldPath": "B", - "uniqueCount": 2, - "nullCount": 0 - }, - { - "fieldPath": "C", - "uniqueCount": 3834, - "nullCount": 0 - }, - { - "fieldPath": "D", - "uniqueCount": 76, + "fieldPath": "id", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "E", - "uniqueCount": 192, + "fieldPath": "urn", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "F", - "uniqueCount": 61, + "fieldPath": "longVal", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "G", - "uniqueCount": 40, + "fieldPath": "stringVal", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "H", - "uniqueCount": 91, + "fieldPath": "doubleVal", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "I", - "uniqueCount": 85, + "fieldPath": "path", + "uniqueCount": 0, "nullCount": 0 }, { - "fieldPath": "A", - "uniqueCount": 2, + "fieldPath": "aspect", + "uniqueCount": 0, "nullCount": 0 } ] @@ -7168,13 +7885,13 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7184,56 +7901,55 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 2, - "columnCount": 7, + "rowCount": 5, + "columnCount": 6, "fieldProfiles": [ { - "fieldPath": "urn", - "uniqueCount": 1, - "nullCount": 0 - }, - { - "fieldPath": "aspect", - "uniqueCount": 2, - "nullCount": 0 + "fieldPath": "id", + "uniqueCount": 5, + "nullCount": 0, + "mean": "3.0", + "stdev": "1.5811388300841898" }, { - "fieldPath": "version", - "uniqueCount": 1, + "fieldPath": "company", + "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "metadata", - "uniqueCount": 2, + "fieldPath": "last_name", + "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "createdon", - "uniqueCount": 1, + "fieldPath": "first_name", + "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "createdby", - "uniqueCount": 1, + "fieldPath": "email_address", + "uniqueCount": 5, "nullCount": 0 }, { - "fieldPath": "createdfor", - "uniqueCount": 0, - "nullCount": 2 + "fieldPath": "priority", + "uniqueCount": 3, + "nullCount": 1, + "mean": "4.175000011920929", + "stdev": "0.4924429489953036" } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -7243,50 +7959,80 @@ "partition": "FULL_TABLE_SNAPSHOT", "type": "FULL_TABLE" }, - "rowCount": 2, - "columnCount": 7, + "rowCount": 10842, + "columnCount": 13, "fieldProfiles": [ { - "fieldPath": "urn", - "uniqueCount": 1, + "fieldPath": "A", + "uniqueCount": 9661, "nullCount": 0 }, { - "fieldPath": "aspect", - "uniqueCount": 2, + "fieldPath": "B", + "uniqueCount": 35, "nullCount": 0 }, { - "fieldPath": "version", - "uniqueCount": 1, + "fieldPath": "C", + "uniqueCount": 42, "nullCount": 0 }, { - "fieldPath": "metadata", - "uniqueCount": 2, + "fieldPath": "D", + "uniqueCount": 6003, "nullCount": 0 }, { - "fieldPath": "createdon", - "uniqueCount": 1, + "fieldPath": "E", + "uniqueCount": 463, "nullCount": 0 }, { - "fieldPath": "createdby", - "uniqueCount": 1, + "fieldPath": "F", + "uniqueCount": 23, "nullCount": 0 }, { - "fieldPath": "createdfor", - "uniqueCount": 0, - "nullCount": 2 + "fieldPath": "G", + "uniqueCount": 5, + "nullCount": 0 + }, + { + "fieldPath": "H", + "uniqueCount": 94, + "nullCount": 0 + }, + { + "fieldPath": "I", + "uniqueCount": 8, + "nullCount": 0 + }, + { + "fieldPath": "J", + "uniqueCount": 121, + "nullCount": 0 + }, + { + "fieldPath": "K", + "uniqueCount": 1379, + "nullCount": 0 + }, + { + "fieldPath": "L", + "uniqueCount": 2835, + "nullCount": 0 + }, + { + "fieldPath": "M", + "uniqueCount": 35, + "nullCount": 1 } ] } }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -7336,7 +8082,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } }, @@ -7415,7 +8161,7 @@ }, "systemMetadata": { "lastObserved": 1697353200000, - "runId": "dremio-2023_10_15-07_00_00-7c7cnk", + "runId": "dremio-2023_10_15-07_00_00-bo12f3", "lastRunId": "no-run-id-provided" } } diff --git a/metadata-ingestion/tests/test_helpers/mce_helpers.py b/metadata-ingestion/tests/test_helpers/mce_helpers.py index f4c629df7dba4..0105e6d596970 100644 --- a/metadata-ingestion/tests/test_helpers/mce_helpers.py +++ b/metadata-ingestion/tests/test_helpers/mce_helpers.py @@ -323,7 +323,7 @@ def assert_entity_mce_aspect( ) -> int: # TODO: Replace with read_metadata_file() test_output = load_json_file(file) - entity_type = Urn.create_from_string(entity_urn).get_type() + entity_type = Urn.from_string(entity_urn).get_type() assert isinstance(test_output, list) # mce urns mces: List[MetadataChangeEventClass] = [ @@ -346,7 +346,7 @@ def assert_entity_mcp_aspect( ) -> int: # TODO: Replace with read_metadata_file() test_output = load_json_file(file) - entity_type = Urn.create_from_string(entity_urn).get_type() + entity_type = Urn.from_string(entity_urn).get_type() assert isinstance(test_output, list) # mcps that match entity_urn mcps: List[MetadataChangeProposalWrapper] = [ diff --git a/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py b/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py index 18b0d9fd40041..52d7aa7f509c9 100644 --- a/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py +++ b/metadata-ingestion/tests/unit/test_generic_aspect_transformer.py @@ -51,7 +51,7 @@ def make_mcpw( ) -> MetadataChangeProposalWrapper: return MetadataChangeProposalWrapper( entityUrn=entity_urn, - entityType=Urn.create_from_string(entity_urn).get_type(), + entityType=Urn.from_string(entity_urn).get_type(), aspectName=aspect_name, changeType="UPSERT", aspect=aspect, @@ -65,7 +65,7 @@ def make_mcpc( ) -> MetadataChangeProposalClass: return MetadataChangeProposalClass( entityUrn=entity_urn, - entityType=Urn.create_from_string(entity_urn).get_type(), + entityType=Urn.from_string(entity_urn).get_type(), aspectName=aspect_name, changeType="UPSERT", aspect=aspect, diff --git a/metadata-ingestion/tests/unit/test_transform_dataset.py b/metadata-ingestion/tests/unit/test_transform_dataset.py index 389f7b70b3311..5151be9c8b199 100644 --- a/metadata-ingestion/tests/unit/test_transform_dataset.py +++ b/metadata-ingestion/tests/unit/test_transform_dataset.py @@ -122,7 +122,7 @@ def make_generic_dataset_mcp( ) -> MetadataChangeProposalWrapper: return MetadataChangeProposalWrapper( entityUrn=entity_urn, - entityType=Urn.create_from_string(entity_urn).get_type(), + entityType=Urn.from_string(entity_urn).get_type(), aspectName=aspect_name, changeType="UPSERT", aspect=aspect, @@ -138,7 +138,7 @@ def make_generic_container_mcp( aspect = models.StatusClass(removed=False) return MetadataChangeProposalWrapper( entityUrn=entity_urn, - entityType=Urn.create_from_string(entity_urn).get_type(), + entityType=Urn.from_string(entity_urn).get_type(), aspectName=aspect_name, changeType="UPSERT", aspect=aspect, diff --git a/metadata-integration/java/acryl-spark-lineage/README.md b/metadata-integration/java/acryl-spark-lineage/README.md index 267e979b0fa07..97851e90e860e 100644 --- a/metadata-integration/java/acryl-spark-lineage/README.md +++ b/metadata-integration/java/acryl-spark-lineage/README.md @@ -125,9 +125,10 @@ information like tokens. configurations under `Spark Config`. ```text - spark.extraListeners datahub.spark.DatahubSparkListener - spark.datahub.rest.server http://localhost:8080 - spark.datahub.databricks.cluster cluster-name + spark.extraListeners datahub.spark.DatahubSparkListener + spark.datahub.rest.server http://localhost:8080 + spark.datahub.stage_metadata_coalescing true + spark.datahub.databricks.cluster cluster-name ``` - Click the **Init Scripts** tab. Set cluster init script as `dbfs:/datahub/init.sh`. diff --git a/metadata-integration/java/acryl-spark-lineage/build.gradle b/metadata-integration/java/acryl-spark-lineage/build.gradle index 3f83e5657bbf4..940ebb9848536 100644 --- a/metadata-integration/java/acryl-spark-lineage/build.gradle +++ b/metadata-integration/java/acryl-spark-lineage/build.gradle @@ -6,7 +6,7 @@ apply plugin: 'com.github.johnrengelman.shadow' apply plugin: 'signing' apply plugin: 'io.codearte.nexus-staging' apply plugin: 'maven-publish' -apply plugin: 'jacoco' +apply from: '../../../gradle/coverage/java-coverage.gradle' apply from: '../versioning.gradle' jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation @@ -163,14 +163,10 @@ checkShadowJar { dependsOn shadowJar } -jacocoTestReport { - dependsOn test // tests are required to run before generating the report -} test { forkEvery = 1 useJUnit() - finalizedBy jacocoTestReport } assemble { diff --git a/metadata-integration/java/datahub-client/build.gradle b/metadata-integration/java/datahub-client/build.gradle index 56a486ad04330..87a9623bbf062 100644 --- a/metadata-integration/java/datahub-client/build.gradle +++ b/metadata-integration/java/datahub-client/build.gradle @@ -2,12 +2,12 @@ plugins { id("com.palantir.git-version") apply false id 'java-library' id 'com.github.johnrengelman.shadow' - id 'jacoco' id 'signing' id 'io.codearte.nexus-staging' id 'maven-publish' } +apply from: "../../../gradle/coverage/java-coverage.gradle" apply from: "../versioning.gradle" import org.apache.tools.ant.filters.ReplaceTokens @@ -59,15 +59,11 @@ task copyAvroSchemas { compileJava.dependsOn copyAvroSchemas -jacocoTestReport { - dependsOn test // tests are required to run before generating the report -} test { // to avoid simultaneous executions of tests when complete build is run mustRunAfter(":metadata-io:test") useJUnit() - finalizedBy jacocoTestReport } task checkShadowJar(type: Exec) { diff --git a/metadata-integration/java/datahub-event/build.gradle b/metadata-integration/java/datahub-event/build.gradle index 395065404d1db..24e119c622936 100644 --- a/metadata-integration/java/datahub-event/build.gradle +++ b/metadata-integration/java/datahub-event/build.gradle @@ -2,12 +2,12 @@ plugins { id("com.palantir.git-version") apply false id 'java' id 'com.github.johnrengelman.shadow' - id 'jacoco' id 'signing' id 'io.codearte.nexus-staging' id 'maven-publish' } +apply from: "../../../gradle/coverage/java-coverage.gradle" apply from: "../versioning.gradle" dependencies { @@ -29,10 +29,6 @@ dependencies { testRuntimeOnly externalDependency.logbackClassicJava8 } -jacocoTestReport { - dependsOn test // tests are required to run before generating the report -} - task copyAvroSchemas { dependsOn(':metadata-events:mxe-schemas:renameNamespace') copy { @@ -47,7 +43,6 @@ test { // to avoid simultaneous executions of tests when complete build is run mustRunAfter(":metadata-io:test") useJUnit() - finalizedBy jacocoTestReport } // task sourcesJar(type: Jar) { diff --git a/metadata-integration/java/datahub-protobuf/build.gradle b/metadata-integration/java/datahub-protobuf/build.gradle index 8bc71c5da8f4a..97595fd1345dc 100644 --- a/metadata-integration/java/datahub-protobuf/build.gradle +++ b/metadata-integration/java/datahub-protobuf/build.gradle @@ -3,12 +3,12 @@ plugins { id "application" } apply plugin: 'java' -apply plugin: 'jacoco' apply plugin: 'com.github.johnrengelman.shadow' apply plugin: 'signing' apply plugin: 'io.codearte.nexus-staging' apply plugin: 'maven-publish' apply from: '../versioning.gradle' +apply from: '../../../gradle/coverage/java-coverage.gradle' jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation @@ -57,13 +57,6 @@ task compileProtobuf { } } -jacocoTestReport { - dependsOn test // tests are required to run before generating the report -} - -test.finalizedBy jacocoTestReport - - task checkShadowJar(type: Exec) { commandLine 'sh', '-c', 'scripts/check_jar.sh' } @@ -202,4 +195,4 @@ nexusStaging { password = System.getenv("NEXUS_PASSWORD") } -startScripts.dependsOn shadowJar \ No newline at end of file +startScripts.dependsOn shadowJar diff --git a/metadata-integration/java/openlineage-converter/build.gradle b/metadata-integration/java/openlineage-converter/build.gradle index d149104f089b3..1bf4a3c0fadb6 100644 --- a/metadata-integration/java/openlineage-converter/build.gradle +++ b/metadata-integration/java/openlineage-converter/build.gradle @@ -2,7 +2,7 @@ apply plugin: 'java-library' apply plugin: 'com.github.johnrengelman.shadow' apply plugin: 'signing' apply plugin: 'maven-publish' -apply plugin: 'jacoco' +apply from: '../../../gradle/coverage/java-coverage.gradle' apply from: '../versioning.gradle' repositories { @@ -27,20 +27,16 @@ dependencies { testImplementation externalDependency.testng } -jacocoTestReport { - dependsOn test // tests are required to run before generating the report -} - test { forkEvery = 1 useJUnit() - finalizedBy jacocoTestReport } test { useJUnit() - finalizedBy jacocoTestReport + useTestNG() } + shadowJar { zip64 = true archiveClassifier = '' diff --git a/metadata-integration/java/spark-lineage-legacy/build.gradle b/metadata-integration/java/spark-lineage-legacy/build.gradle index 8db8a09f8cc81..d33290c16b3a5 100644 --- a/metadata-integration/java/spark-lineage-legacy/build.gradle +++ b/metadata-integration/java/spark-lineage-legacy/build.gradle @@ -6,7 +6,6 @@ apply plugin: 'com.github.johnrengelman.shadow' apply plugin: 'signing' apply plugin: 'io.codearte.nexus-staging' apply plugin: 'maven-publish' -apply plugin: 'jacoco' apply from: '../versioning.gradle' jar.enabled = false // Since we only want to build shadow jars, disabling the regular jar creation @@ -123,15 +122,6 @@ shadowJar { finalizedBy checkShadowJar } -checkShadowJar { - dependsOn shadowJar -} - - -jacocoTestReport { - dependsOn test // tests are required to run before generating the report -} - test { forkEvery = 1 useJUnit() @@ -154,7 +144,6 @@ test { ] jvmArgs = sparkJava17CompatibleJvmArgs - finalizedBy jacocoTestReport } assemble { diff --git a/metadata-io/build.gradle b/metadata-io/build.gradle index 41294fab7b24a..516a77d59d50b 100644 --- a/metadata-io/build.gradle +++ b/metadata-io/build.gradle @@ -4,6 +4,8 @@ plugins { id 'io.ebean' version "${ebeanVersion}" // Use the latest version from global build.gradle } +apply from: '../gradle/coverage/java-coverage.gradle' + configurations { enhance } diff --git a/metadata-io/metadata-io-api/build.gradle b/metadata-io/metadata-io-api/build.gradle index 5273177b75281..a49da206a69a8 100644 --- a/metadata-io/metadata-io-api/build.gradle +++ b/metadata-io/metadata-io-api/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java-library' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':entity-registry') implementation project(':metadata-service:services') diff --git a/metadata-jobs/common/build.gradle b/metadata-jobs/common/build.gradle index b0a3a6827b729..120806431b1df 100644 --- a/metadata-jobs/common/build.gradle +++ b/metadata-jobs/common/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java-library' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation(project(':metadata-service:factories')) { exclude group: 'org.neo4j.test' diff --git a/metadata-jobs/mae-consumer-job/build.gradle b/metadata-jobs/mae-consumer-job/build.gradle index 4980f81d61bb4..1fb46582c0381 100644 --- a/metadata-jobs/mae-consumer-job/build.gradle +++ b/metadata-jobs/mae-consumer-job/build.gradle @@ -4,6 +4,7 @@ plugins { id 'com.palantir.docker' } +apply from: '../../gradle/coverage/java-coverage.gradle' apply from: "../../gradle/versioning/versioning.gradle" ext { diff --git a/metadata-jobs/mae-consumer/build.gradle b/metadata-jobs/mae-consumer/build.gradle index 6afc12c30e5f1..b4990e289b10d 100644 --- a/metadata-jobs/mae-consumer/build.gradle +++ b/metadata-jobs/mae-consumer/build.gradle @@ -1,7 +1,7 @@ plugins { id 'java-library' } - +apply from: '../../gradle/coverage/java-coverage.gradle' apply plugin: 'pegasus' configurations { diff --git a/metadata-jobs/mce-consumer-job/build.gradle b/metadata-jobs/mce-consumer-job/build.gradle index 49c69626c211c..d9b8965d7c8ac 100644 --- a/metadata-jobs/mce-consumer-job/build.gradle +++ b/metadata-jobs/mce-consumer-job/build.gradle @@ -4,6 +4,7 @@ plugins { id 'com.palantir.docker' } +apply from: '../../gradle/coverage/java-coverage.gradle' apply from: "../../gradle/versioning/versioning.gradle" ext { diff --git a/metadata-jobs/mce-consumer/build.gradle b/metadata-jobs/mce-consumer/build.gradle index 5ea24059a3ee3..2da3957c4bb21 100644 --- a/metadata-jobs/mce-consumer/build.gradle +++ b/metadata-jobs/mce-consumer/build.gradle @@ -3,6 +3,7 @@ plugins { id 'pegasus' } +apply from: '../../gradle/coverage/java-coverage.gradle' configurations { avro diff --git a/metadata-jobs/pe-consumer/build.gradle b/metadata-jobs/pe-consumer/build.gradle index 35fdf0231c406..ad17a44377085 100644 --- a/metadata-jobs/pe-consumer/build.gradle +++ b/metadata-jobs/pe-consumer/build.gradle @@ -3,6 +3,8 @@ plugins { id 'pegasus' } +apply from: '../../gradle/coverage/java-coverage.gradle' + configurations { avro } diff --git a/metadata-models-custom/build.gradle b/metadata-models-custom/build.gradle index 412c19194c733..9babf9467e5d6 100644 --- a/metadata-models-custom/build.gradle +++ b/metadata-models-custom/build.gradle @@ -19,6 +19,8 @@ plugins { id 'pegasus' } +apply from: '../gradle/coverage/java-coverage.gradle' + if (project.hasProperty('projVersion')) { project.version = project.projVersion } else { diff --git a/metadata-models-validator/build.gradle b/metadata-models-validator/build.gradle index 1dae53e817ae1..c828ab041c27b 100644 --- a/metadata-models-validator/build.gradle +++ b/metadata-models-validator/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(":entity-registry") implementation spec.product.pegasus.data diff --git a/metadata-models/build.gradle b/metadata-models/build.gradle index 179e1eac177ac..e9379163ecaec 100644 --- a/metadata-models/build.gradle +++ b/metadata-models/build.gradle @@ -6,6 +6,7 @@ plugins { id 'org.hidetake.swagger.generator' } +apply from: '../gradle/coverage/java-coverage.gradle' dependencies { api spec.product.pegasus.data diff --git a/metadata-operation-context/build.gradle b/metadata-operation-context/build.gradle index 650082ef0d25e..71b61528d187b 100644 --- a/metadata-operation-context/build.gradle +++ b/metadata-operation-context/build.gradle @@ -1,6 +1,7 @@ plugins { id 'java-library' } +apply from: '../gradle/coverage/java-coverage.gradle' dependencies { api project(':metadata-utils') diff --git a/metadata-service/auth-config/build.gradle b/metadata-service/auth-config/build.gradle index 8302e3b0c2fe6..de787e9f350dc 100644 --- a/metadata-service/auth-config/build.gradle +++ b/metadata-service/auth-config/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(path: ':metadata-models') implementation project(path: ':metadata-auth:auth-api') diff --git a/metadata-service/auth-filter/build.gradle b/metadata-service/auth-filter/build.gradle index 9d763ca11421b..5e73246317677 100644 --- a/metadata-service/auth-filter/build.gradle +++ b/metadata-service/auth-filter/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-auth:auth-api') implementation project(':metadata-service:auth-impl') diff --git a/metadata-service/auth-impl/build.gradle b/metadata-service/auth-impl/build.gradle index 4f4b0658caf24..b5c20ccabe6d2 100644 --- a/metadata-service/auth-impl/build.gradle +++ b/metadata-service/auth-impl/build.gradle @@ -2,6 +2,7 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' compileJava { options.debug = true diff --git a/metadata-service/auth-servlet-impl/build.gradle b/metadata-service/auth-servlet-impl/build.gradle index 29e452472358b..21fe63778cbeb 100644 --- a/metadata-service/auth-servlet-impl/build.gradle +++ b/metadata-service/auth-servlet-impl/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-auth:auth-api') implementation project(':metadata-service:auth-impl') diff --git a/metadata-service/configuration/build.gradle b/metadata-service/configuration/build.gradle index f912e2ac01f0b..3c7e38b004fae 100644 --- a/metadata-service/configuration/build.gradle +++ b/metadata-service/configuration/build.gradle @@ -1,6 +1,8 @@ plugins { id 'java-library' } + +apply from: '../../gradle/coverage/java-coverage.gradle' apply from: "../../gradle/versioning/versioning.gradle" dependencies { diff --git a/metadata-service/factories/build.gradle b/metadata-service/factories/build.gradle index 5e52efd245b7f..501c46d64d6f9 100644 --- a/metadata-service/factories/build.gradle +++ b/metadata-service/factories/build.gradle @@ -1,6 +1,7 @@ plugins { id 'java-library' } +apply from: '../../gradle/coverage/java-coverage.gradle' dependencies { api project(':metadata-io') diff --git a/metadata-service/graphql-servlet-impl/build.gradle b/metadata-service/graphql-servlet-impl/build.gradle index 5767698242118..6848696db6788 100644 --- a/metadata-service/graphql-servlet-impl/build.gradle +++ b/metadata-service/graphql-servlet-impl/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':datahub-graphql-core') implementation project(':metadata-auth:auth-api') diff --git a/metadata-service/openapi-analytics-servlet/build.gradle b/metadata-service/openapi-analytics-servlet/build.gradle index 5d1372a293775..45e0ee1758ebc 100644 --- a/metadata-service/openapi-analytics-servlet/build.gradle +++ b/metadata-service/openapi-analytics-servlet/build.gradle @@ -3,6 +3,8 @@ plugins { id 'org.hidetake.swagger.generator' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-auth:auth-api') diff --git a/metadata-service/openapi-entity-servlet/build.gradle b/metadata-service/openapi-entity-servlet/build.gradle index 4c2d587a81fd7..a47e487f871c6 100644 --- a/metadata-service/openapi-entity-servlet/build.gradle +++ b/metadata-service/openapi-entity-servlet/build.gradle @@ -3,6 +3,8 @@ plugins { id 'org.hidetake.swagger.generator' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-auth:auth-api') diff --git a/metadata-service/openapi-servlet/build.gradle b/metadata-service/openapi-servlet/build.gradle index e26b1ceea1a3c..77679790f25de 100644 --- a/metadata-service/openapi-servlet/build.gradle +++ b/metadata-service/openapi-servlet/build.gradle @@ -3,6 +3,8 @@ plugins { id 'org.hidetake.swagger.generator' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-auth:auth-api') diff --git a/metadata-service/plugin/build.gradle b/metadata-service/plugin/build.gradle index 501589c24d60a..526744ceb85c5 100644 --- a/metadata-service/plugin/build.gradle +++ b/metadata-service/plugin/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation 'org.apache.commons:commons-lang3:3.12.0' diff --git a/metadata-service/restli-api/build.gradle b/metadata-service/restli-api/build.gradle index 505320e8267ee..38a3e9298ff96 100644 --- a/metadata-service/restli-api/build.gradle +++ b/metadata-service/restli-api/build.gradle @@ -2,6 +2,8 @@ plugins { id 'pegasus' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { dataModel project(':metadata-models') dataModel spec.product.pegasus.restliCommon diff --git a/metadata-service/restli-client-api/build.gradle b/metadata-service/restli-client-api/build.gradle index c2dfd4d2a1344..4e488a98023cb 100644 --- a/metadata-service/restli-client-api/build.gradle +++ b/metadata-service/restli-client-api/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java-library' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { api project(path: ':metadata-service:restli-api', configuration: 'restClient') api project(':metadata-operation-context') diff --git a/metadata-service/restli-client/build.gradle b/metadata-service/restli-client/build.gradle index 587ff5b2ef053..92b3433016633 100644 --- a/metadata-service/restli-client/build.gradle +++ b/metadata-service/restli-client/build.gradle @@ -3,6 +3,8 @@ plugins { id 'java-library' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { api project(':metadata-service:restli-api') api project(':metadata-auth:auth-api') diff --git a/metadata-service/restli-servlet-impl/build.gradle b/metadata-service/restli-servlet-impl/build.gradle index 6b68abfe7fb15..dc426bf5639b1 100644 --- a/metadata-service/restli-servlet-impl/build.gradle +++ b/metadata-service/restli-servlet-impl/build.gradle @@ -3,6 +3,8 @@ plugins { id 'pegasus' } +apply from: '../../gradle/coverage/java-coverage.gradle' + sourceSets { integTest { compileClasspath += sourceSets.main.output diff --git a/metadata-service/schema-registry-api/build.gradle b/metadata-service/schema-registry-api/build.gradle index 201e98beb490e..525eceda685be 100644 --- a/metadata-service/schema-registry-api/build.gradle +++ b/metadata-service/schema-registry-api/build.gradle @@ -3,6 +3,7 @@ plugins { id 'java-library' } +apply from: '../../gradle/coverage/java-coverage.gradle' dependencies { // Dependencies for open api diff --git a/metadata-service/schema-registry-servlet/build.gradle b/metadata-service/schema-registry-servlet/build.gradle index f19237c32cee7..94ca8fe03822e 100644 --- a/metadata-service/schema-registry-servlet/build.gradle +++ b/metadata-service/schema-registry-servlet/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-service:factories') implementation project(':metadata-service:schema-registry-api') diff --git a/metadata-service/services/build.gradle b/metadata-service/services/build.gradle index ea1ff32cb3838..0e84580cc04da 100644 --- a/metadata-service/services/build.gradle +++ b/metadata-service/services/build.gradle @@ -3,6 +3,8 @@ plugins { id 'java-library' } +apply from: '../../gradle/coverage/java-coverage.gradle' + configurations { enhance } diff --git a/metadata-service/servlet/build.gradle b/metadata-service/servlet/build.gradle index f961bf6a9de7e..725352e69d52b 100644 --- a/metadata-service/servlet/build.gradle +++ b/metadata-service/servlet/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':metadata-io') implementation project(':datahub-graphql-core') diff --git a/metadata-service/war/build.gradle b/metadata-service/war/build.gradle index ab9019e470dbc..5e00207c1726e 100644 --- a/metadata-service/war/build.gradle +++ b/metadata-service/war/build.gradle @@ -3,6 +3,7 @@ plugins { id 'com.palantir.docker' } +apply from: '../../gradle/coverage/java-coverage.gradle' apply from: "../../gradle/versioning/versioning.gradle" ext { diff --git a/metadata-utils/build.gradle b/metadata-utils/build.gradle index 919d93c5f9fe1..07ce50993655d 100644 --- a/metadata-utils/build.gradle +++ b/metadata-utils/build.gradle @@ -3,6 +3,8 @@ plugins { id 'pegasus' } +apply from: '../gradle/coverage/java-coverage.gradle' + dependencies { api externalDependency.avro implementation externalDependency.commonsLang diff --git a/mock-entity-registry/build.gradle b/mock-entity-registry/build.gradle index 8242d6451dd60..3a6ce0eedcba6 100644 --- a/mock-entity-registry/build.gradle +++ b/mock-entity-registry/build.gradle @@ -2,6 +2,8 @@ plugins { id 'java' } +apply from: '../gradle/coverage/java-coverage.gradle' + dependencies { implementation project(':entity-registry') } \ No newline at end of file