From c8d40fd604cab112bbef0e74bf7e3b1fdb1fecad Mon Sep 17 00:00:00 2001 From: OpenLineage deploy bot Date: Wed, 28 Aug 2024 12:42:29 +0000 Subject: [PATCH] openlineage specification update --- .last_spec_commit_id | 2 +- .../1-0-0/GcpDataprocSparkRunFacet.json | 75 +++++++++++++++++++ .../spec/facets/1-0-0/GcpLineageJobFacet.json | 43 +++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 static/spec/facets/1-0-0/GcpDataprocSparkRunFacet.json create mode 100644 static/spec/facets/1-0-0/GcpLineageJobFacet.json diff --git a/.last_spec_commit_id b/.last_spec_commit_id index 1c0ac1fff1..60856f8740 100644 --- a/.last_spec_commit_id +++ b/.last_spec_commit_id @@ -1 +1 @@ -00a7e9cfb47da961f710a3dedb4483bf5901b937 +1a64c0e47ed51e412a97d1753cdf24a0dae0e492 diff --git a/static/spec/facets/1-0-0/GcpDataprocSparkRunFacet.json b/static/spec/facets/1-0-0/GcpDataprocSparkRunFacet.json new file mode 100644 index 0000000000..81bf9d77f7 --- /dev/null +++ b/static/spec/facets/1-0-0/GcpDataprocSparkRunFacet.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openlineage.io/spec/facets/1-0-0/GcpDataprocSparkRunFacet.json", + "$defs": { + "GcpDataprocSparkRunFacet": { + "allOf": [ + { + "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunFacet" + }, + { + "type": "object", + "properties": { + "appId": { + "description": "Application ID set in the spark configuration of the current context. Its format depends on the resource manager.", + "type": "string" + }, + "appName": { + "description": "App name set in the spark configuration of the current context. It may be provided by the user.", + "type": "string" + }, + "batchId": { + "description": "Populated only for Dataproc serverless batches. The resource id of the batch.", + "type": "string" + }, + "batchUuid": { + "description": "Populated only for Dataproc serverless batches. A UUID generated by the service when it creates the batch.", + "type": "string" + }, + "clusterName": { + "description": "Populated only for Dataproc GCE workloads. The cluster name is unique within a GCP project.", + "type": "string" + }, + "clusterUuid": { + "description": "Populated only for Dataproc GCE workloads. A UUID generated by the service at the time of cluster creation.", + "type": "string" + }, + "jobId": { + "description": "Populated only for Dataproc GCE workloads. If not specified by the user, the job ID will be provided by the service.", + "type": "string" + }, + "jobUuid": { + "description": "Populated only for Dataproc GCE workloads. A UUID that uniquely identifies a job within the project over time.", + "type": "string" + }, + "projectId": { + "description": "The GCP project ID that the resource belongs to.", + "type": "string" + }, + "queryNodeName": { + "description": "The name of the query node in the executed Spark Plan. Often used to describe the command being executed.", + "type": "string" + }, + "sessionId": { + "description": "Populated only for Dataproc serverless interactive sessions. The resource id of the session, used for URL generation.", + "type": "string" + }, + "sessionUuid": { + "description": "Populated only for Dataproc serverless interactive sessions. A UUID generated by the service when it creates the session.", + "type": "string" + } + }, + "required": ["appId", "appName", "projectId"], + "additionalProperties": true + } + ], + "type": "object" + } + }, + "type": "object", + "properties": { + "gcp_dataproc_spark": { + "$ref": "#/$defs/GcpDataprocSparkRunFacet" + } + } +} diff --git a/static/spec/facets/1-0-0/GcpLineageJobFacet.json b/static/spec/facets/1-0-0/GcpLineageJobFacet.json new file mode 100644 index 0000000000..ae478c4b32 --- /dev/null +++ b/static/spec/facets/1-0-0/GcpLineageJobFacet.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openlineage.io/spec/facets/1-0-0/GcpLineageJobFacet.json", + "$defs": { + "GcpLineageJobFacet": { + "allOf": [ + { + "$ref": "https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/JobFacet" + }, + { + "type": "object", + "properties": { + "displayName": { + "description": "The name of the job to be used on UI", + "type": "string" + }, + "origin": { + "type": "object", + "properties": { + "sourceType": { + "description": "Type of the source. Possible values can be found in GCP documentation (https://cloud.google.com/data-catalog/docs/reference/data-lineage/rest/v1/projects.locations.processes#SourceType) ", + "type": "string" + }, + "name": { + "description": "If the sourceType isn't CUSTOM, the value of this field should be a GCP resource name of the system, which reports lineage. The project and location parts of the resource name must match the project and location of the lineage resource being created. More details in GCP documentation https://cloud.google.com/data-catalog/docs/reference/data-lineage/rest/v1/projects.locations.processes#origin", + "type": "string" + } + } + } + }, + "additionalProperties": true + } + ], + "type": "object" + } + }, + "type": "object", + "properties": { + "gcp_lineage": { + "$ref": "#/$defs/GcpLineageJobFacet" + } + } +}