diff --git a/core/src/main/java/org/apache/iceberg/MetadataUpdate.java b/core/src/main/java/org/apache/iceberg/MetadataUpdate.java index 2cf16bca6c32..363aabbff24f 100644 --- a/core/src/main/java/org/apache/iceberg/MetadataUpdate.java +++ b/core/src/main/java/org/apache/iceberg/MetadataUpdate.java @@ -53,6 +53,11 @@ public String uuid() { public void applyTo(TableMetadata.Builder metadataBuilder) { metadataBuilder.assignUUID(uuid); } + + @Override + public void applyTo(ViewMetadata.Builder metadataBuilder) { + metadataBuilder.assignUUID(uuid); + } } class UpgradeFormatVersion implements MetadataUpdate { diff --git a/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java b/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java index d4df7169fd1e..fe2f2c8b684a 100644 --- a/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java +++ b/core/src/main/java/org/apache/iceberg/view/ViewMetadata.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.iceberg.MetadataUpdate; @@ -46,6 +47,8 @@ public interface ViewMetadata extends Serializable { int SUPPORTED_VIEW_FORMAT_VERSION = 1; int DEFAULT_VIEW_FORMAT_VERSION = 1; + String uuid(); + int formatVersion(); String location(); @@ -141,6 +144,7 @@ class Builder { private int formatVersion = DEFAULT_VIEW_FORMAT_VERSION; private int currentVersionId; private String location; + private String uuid; // internal change tracking private Integer lastAddedVersionId = null; @@ -157,6 +161,7 @@ private Builder() { this.history = Lists.newArrayList(); this.properties = Maps.newHashMap(); this.changes = Lists.newArrayList(); + this.uuid = null; } private Builder(ViewMetadata base) { @@ -170,6 +175,7 @@ private Builder(ViewMetadata base) { this.formatVersion = base.formatVersion(); this.currentVersionId = base.currentVersionId(); this.location = base.location(); + this.uuid = base.uuid(); } public Builder upgradeFormatVersion(int newFormatVersion) { @@ -353,6 +359,18 @@ public Builder removeProperties(Set propertiesToRemove) { return this; } + public ViewMetadata.Builder assignUUID(String newUUID) { + Preconditions.checkArgument(newUUID != null, "Cannot set uuid to null"); + Preconditions.checkArgument(uuid == null || newUUID.equals(uuid), "Cannot reassign uuid"); + + if (!newUUID.equals(uuid)) { + this.uuid = newUUID; + changes.add(new MetadataUpdate.AssignUUID(uuid)); + } + + return this; + } + public ViewMetadata build() { Preconditions.checkArgument(null != location, "Invalid location: null"); Preconditions.checkArgument(versions.size() > 0, "Invalid view: no versions were added"); @@ -386,6 +404,7 @@ public ViewMetadata build() { } return ImmutableViewMetadata.of( + null == uuid ? UUID.randomUUID().toString() : uuid, formatVersion, location, schemas, diff --git a/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java b/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java index c994c82ea875..0852db6a516b 100644 --- a/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java +++ b/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java @@ -39,6 +39,7 @@ public class ViewMetadataParser { + static final String VIEW_UUID = "view-uuid"; static final String FORMAT_VERSION = "format-version"; static final String LOCATION = "location"; static final String CURRENT_VERSION_ID = "current-version-id"; @@ -62,6 +63,7 @@ static void toJson(ViewMetadata metadata, JsonGenerator gen) throws IOException gen.writeStartObject(); + gen.writeStringField(VIEW_UUID, metadata.uuid()); gen.writeNumberField(FORMAT_VERSION, metadata.formatVersion()); gen.writeStringField(LOCATION, metadata.location()); JsonUtil.writeStringMap(PROPERTIES, metadata.properties(), gen); @@ -98,6 +100,7 @@ public static ViewMetadata fromJson(JsonNode json) { Preconditions.checkArgument( json.isObject(), "Cannot parse view metadata from non-object: %s", json); + String uuid = JsonUtil.getString(VIEW_UUID, json); int formatVersion = JsonUtil.getInt(FORMAT_VERSION, json); String location = JsonUtil.getString(LOCATION, json); Map properties = JsonUtil.getStringMap(PROPERTIES, json); @@ -131,6 +134,7 @@ public static ViewMetadata fromJson(JsonNode json) { } return ImmutableViewMetadata.of( + uuid, formatVersion, location, schemas, diff --git a/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java b/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java index a852a716d53f..acb344ffab97 100644 --- a/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java +++ b/core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.UUID; import org.apache.iceberg.MetadataUpdate; import org.apache.iceberg.Schema; import org.apache.iceberg.catalog.Namespace; @@ -43,6 +44,8 @@ private ViewVersion newViewVersion(int id, String sql) { .defaultCatalog("prod") .defaultNamespace(Namespace.of("default")) .summary(ImmutableMap.of("operation", "create")) + .addRepresentations( + ImmutableSQLViewRepresentation.builder().dialect("spark").sql(sql).build()) .schemaId(1) .build(); } @@ -101,6 +104,10 @@ public void nullAndMissingFields() { () -> ViewMetadata.builder().setLocation("location").setCurrentVersionId(1).build()) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Cannot set current version to unknown version: 1"); + + assertThatThrownBy(() -> ViewMetadata.builder().assignUUID(null).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot set uuid to null"); } @Test @@ -380,8 +387,10 @@ public void viewMetadataAndMetadataChanges() { .defaultNamespace(Namespace.of("ns")) .build(); + String uuid = "fa6506c3-7681-40c8-86dc-e36561f83385"; ViewMetadata viewMetadata = ViewMetadata.builder() + .assignUUID(uuid) .setLocation("custom-location") .setProperties(properties) .addSchema(schemaOne) @@ -406,23 +415,30 @@ public void viewMetadataAndMetadataChanges() { assertThat(viewMetadata.properties()).isEqualTo(properties); List changes = viewMetadata.changes(); - assertThat(changes).hasSize(8); + assertThat(changes).hasSize(9); assertThat(changes) .element(0) + .isInstanceOf(MetadataUpdate.AssignUUID.class) + .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AssignUUID.class)) + .extracting(MetadataUpdate.AssignUUID::uuid) + .isEqualTo(uuid); + + assertThat(changes) + .element(1) .isInstanceOf(MetadataUpdate.SetLocation.class) .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.SetLocation.class)) .extracting(MetadataUpdate.SetLocation::location) .isEqualTo("custom-location"); assertThat(changes) - .element(1) + .element(2) .isInstanceOf(MetadataUpdate.SetProperties.class) .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.SetProperties.class)) .extracting(MetadataUpdate.SetProperties::updated) .isEqualTo(properties); assertThat(changes) - .element(2) + .element(3) .isInstanceOf(MetadataUpdate.AddSchema.class) .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddSchema.class)) .extracting(MetadataUpdate.AddSchema::schema) @@ -430,7 +446,7 @@ public void viewMetadataAndMetadataChanges() { .isEqualTo(1); assertThat(changes) - .element(3) + .element(4) .isInstanceOf(MetadataUpdate.AddSchema.class) .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddSchema.class)) .extracting(MetadataUpdate.AddSchema::schema) @@ -438,31 +454,72 @@ public void viewMetadataAndMetadataChanges() { .isEqualTo(2); assertThat(changes) - .element(4) + .element(5) .isInstanceOf(MetadataUpdate.AddViewVersion.class) .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddViewVersion.class)) .extracting(MetadataUpdate.AddViewVersion::viewVersion) .isEqualTo(viewVersionOne); assertThat(changes) - .element(5) + .element(6) .isInstanceOf(MetadataUpdate.AddViewVersion.class) .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddViewVersion.class)) .extracting(MetadataUpdate.AddViewVersion::viewVersion) .isEqualTo(viewVersionTwo); assertThat(changes) - .element(6) + .element(7) .isInstanceOf(MetadataUpdate.AddViewVersion.class) .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AddViewVersion.class)) .extracting(MetadataUpdate.AddViewVersion::viewVersion) .isEqualTo(viewVersionThree); assertThat(changes) - .element(7) + .element(8) .isInstanceOf(MetadataUpdate.SetCurrentViewVersion.class) .asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.SetCurrentViewVersion.class)) .extracting(MetadataUpdate.SetCurrentViewVersion::versionId) .isEqualTo(-1); } + + @Test + public void uuidAssignment() { + String uuid = "fa6506c3-7681-40c8-86dc-e36561f83385"; + ViewMetadata viewMetadata = + ViewMetadata.builder() + .assignUUID(uuid) + .setLocation("custom-location") + .addSchema(new Schema(1, Types.NestedField.required(1, "x", Types.LongType.get()))) + .addVersion( + ImmutableViewVersion.builder() + .schemaId(1) + .versionId(1) + .timestampMillis(23L) + .putSummary("operation", "create") + .defaultNamespace(Namespace.of("ns")) + .build()) + .setCurrentVersionId(1) + .build(); + + assertThat(viewMetadata.uuid()).isEqualTo(uuid); + + // uuid should be carried over + ViewMetadata updated = ViewMetadata.buildFrom(viewMetadata).build(); + assertThat(updated.uuid()).isEqualTo(uuid); + assertThat(updated.changes()).isEmpty(); + + // assigning the same uuid shouldn't fail and shouldn't cause any changes + updated = ViewMetadata.buildFrom(viewMetadata).assignUUID(uuid).build(); + assertThat(updated.uuid()).isEqualTo(uuid); + assertThat(updated.changes()).isEmpty(); + + // can't reassign view uuid + assertThatThrownBy( + () -> + ViewMetadata.buildFrom(viewMetadata) + .assignUUID(UUID.randomUUID().toString()) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot reassign uuid"); + } } diff --git a/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java b/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java index 807a1df9b1d1..5efbcf026f08 100644 --- a/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java +++ b/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java @@ -89,6 +89,7 @@ public void readAndWriteValidViewMetadata() throws Exception { String json = readViewMetadataInputFile("org/apache/iceberg/view/ValidViewMetadata.json"); ViewMetadata expectedViewMetadata = ViewMetadata.builder() + .assignUUID("fa6506c3-7681-40c8-86dc-e36561f83385") .addSchema(TEST_SCHEMA) .addVersion(version1) .addVersion(version2) diff --git a/core/src/test/resources/org/apache/iceberg/view/ValidViewMetadata.json b/core/src/test/resources/org/apache/iceberg/view/ValidViewMetadata.json index 9c0ae0ecbef6..4e29ed8702a0 100644 --- a/core/src/test/resources/org/apache/iceberg/view/ValidViewMetadata.json +++ b/core/src/test/resources/org/apache/iceberg/view/ValidViewMetadata.json @@ -1,4 +1,5 @@ { + "view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385", "format-version": 1, "location": "s3://bucket/test/location", "properties": {"some-key": "some-value"}, diff --git a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentSchema.json b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentSchema.json index b63c6a628518..e6bdff2aadbf 100644 --- a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentSchema.json +++ b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentSchema.json @@ -1,4 +1,5 @@ { + "view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385", "format-version": 1, "location": "s3://bucket/test/location", "properties": {"some-key": "some-value"}, diff --git a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentVersion.json b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentVersion.json index fbcb2c9a4176..8db0359c0d11 100644 --- a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentVersion.json +++ b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataInvalidCurrentVersion.json @@ -1,4 +1,5 @@ { + "view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385", "format-version": 1, "location": "s3://bucket/test/location", "properties": {"some-key": "some-value"}, diff --git a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentVersion.json b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentVersion.json index f09a7a4aa6b5..07febf71c9da 100644 --- a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentVersion.json +++ b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingCurrentVersion.json @@ -1,4 +1,5 @@ { + "view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385", "format-version": 1, "location": "s3://bucket/test/location", "properties": {"some-key": "some-value"}, diff --git a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingLocation.json b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingLocation.json index d0fa7d9392a9..aa6d56ead3d4 100644 --- a/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingLocation.json +++ b/core/src/test/resources/org/apache/iceberg/view/ViewMetadataMissingLocation.json @@ -1,4 +1,5 @@ { + "view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385", "format-version": 1, "properties": {"some-key": "some-value"}, "current-schema-id": 1, diff --git a/format/view-spec.md b/format/view-spec.md index d7e0b7b7a60d..26313193afad 100644 --- a/format/view-spec.md +++ b/format/view-spec.md @@ -58,6 +58,7 @@ The view version metadata file has the following fields: | Requirement | Field name | Description | |-------------|----------------------|-------------| +| _required_ | `view-uuid` | A UUID that identifies the view, generated when the view is created. Implementations must throw an exception if a view's UUID does not match the expected UUID after refreshing metadata | | _required_ | `format-version` | An integer version number for the view format; must be 1 | | _required_ | `location` | The view's base location; used to create metadata file locations | | _required_ | `schemas` | A list of known schemas | @@ -192,6 +193,7 @@ s3://bucket/warehouse/default.db/event_agg/metadata/00001-(uuid).metadata.json ``` ``` { + "view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385", "format-version" : 1, "location" : "s3://bucket/warehouse/default.db/event_agg", "current-version-id" : 1, @@ -259,6 +261,7 @@ s3://bucket/warehouse/default.db/event_agg/metadata/00002-(uuid).metadata.json ``` ``` { + "view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385", "format-version" : 1, "location" : "s3://bucket/warehouse/default.db/event_agg", "current-version-id" : 1,