Skip to content

Commit

Permalink
Core: Add uuid to ViewMetadata
Browse files Browse the repository at this point in the history
  • Loading branch information
nastra committed Sep 19, 2023
1 parent 1b9a835 commit d4c0a97
Show file tree
Hide file tree
Showing 11 changed files with 54 additions and 0 deletions.
5 changes: 5 additions & 0 deletions core/src/main/java/org/apache/iceberg/MetadataUpdate.java
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,11 @@ public String uuid() {
public void applyTo(TableMetadata.Builder metadataBuilder) {
metadataBuilder.assignUUID(uuid);
}

@Override
public void applyTo(ViewMetadata.Builder metadataBuilder) {
metadataBuilder.assignUUID(uuid);
}
}

class UpgradeFormatVersion implements MetadataUpdate {
Expand Down
18 changes: 18 additions & 0 deletions core/src/main/java/org/apache/iceberg/view/ViewMetadata.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.iceberg.MetadataUpdate;
Expand All @@ -46,6 +47,8 @@ public interface ViewMetadata extends Serializable {
int SUPPORTED_VIEW_FORMAT_VERSION = 1;
int DEFAULT_VIEW_FORMAT_VERSION = 1;

String uuid();

int formatVersion();

String location();
Expand Down Expand Up @@ -141,6 +144,7 @@ class Builder {
private int formatVersion = DEFAULT_VIEW_FORMAT_VERSION;
private int currentVersionId;
private String location;
private String uuid;

// internal change tracking
private Integer lastAddedVersionId = null;
Expand All @@ -157,6 +161,7 @@ private Builder() {
this.history = Lists.newArrayList();
this.properties = Maps.newHashMap();
this.changes = Lists.newArrayList();
this.uuid = UUID.randomUUID().toString();
}

private Builder(ViewMetadata base) {
Expand All @@ -170,6 +175,7 @@ private Builder(ViewMetadata base) {
this.formatVersion = base.formatVersion();
this.currentVersionId = base.currentVersionId();
this.location = base.location();
this.uuid = base.uuid();
}

public Builder upgradeFormatVersion(int newFormatVersion) {
Expand Down Expand Up @@ -353,6 +359,17 @@ public Builder removeProperties(Set<String> propertiesToRemove) {
return this;
}

public ViewMetadata.Builder assignUUID(String newUUID) {
Preconditions.checkArgument(newUUID != null, "Cannot set uuid to null");

if (!newUUID.equals(uuid)) {
this.uuid = newUUID;
changes.add(new MetadataUpdate.AssignUUID(uuid));
}

return this;
}

public ViewMetadata build() {
Preconditions.checkArgument(null != location, "Invalid location: null");
Preconditions.checkArgument(versions.size() > 0, "Invalid view: no versions were added");
Expand Down Expand Up @@ -386,6 +403,7 @@ public ViewMetadata build() {
}

return ImmutableViewMetadata.of(
uuid,
formatVersion,
location,
schemas,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@

public class ViewMetadataParser {

static final String VIEW_UUID = "view-uuid";
static final String FORMAT_VERSION = "format-version";
static final String LOCATION = "location";
static final String CURRENT_VERSION_ID = "current-version-id";
Expand All @@ -62,6 +63,7 @@ static void toJson(ViewMetadata metadata, JsonGenerator gen) throws IOException

gen.writeStartObject();

gen.writeStringField(VIEW_UUID, metadata.uuid());
gen.writeNumberField(FORMAT_VERSION, metadata.formatVersion());
gen.writeStringField(LOCATION, metadata.location());
JsonUtil.writeStringMap(PROPERTIES, metadata.properties(), gen);
Expand Down Expand Up @@ -98,6 +100,7 @@ public static ViewMetadata fromJson(JsonNode json) {
Preconditions.checkArgument(
json.isObject(), "Cannot parse view metadata from non-object: %s", json);

String uuid = JsonUtil.getString(VIEW_UUID, json);
int formatVersion = JsonUtil.getInt(FORMAT_VERSION, json);
String location = JsonUtil.getString(LOCATION, json);
Map<String, String> properties = JsonUtil.getStringMap(PROPERTIES, json);
Expand Down Expand Up @@ -131,6 +134,7 @@ public static ViewMetadata fromJson(JsonNode json) {
}

return ImmutableViewMetadata.of(
uuid,
formatVersion,
location,
schemas,
Expand Down
18 changes: 18 additions & 0 deletions core/src/test/java/org/apache/iceberg/view/TestViewMetadata.java
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ private ViewVersion newViewVersion(int id, String sql) {
.defaultCatalog("prod")
.defaultNamespace(Namespace.of("default"))
.summary(ImmutableMap.of("operation", "create"))
.addRepresentations(
ImmutableSQLViewRepresentation.builder().dialect("spark").sql(sql).build())
.schemaId(1)
.build();
}
Expand Down Expand Up @@ -101,6 +103,10 @@ public void nullAndMissingFields() {
() -> ViewMetadata.builder().setLocation("location").setCurrentVersionId(1).build())
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot set current version to unknown version: 1");

assertThatThrownBy(() -> ViewMetadata.builder().assignUUID(null).build())
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Cannot set uuid to null");
}

@Test
Expand Down Expand Up @@ -464,5 +470,17 @@ public void viewMetadataAndMetadataChanges() {
.asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.SetCurrentViewVersion.class))
.extracting(MetadataUpdate.SetCurrentViewVersion::versionId)
.isEqualTo(-1);

ViewMetadata updatedMetadata =
ViewMetadata.buildFrom(viewMetadata)
.assignUUID("fa6506c3-7681-40c8-86dc-e36561f83385")
.build();
assertThat(updatedMetadata.changes())
.hasSize(1)
.first()
.isInstanceOf(MetadataUpdate.AssignUUID.class)
.asInstanceOf(InstanceOfAssertFactories.type(MetadataUpdate.AssignUUID.class))
.extracting(MetadataUpdate.AssignUUID::uuid)
.isEqualTo("fa6506c3-7681-40c8-86dc-e36561f83385");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ public void readAndWriteValidViewMetadata() throws Exception {
String json = readViewMetadataInputFile("org/apache/iceberg/view/ValidViewMetadata.json");
ViewMetadata expectedViewMetadata =
ViewMetadata.builder()
.assignUUID("fa6506c3-7681-40c8-86dc-e36561f83385")
.addSchema(TEST_SCHEMA)
.addVersion(version1)
.addVersion(version2)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{
"view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version": 1,
"location": "s3://bucket/test/location",
"properties": {"some-key": "some-value"},
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{
"view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version": 1,
"location": "s3://bucket/test/location",
"properties": {"some-key": "some-value"},
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{
"view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version": 1,
"location": "s3://bucket/test/location",
"properties": {"some-key": "some-value"},
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{
"view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version": 1,
"location": "s3://bucket/test/location",
"properties": {"some-key": "some-value"},
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{
"view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version": 1,
"properties": {"some-key": "some-value"},
"current-schema-id": 1,
Expand Down
3 changes: 3 additions & 0 deletions format/view-spec.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ The view version metadata file has the following fields:

| Requirement | Field name | Description |
|-------------|----------------------|-------------|
| _required_ | `view-uuid` | A UUID that identifies the view, generated when the view is created. Implementations must throw an exception if a view's UUID does not match the expected UUID after refreshing metadata |
| _required_ | `format-version` | An integer version number for the view format; must be 1 |
| _required_ | `location` | The view's base location; used to create metadata file locations |
| _required_ | `schemas` | A list of known schemas |
Expand Down Expand Up @@ -192,6 +193,7 @@ s3://bucket/warehouse/default.db/event_agg/metadata/00001-(uuid).metadata.json
```
```
{
"view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version" : 1,
"location" : "s3://bucket/warehouse/default.db/event_agg",
"current-version-id" : 1,
Expand Down Expand Up @@ -259,6 +261,7 @@ s3://bucket/warehouse/default.db/event_agg/metadata/00002-(uuid).metadata.json
```
```
{
"view-uuid": "fa6506c3-7681-40c8-86dc-e36561f83385",
"format-version" : 1,
"location" : "s3://bucket/warehouse/default.db/event_agg",
"current-version-id" : 1,
Expand Down

0 comments on commit d4c0a97

Please sign in to comment.