From 33dd10d646f25b45fcd9cb0c69347c1763584948 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 1 Dec 2023 18:25:43 +0100 Subject: [PATCH] Extensible object types (#7771) --- CHANGELOG.md | 13 +- .../storage/batching/BatchingPersistImpl.java | 2 +- .../storage/bigtable/BigTablePersist.java | 35 +- .../storage/cache/CachingPersistImpl.java | 2 +- .../storage/cassandra/CassandraBackend.java | 114 ++-- .../storage/cassandra/CassandraConstants.java | 186 +++-- .../storage/cassandra/CassandraPersist.java | 622 +++-------------- .../storage/cassandra/CassandraSerde.java | 86 +++ .../storage/cassandra/CqlColumn.java | 56 ++ .../storage/cassandra/CqlColumnType.java | 33 +- .../serializers/CommitObjSerializer.java | 204 ++++++ .../ContentValueObjSerializer.java | 94 +++ .../serializers/CustomObjSerializer.java | 81 +++ .../serializers/IndexObjSerializer.java | 88 +++ .../IndexSegmentsObjSerializer.java | 111 +++ .../cassandra/serializers/ObjSerializer.java | 37 + .../cassandra/serializers/ObjSerializers.java | 86 +++ .../serializers/RefObjSerializer.java | 94 +++ .../serializers/StringObjSerializer.java | 107 +++ .../serializers/TagObjSerializer.java | 119 ++++ .../storage/common/proto/storage.proto | 6 + .../storage/common-serialize/build.gradle.kts | 7 +- .../storage/serialize/ProtoSerialization.java | 67 +- .../storage/serialize/SmileSerialization.java | 77 +++ .../storage/common-tests/build.gradle.kts | 10 +- .../AbstractBackendRepositoryTests.java | 13 +- .../commontests/AbstractBasePersistTests.java | 258 +++---- .../AbstractConsistencyLogicTests.java | 8 +- .../AbstractIndexesLogicTests.java | 4 +- .../AbstractRepositoryLogicTests.java | 7 +- .../commontests/objtypes/SimpleCustomObj.java | 69 ++ .../objtypes/SimpleCustomObjDeserializer.java | 33 + .../objtypes/SimpleCustomObjType.java | 40 ++ .../objtypes/SimpleCustomObjTypeBundle.java | 28 + ...ioned.storage.common.persist.ObjTypeBundle | 17 + .../common/json/ObjIdDeserializer.java | 34 + .../storage/common/json/ObjIdHelper.java | 38 ++ .../storage/common/json/ObjIdSerializer.java | 35 + .../storage/common/json/PersistModule.java | 29 + .../storage/common/logic/CommitLogicImpl.java | 6 +- .../common/logic/ConsistencyLogicImpl.java | 4 +- .../common/logic/IndexesLogicImpl.java | 34 +- .../common/logic/ReferenceLogicImpl.java | 4 +- .../storage/common/logic/StringLogicImpl.java | 2 +- .../storage/common/objtypes/CommitObj.java | 22 +- .../common/objtypes/ContentValueObj.java | 2 +- .../storage/common/objtypes/Hashes.java | 12 +- .../storage/common/objtypes/IndexObj.java | 2 +- .../common/objtypes/IndexSegmentsObj.java | 2 +- .../storage/common/objtypes/RefObj.java | 2 +- .../common/objtypes/StandardObjType.java | 68 ++ .../objtypes/StandardObjTypeBundle.java | 30 + .../storage/common/objtypes/StringObj.java | 2 +- .../storage/common/objtypes/TagObj.java | 2 +- .../versioned/storage/common/persist/Obj.java | 3 + .../storage/common/persist/ObjType.java | 63 +- .../storage/common/persist/ObjTypeBundle.java | 23 + .../storage/common/persist/ObjTypes.java | 78 +++ .../storage/common/persist/Reference.java | 5 +- .../com.fasterxml.jackson.databind.Module | 17 + ...ioned.storage.common.persist.ObjTypeBundle | 17 + .../common/json/TestObjIdSerialization.java | 47 ++ .../common/logic/TestReferenceIndexes.java | 10 +- ...bjTypes.java => TestStandardObjTypes.java} | 22 +- .../storage/dynamodb/DynamoDBConstants.java | 46 -- .../storage/dynamodb/DynamoDBPersist.java | 510 +------------- .../storage/dynamodb/DynamoDBSerde.java | 129 ++++ .../serializers/CommitObjSerializer.java | 144 ++++ .../ContentValueObjSerializer.java | 66 ++ .../serializers/CustomObjSerializer.java | 62 ++ .../serializers/IndexObjSerializer.java | 62 ++ .../IndexSegmentsObjSerializer.java | 59 ++ .../dynamodb/serializers/ObjSerializer.java | 39 ++ .../dynamodb/serializers/ObjSerializers.java | 86 +++ .../serializers/RefObjSerializer.java | 73 ++ .../serializers/StringObjSerializer.java | 84 +++ .../serializers/TagObjSerializer.java | 102 +++ .../storage/inmemory/InmemoryPersist.java | 2 +- .../storage/jdbc/AbstractJdbcPersist.java | 644 +++--------------- .../versioned/storage/jdbc/JdbcBackend.java | 86 ++- .../versioned/storage/jdbc/JdbcSerde.java | 123 ++++ .../versioned/storage/jdbc/SqlConstants.java | 156 +---- .../jdbc/serializers/CommitObjSerializer.java | 185 +++++ .../ContentValueObjSerializer.java | 77 +++ .../jdbc/serializers/CustomObjSerializer.java | 75 ++ .../jdbc/serializers/IndexObjSerializer.java | 75 ++ .../IndexSegmentsObjSerializer.java | 97 +++ .../jdbc/serializers/ObjSerializer.java | 55 ++ .../jdbc/serializers/ObjSerializers.java | 85 +++ .../jdbc/serializers/RefObjSerializer.java | 82 +++ .../jdbc/serializers/StringObjSerializer.java | 88 +++ .../jdbc/serializers/TagObjSerializer.java | 100 +++ .../storage/mongodb/MongoDBConstants.java | 47 -- .../storage/mongodb/MongoDBPersist.java | 445 +----------- .../storage/mongodb/MongoDBSerde.java | 112 +++ .../serializers/CommitObjSerializer.java | 138 ++++ .../ContentValueObjSerializer.java | 60 ++ .../serializers/CustomObjSerializer.java | 53 ++ .../serializers/IndexObjSerializer.java | 58 ++ .../IndexSegmentsObjSerializer.java | 55 ++ .../mongodb/serializers/ObjSerializer.java | 36 + .../mongodb/serializers/ObjSerializers.java | 86 +++ .../mongodb/serializers/RefObjSerializer.java | 66 ++ .../serializers/StringObjSerializer.java | 75 ++ .../mongodb/serializers/TagObjSerializer.java | 94 +++ .../storage/rocksdb/RocksDBPersist.java | 2 +- .../storage/versionstore/ContentMapping.java | 2 +- .../versionstore/VersionStoreImpl.java | 2 +- .../storage/versionstore/TestRefMapping.java | 2 +- .../transfer/TestExportImportV2.java | 6 +- ...MigrationFromDatabaseAdapterToPersist.java | 6 +- 111 files changed, 5440 insertions(+), 2726 deletions(-) create mode 100644 versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraSerde.java create mode 100644 versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CqlColumn.java create mode 100644 versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/CommitObjSerializer.java create mode 100644 versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/ContentValueObjSerializer.java create mode 100644 versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/CustomObjSerializer.java create mode 100644 versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/IndexObjSerializer.java create mode 100644 versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/IndexSegmentsObjSerializer.java create mode 100644 versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/ObjSerializer.java create mode 100644 versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/ObjSerializers.java create mode 100644 versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/RefObjSerializer.java create mode 100644 versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/StringObjSerializer.java create mode 100644 versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/TagObjSerializer.java create mode 100644 versioned/storage/common-serialize/src/main/java/org/projectnessie/versioned/storage/serialize/SmileSerialization.java create mode 100644 versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObj.java create mode 100644 versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObjDeserializer.java create mode 100644 versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObjType.java create mode 100644 versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObjTypeBundle.java create mode 100644 versioned/storage/common-tests/src/main/resources/META-INF/services/org.projectnessie.versioned.storage.common.persist.ObjTypeBundle create mode 100644 versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/ObjIdDeserializer.java create mode 100644 versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/ObjIdHelper.java create mode 100644 versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/ObjIdSerializer.java create mode 100644 versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/PersistModule.java create mode 100644 versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/StandardObjType.java create mode 100644 versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/StandardObjTypeBundle.java create mode 100644 versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/ObjTypeBundle.java create mode 100644 versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/ObjTypes.java create mode 100644 versioned/storage/common/src/main/resources/META-INF/services/com.fasterxml.jackson.databind.Module create mode 100644 versioned/storage/common/src/main/resources/META-INF/services/org.projectnessie.versioned.storage.common.persist.ObjTypeBundle create mode 100644 versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/json/TestObjIdSerialization.java rename versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/objtypes/{TestObjTypes.java => TestStandardObjTypes.java} (92%) create mode 100644 versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/DynamoDBSerde.java create mode 100644 versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/CommitObjSerializer.java create mode 100644 versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/ContentValueObjSerializer.java create mode 100644 versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/CustomObjSerializer.java create mode 100644 versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/IndexObjSerializer.java create mode 100644 versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/IndexSegmentsObjSerializer.java create mode 100644 versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/ObjSerializer.java create mode 100644 versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/ObjSerializers.java create mode 100644 versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/RefObjSerializer.java create mode 100644 versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/StringObjSerializer.java create mode 100644 versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/TagObjSerializer.java create mode 100644 versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/JdbcSerde.java create mode 100644 versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/CommitObjSerializer.java create mode 100644 versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/ContentValueObjSerializer.java create mode 100644 versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/CustomObjSerializer.java create mode 100644 versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/IndexObjSerializer.java create mode 100644 versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/IndexSegmentsObjSerializer.java create mode 100644 versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/ObjSerializer.java create mode 100644 versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/ObjSerializers.java create mode 100644 versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/RefObjSerializer.java create mode 100644 versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/StringObjSerializer.java create mode 100644 versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/TagObjSerializer.java create mode 100644 versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/MongoDBSerde.java create mode 100644 versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/CommitObjSerializer.java create mode 100644 versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/ContentValueObjSerializer.java create mode 100644 versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/CustomObjSerializer.java create mode 100644 versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/IndexObjSerializer.java create mode 100644 versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/IndexSegmentsObjSerializer.java create mode 100644 versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/ObjSerializer.java create mode 100644 versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/ObjSerializers.java create mode 100644 versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/RefObjSerializer.java create mode 100644 versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/StringObjSerializer.java create mode 100644 versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/TagObjSerializer.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 4811597133d..e4feed54045 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,19 @@ as necessary. Empty sections will not end in the release notes. ### Upgrade notes -* Nessie Quarkus parts are now built against Java 17 and Java 17 is required to run Nessie Quarkus Server directly. +- Nessie Quarkus parts are now built against Java 17 and Java 17 is required to run Nessie Quarkus Server directly. If you use the Docker image, nothing needs to be done, because the image already contains a compatible Java runtime. +- Due to the introduction of extensible object types in the storage layer, some storage backends + will require a schema upgrade: + - JDBC: the following SQL statement must be executed on the Nessie database (please adapt the + statement to the actual database SQL dialect): + ```sql + ALTER TABLE objs ADD COLUMN x_class VARCHAR, ADD COLUMN x_data BYTEA; + ``` + - Cassandra: the following CQL statement must be executed on the Nessie database and keyspace: + ```cql + ALTER TABLE .objs ADD x_class text, ADD x_data blob; + ``` ### Breaking changes diff --git a/versioned/storage/batching/src/main/java/org/projectnessie/versioned/storage/batching/BatchingPersistImpl.java b/versioned/storage/batching/src/main/java/org/projectnessie/versioned/storage/batching/BatchingPersistImpl.java index c8ffe66fed5..ff4fe45588c 100644 --- a/versioned/storage/batching/src/main/java/org/projectnessie/versioned/storage/batching/BatchingPersistImpl.java +++ b/versioned/storage/batching/src/main/java/org/projectnessie/versioned/storage/batching/BatchingPersistImpl.java @@ -212,7 +212,7 @@ public T fetchTypedObj( try { Obj r = pendingObj(id); if (r != null) { - if (r.type() != type) { + if (!r.type().equals(type)) { throw new ObjNotFoundException(id); } @SuppressWarnings("unchecked") diff --git a/versioned/storage/bigtable/src/main/java/org/projectnessie/versioned/storage/bigtable/BigTablePersist.java b/versioned/storage/bigtable/src/main/java/org/projectnessie/versioned/storage/bigtable/BigTablePersist.java index d72cd88811b..b75f470fcc3 100644 --- a/versioned/storage/bigtable/src/main/java/org/projectnessie/versioned/storage/bigtable/BigTablePersist.java +++ b/versioned/storage/bigtable/src/main/java/org/projectnessie/versioned/storage/bigtable/BigTablePersist.java @@ -48,12 +48,13 @@ import com.google.cloud.bigtable.data.v2.models.RowMutation; import com.google.cloud.bigtable.data.v2.models.RowMutationEntry; import com.google.common.collect.AbstractIterator; +import com.google.common.collect.ImmutableMap; import com.google.protobuf.ByteString; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeoutException; @@ -72,6 +73,7 @@ import org.projectnessie.versioned.storage.common.persist.Obj; import org.projectnessie.versioned.storage.common.persist.ObjId; import org.projectnessie.versioned.storage.common.persist.ObjType; +import org.projectnessie.versioned.storage.common.persist.ObjTypes; import org.projectnessie.versioned.storage.common.persist.Persist; import org.projectnessie.versioned.storage.common.persist.Reference; @@ -316,7 +318,7 @@ public T fetchTypedObj( @Nonnull @jakarta.annotation.Nonnull ObjId id, ObjType type, Class typeClass) throws ObjNotFoundException { Obj obj = fetchObj(id); - if (obj.type() != type) { + if (!obj.type().equals(type)) { throw new ObjNotFoundException(id); } @SuppressWarnings("unchecked") @@ -385,11 +387,11 @@ public boolean storeObj( } } - private static final ByteString[] OBJ_TYPE_VALUES = - Arrays.stream(ObjType.values()) - .map(Enum::name) - .map(ByteString::copyFromUtf8) - .toArray(ByteString[]::new); + private static final Map OBJ_TYPE_VALUES = + ObjTypes.allObjTypes().stream() + .collect( + ImmutableMap.toImmutableMap( + Function.identity(), (ObjType type) -> ByteString.copyFromUtf8(type.name()))); @NotNull private ConditionalRowMutation mutationForStoreObj( @@ -409,10 +411,7 @@ private ConditionalRowMutation mutationForStoreObj( Mutation.create() .setCell(FAMILY_OBJS, QUALIFIER_OBJS, CELL_TIMESTAMP, ref) .setCell( - FAMILY_OBJS, - QUALIFIER_OBJ_TYPE, - CELL_TIMESTAMP, - OBJ_TYPE_VALUES[obj.type().ordinal()]); + FAMILY_OBJS, QUALIFIER_OBJ_TYPE, CELL_TIMESTAMP, OBJ_TYPE_VALUES.get(obj.type())); Filter condition = FILTERS .chain() @@ -441,14 +440,12 @@ public boolean[] storeObjs(@Nonnull @jakarta.annotation.Nonnull Obj[] objs) @SuppressWarnings("unchecked") ApiFuture[] futures = new ApiFuture[objs.length]; - int idx = 0; for (int i = 0; i < objs.length; i++) { Obj obj = objs[i]; if (obj != null) { ConditionalRowMutation conditionalRowMutation = mutationForStoreObj(obj, false); - futures[idx] = backend.client().checkAndMutateRowAsync(conditionalRowMutation); + futures[i] = backend.client().checkAndMutateRowAsync(conditionalRowMutation); } - idx++; } boolean[] r = new boolean[objs.length]; @@ -520,7 +517,7 @@ public void upsertObj(@Nonnull @jakarta.annotation.Nonnull Obj obj) throws ObjTo FAMILY_OBJS, QUALIFIER_OBJ_TYPE, CELL_TIMESTAMP, - OBJ_TYPE_VALUES[obj.type().ordinal()])); + OBJ_TYPE_VALUES.get(obj.type()))); } catch (ApiException e) { throw apiException(e); } @@ -556,7 +553,7 @@ public void upsertObjs(@Nonnull @jakarta.annotation.Nonnull Obj[] objs) FAMILY_OBJS, QUALIFIER_OBJ_TYPE, CELL_TIMESTAMP, - OBJ_TYPE_VALUES[obj.type().ordinal()])); + OBJ_TYPE_VALUES.get(obj.type()))); } } catch (ApiException e) { throw apiException(e); @@ -596,7 +593,7 @@ private class ScanAllObjectsIterator extends AbstractIterator Filters.InterleaveFilter typeFilter = null; boolean all = true; - for (ObjType type : ObjType.values()) { + for (ObjType type : ObjTypes.allObjTypes()) { boolean match = filter.test(type); if (match) { if (typeFilter == null) { @@ -606,7 +603,7 @@ private class ScanAllObjectsIterator extends AbstractIterator FILTERS .chain() .filter(FILTERS.qualifier().exactMatch(QUALIFIER_OBJ_TYPE)) - .filter(FILTERS.value().exactMatch(OBJ_TYPE_VALUES[type.ordinal()]))); + .filter(FILTERS.value().exactMatch(OBJ_TYPE_VALUES.get(type)))); } else { all = false; } @@ -658,7 +655,7 @@ protected Obj computeNext() { } } - public static ObjId deserializeObjId(ByteString bytes) { + private static ObjId deserializeObjId(ByteString bytes) { if (bytes == null) { return null; } diff --git a/versioned/storage/cache/src/main/java/org/projectnessie/versioned/storage/cache/CachingPersistImpl.java b/versioned/storage/cache/src/main/java/org/projectnessie/versioned/storage/cache/CachingPersistImpl.java index 3983b4882dd..6ebd81e46fa 100644 --- a/versioned/storage/cache/src/main/java/org/projectnessie/versioned/storage/cache/CachingPersistImpl.java +++ b/versioned/storage/cache/src/main/java/org/projectnessie/versioned/storage/cache/CachingPersistImpl.java @@ -66,7 +66,7 @@ public T fetchTypedObj( throws ObjNotFoundException { Obj o = cache.get(id); if (o != null) { - if (o.type() != type) { + if (!o.type().equals(type)) { throw new ObjNotFoundException(id); } } else { diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraBackend.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraBackend.java index a3746b8ab20..62a21756fc0 100644 --- a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraBackend.java +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraBackend.java @@ -18,7 +18,10 @@ import static com.datastax.oss.driver.api.core.ConsistencyLevel.LOCAL_QUORUM; import static com.datastax.oss.driver.api.core.ConsistencyLevel.LOCAL_SERIAL; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.lang.String.format; +import static java.util.Map.entry; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.SECONDS; import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COLS_OBJS_ALL; @@ -41,14 +44,12 @@ import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.SELECT_BATCH_SIZE; import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.TABLE_OBJS; import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.TABLE_REFS; -import static org.projectnessie.versioned.storage.cassandra.CqlColumnType.NAME; -import static org.projectnessie.versioned.storage.cassandra.CqlColumnType.OBJ_ID; -import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.DriverException; import com.datastax.oss.driver.api.core.cql.AsyncResultSet; import com.datastax.oss.driver.api.core.cql.BoundStatement; +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.cql.PreparedStatement; import com.datastax.oss.driver.api.core.cql.ResultSet; import com.datastax.oss.driver.api.core.cql.Row; @@ -58,13 +59,11 @@ import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; import com.datastax.oss.driver.api.core.servererrors.CASWriteUnknownException; -import com.google.common.collect.ImmutableMap; import java.lang.reflect.Array; -import java.text.MessageFormat; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletionStage; @@ -75,11 +74,11 @@ import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; -import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; import org.agrona.collections.Hashing; import org.agrona.collections.Object2IntHashMap; +import org.jetbrains.annotations.NotNull; import org.projectnessie.versioned.storage.common.persist.Backend; import org.projectnessie.versioned.storage.common.persist.PersistFactory; import org.slf4j.Logger; @@ -299,19 +298,28 @@ private R[] resultToArray() { } } - private BoundStatement buildStatement(String cql, Object[] values) { + @NotNull + BoundStatement buildStatement(String cql, Object... values) { PreparedStatement prepared = statements.computeIfAbsent(cql, c -> session.prepare(format(c, config.keyspace()))); return prepared - .bind(values) + .boundStatementBuilder(values) .setTimeout(config.dmlTimeout()) .setConsistencyLevel(LOCAL_QUORUM) - .setSerialConsistencyLevel(LOCAL_SERIAL); + .setSerialConsistencyLevel(LOCAL_SERIAL) + .build(); } - boolean executeCas(String cql, Object... values) { + @NotNull + BoundStatementBuilder newBoundStatementBuilder(String cql) { + PreparedStatement prepared = + statements.computeIfAbsent(cql, c -> session.prepare(format(c, config.keyspace()))); + return prepared.boundStatementBuilder(); + } + + boolean executeCas(BoundStatement stmt) { try { - ResultSet rs = execute(cql, values); + ResultSet rs = execute(stmt); return rs.wasApplied(); } catch (DriverException e) { handleDriverException(e); @@ -319,12 +327,12 @@ boolean executeCas(String cql, Object... values) { } } - ResultSet execute(String cql, Object... values) { - return session.execute(buildStatement(cql, values)); + ResultSet execute(BoundStatement stmt) { + return session.execute(stmt); } - CompletionStage executeAsync(String cql, Object... values) { - return session.executeAsync(buildStatement(cql, values)); + CompletionStage executeAsync(BoundStatement stmt) { + return session.executeAsync(stmt); } void handleDriverException(DriverException e) { @@ -383,55 +391,46 @@ public void setupSchema() { COL_REFS_CREATED_AT, COL_REFS_EXTENDED_INFO, COL_REFS_PREVIOUS) - .collect(Collectors.toSet()), - ImmutableMap.of(COL_REPO_ID, NAME.type(), COL_REFS_NAME, NAME.type())); + .collect(toImmutableSet()), + List.of(COL_REPO_ID, COL_REFS_NAME)); createTableIfNotExists( keyspace.get(), TABLE_OBJS, CREATE_TABLE_OBJS, - Stream.concat( - Stream.of(COL_REPO_ID), Arrays.stream(COLS_OBJS_ALL.split(",")).map(String::trim)) - .collect(Collectors.toSet()), - ImmutableMap.of(COL_REPO_ID, NAME.type(), COL_OBJ_ID, OBJ_ID.type())); + Stream.concat(Stream.of(COL_REPO_ID), COLS_OBJS_ALL.stream()).collect(toImmutableSet()), + List.of(COL_REPO_ID, COL_OBJ_ID)); } private void createTableIfNotExists( KeyspaceMetadata meta, String tableName, String createTable, - Set expectedColumns, - Map expectedPrimaryKey) { + Set expectedColumns, + List expectedPrimaryKey) { Optional table = meta.getTable(tableName); - Object[] types = - Arrays.stream(CqlColumnType.values()).map(CqlColumnType::type).toArray(Object[]::new); createTable = format(createTable, meta.getName()); - createTable = MessageFormat.format(createTable, types); if (table.isPresent()) { - Set columns = - table.get().getColumns().values().stream() - .map(ColumnMetadata::getName) - .map(CqlIdentifier::asInternal) - .collect(Collectors.toSet()); - Map primaryKey = - table.get().getPartitionKey().stream() - .collect( - Collectors.toMap(c -> c.getName().asInternal(), c -> c.getType().toString())); checkState( - primaryKey.equals(expectedPrimaryKey), + checkPrimaryKey(table.get(), expectedPrimaryKey), "Expected primary key columns %s do not match existing primary key columns %s for table '%s'. DDL template:\n%s", - expectedPrimaryKey, - primaryKey, + expectedPrimaryKey.stream() + .map(col -> entry(col.name(), col.type().dataType())) + .collect(toImmutableMap(Entry::getKey, Entry::getValue)), + table.get().getPartitionKey().stream() + .map(col -> entry(col.getName(), col.getType())) + .collect(toImmutableMap(Entry::getKey, Entry::getValue)), tableName, createTable); + checkState( - columns.containsAll(expectedColumns), + checkColumns(table.get(), expectedColumns), "Expected columns %s do not contain all columns %s for table '%s'. DDL template:\n%s", expectedColumns, - columns, + table.get().getColumns().keySet(), tableName, createTable); @@ -444,6 +443,31 @@ private void createTableIfNotExists( session.execute(stmt); } + private boolean checkPrimaryKey(TableMetadata table, List expectedPrimaryKey) { + List partitionKey = table.getPartitionKey(); + if (partitionKey.size() == expectedPrimaryKey.size()) { + for (int i = 0; i < partitionKey.size(); i++) { + ColumnMetadata column = partitionKey.get(i); + CqlColumn expectedColumn = expectedPrimaryKey.get(i); + if (!column.getName().asInternal().equals(expectedColumn.name()) + || !column.getType().equals(expectedColumn.type().dataType())) { + return false; + } + } + return true; + } + return false; + } + + private boolean checkColumns(TableMetadata table, Set expectedColumns) { + for (CqlColumn expectedColumn : expectedColumns) { + if (table.getColumn(expectedColumn.name()).isEmpty()) { + return false; + } + } + return true; + } + @Override public String configInfo() { return "keyspace: " @@ -464,16 +488,16 @@ public void eraseRepositories(Set repositoryIds) { try (LimitedConcurrentRequests requests = new LimitedConcurrentRequests(MAX_CONCURRENT_DELETES)) { - for (Row row : execute(ERASE_REFS_SCAN, repoIdList)) { + for (Row row : execute(buildStatement(ERASE_REFS_SCAN, repoIdList))) { String repoId = row.getString(0); String ref = row.getString(1); - requests.submitted(executeAsync(ERASE_REF, repoId, ref)); + requests.submitted(executeAsync(buildStatement(ERASE_REF, repoId, ref))); } - for (Row row : execute(ERASE_OBJS_SCAN, repoIdList)) { + for (Row row : execute(buildStatement(ERASE_OBJS_SCAN, repoIdList))) { String repoId = row.getString(0); String objId = row.getString(1); - requests.submitted(executeAsync(ERASE_OBJ, repoId, objId)); + requests.submitted(executeAsync(buildStatement(ERASE_OBJ, repoId, objId))); } } // We must ensure that the system clock advances a little, so that C*'s next write-timestamp diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraConstants.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraConstants.java index 269df8b4f2e..7edb380075d 100644 --- a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraConstants.java +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraConstants.java @@ -15,7 +15,14 @@ */ package org.projectnessie.versioned.storage.cassandra; -final class CassandraConstants { +import com.google.common.collect.ImmutableSet; +import java.util.Comparator; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.projectnessie.versioned.storage.cassandra.serializers.ObjSerializers; + +public final class CassandraConstants { static final int SELECT_BATCH_SIZE = 20; static final int MAX_CONCURRENT_BATCH_READS = 20; @@ -24,24 +31,14 @@ final class CassandraConstants { static final String TABLE_REFS = "refs"; static final String TABLE_OBJS = "objs"; - static final String COL_REPO_ID = "repo"; + static final CqlColumn COL_REPO_ID = new CqlColumn("repo", CqlColumnType.NAME); - static final String COL_OBJ_ID = "obj_id"; + static final CqlColumn COL_OBJ_ID = new CqlColumn("obj_id", CqlColumnType.OBJ_ID); static final String DELETE_OBJ = "DELETE FROM %s." + TABLE_OBJS + " WHERE " + COL_REPO_ID + "=? AND " + COL_OBJ_ID + "=?"; - static final String COL_OBJ_TYPE = "obj_type"; - - static final String COLS_COMMIT = - "c_created, c_seq, c_message, c_headers, c_reference_index, c_reference_index_stripes, c_tail, c_secondary_parents, c_incremental_index, c_incomplete_index, c_commit_type"; - static final String COLS_REF = "r_name, r_initial_pointer, r_created_at, r_extended_info"; - static final String COLS_VALUE = "v_content_id, v_payload, v_data"; - static final String COLS_SEGMENTS = "i_stripes"; - static final String COLS_INDEX = "i_index"; - static final String COLS_TAG = "t_message, t_headers, t_signature"; - static final String COLS_STRING = - "s_content_type, s_compression, s_filename, s_predecessors, s_text"; + static final CqlColumn COL_OBJ_TYPE = new CqlColumn("obj_type", CqlColumnType.NAME); - static final String INSERT_OBJ_PREFIX = + public static final String INSERT_OBJ_PREFIX = "INSERT INTO %s." + TABLE_OBJS + " (" @@ -51,46 +48,48 @@ final class CassandraConstants { + ", " + COL_OBJ_TYPE + ", "; - static final String STORE_OBJ_SUFFIX = " IF NOT EXISTS"; - static final String INSERT_OBJ_STRING = - INSERT_OBJ_PREFIX + COLS_STRING + ") VALUES (?,?,?, ?,?,?,?,?)"; - static final String INSERT_OBJ_TAG = INSERT_OBJ_PREFIX + COLS_TAG + ") VALUES (?,?,?, ?,?,?)"; - static final String INSERT_OBJ_INDEX = INSERT_OBJ_PREFIX + COLS_INDEX + ") VALUES (?,?,?, ?)"; - static final String INSERT_OBJ_SEGMENTS = - INSERT_OBJ_PREFIX + COLS_SEGMENTS + ") VALUES (?,?,?, ?)"; - static final String INSERT_OBJ_VALUE = INSERT_OBJ_PREFIX + COLS_VALUE + ") VALUES (?,?,?, ?,?,?)"; - static final String INSERT_OBJ_REF = INSERT_OBJ_PREFIX + COLS_REF + ") VALUES (?,?,?, ?,?,?,?)"; - static final String INSERT_OBJ_COMMIT = - INSERT_OBJ_PREFIX + COLS_COMMIT + ") VALUES (?,?,?, ?,?,?,?,?,?,?,?,?,?,?)"; - static final String CREATE_TABLE_OBJS = - "CREATE TABLE %s." - + TABLE_OBJS - + " (\n " - + COL_REPO_ID - + " {0}, " - + COL_OBJ_ID - + " {1}, " - + COL_OBJ_TYPE - + " {0}" - + ",\n c_created {5}, c_seq {5}, c_message {6}, c_headers {4}, c_reference_index {1}, c_reference_index_stripes {4}, c_tail {2}, c_secondary_parents {2}, c_incremental_index {4}, c_incomplete_index {3}, c_commit_type {0}" - + ",\n r_name {0}, r_initial_pointer {1}, r_created_at {5}, r_extended_info {1}" - + ",\n v_content_id {0}, v_payload {7}, v_data {4}" - + ",\n i_stripes {4}" - + ",\n i_index {4}" - + ",\n t_message {6}, t_headers {4}, t_signature {4}" - + ",\n s_content_type {0}, s_compression {0}, s_filename {0}, s_predecessors {2}, s_text {4}" - + ",\n PRIMARY KEY ((" - + COL_REPO_ID - + ", " - + COL_OBJ_ID - + "))\n )"; - static final String COL_REFS_NAME = "ref_name"; - static final String COL_REFS_POINTER = "pointer"; - static final String COL_REFS_DELETED = "deleted"; - static final String COL_REFS_CREATED_AT = "created_at"; - static final String COL_REFS_EXTENDED_INFO = "ext_info"; - static final String COL_REFS_PREVIOUS = "prev_ptr"; + public static final String INSERT_OBJ_VALUES = + ") VALUES (:" + COL_REPO_ID + ", :" + COL_OBJ_ID + ", :" + COL_OBJ_TYPE + ", "; + public static final String STORE_OBJ_SUFFIX = " IF NOT EXISTS"; + + static final Set COLS_OBJS_ALL = + Stream.concat( + Stream.of(COL_OBJ_ID, COL_OBJ_TYPE), + ObjSerializers.ALL_SERIALIZERS.stream() + .flatMap(serializer -> serializer.columns().stream()) + .sorted(Comparator.comparing(CqlColumn::name))) + .collect(ImmutableSet.toImmutableSet()); + + static final String CREATE_TABLE_OBJS; + + static { + StringBuilder sb = + new StringBuilder() + .append("CREATE TABLE %s.") + .append(TABLE_OBJS) + .append(" (\n ") + .append(COL_REPO_ID) + .append(" ") + .append(CqlColumnType.NAME.cqlName()); + for (CqlColumn col : COLS_OBJS_ALL) { + sb.append(",\n ").append(col.name()).append(" ").append(col.type().cqlName()); + } + sb.append(",\n PRIMARY KEY ((") + .append(COL_REPO_ID) + .append(", ") + .append(COL_OBJ_ID) + .append("))\n )"); + CREATE_TABLE_OBJS = sb.toString(); + } + + static final CqlColumn COL_REFS_NAME = new CqlColumn("ref_name", CqlColumnType.NAME); + static final CqlColumn COL_REFS_POINTER = new CqlColumn("pointer", CqlColumnType.OBJ_ID); + static final CqlColumn COL_REFS_DELETED = new CqlColumn("deleted", CqlColumnType.BOOL); + static final CqlColumn COL_REFS_CREATED_AT = new CqlColumn("created_at", CqlColumnType.BIGINT); + static final CqlColumn COL_REFS_EXTENDED_INFO = new CqlColumn("ext_info", CqlColumnType.OBJ_ID); + static final CqlColumn COL_REFS_PREVIOUS = new CqlColumn("prev_ptr", CqlColumnType.VARBINARY); + static final String UPDATE_REFERENCE_POINTER = "UPDATE %s." + TABLE_REFS @@ -188,70 +187,37 @@ final class CassandraConstants { + TABLE_REFS + "\n (\n " + COL_REPO_ID - + " {0}, " + + " " + + COL_REPO_ID.type().cqlName() + + ",\n " + COL_REFS_NAME - + " {0}, " + + " " + + COL_REFS_NAME.type().cqlName() + + ",\n " + COL_REFS_POINTER - + " {1}, " + + " " + + COL_REFS_POINTER.type().cqlName() + + ",\n " + COL_REFS_DELETED - + " {3}, " + + " " + + COL_REFS_DELETED.type().cqlName() + + ",\n " + COL_REFS_CREATED_AT - + " {5}, " + + " " + + COL_REFS_CREATED_AT.type().cqlName() + + ",\n " + COL_REFS_EXTENDED_INFO - + " {1}, " + + " " + + COL_REFS_EXTENDED_INFO.type().cqlName() + + ",\n " + COL_REFS_PREVIOUS - + " {4}, " - + "\n PRIMARY KEY ((" + + " " + + COL_REFS_PREVIOUS.type().cqlName() + + ",\n PRIMARY KEY ((" + COL_REPO_ID + ", " + COL_REFS_NAME + "))\n )"; - static final String COLS_OBJS_ALL = - COL_OBJ_ID - + ", " - + COL_OBJ_TYPE - + ", " - + COLS_COMMIT - + ", " - + COLS_REF - + ", " - + COLS_VALUE - + ", " - + COLS_SEGMENTS - + ", " - + COLS_INDEX - + ", " - + COLS_TAG - + ", " - + COLS_STRING; - static final int COL_COMMIT_CREATED = 2; // obj_id + obj_type before this column - static final int COL_COMMIT_SEQ = COL_COMMIT_CREATED + 1; - static final int COL_COMMIT_MESSAGE = COL_COMMIT_SEQ + 1; - static final int COL_COMMIT_HEADERS = COL_COMMIT_MESSAGE + 1; - static final int COL_COMMIT_REFERENCE_INDEX = COL_COMMIT_HEADERS + 1; - static final int COL_COMMIT_REFERENCE_INDEX_STRIPES = COL_COMMIT_REFERENCE_INDEX + 1; - static final int COL_COMMIT_TAIL = COL_COMMIT_REFERENCE_INDEX_STRIPES + 1; - static final int COL_COMMIT_SECONDARY_PARENTS = COL_COMMIT_TAIL + 1; - static final int COL_COMMIT_INCREMENTAL_INDEX = COL_COMMIT_SECONDARY_PARENTS + 1; - static final int COL_COMMIT_INCOMPLETE_INDEX = COL_COMMIT_INCREMENTAL_INDEX + 1; - static final int COL_COMMIT_TYPE = COL_COMMIT_INCOMPLETE_INDEX + 1; - static final int COL_REF_NAME = COL_COMMIT_TYPE + 1; - static final int COL_REF_INITIAL_POINTER = COL_REF_NAME + 1; - static final int COL_REF_CREATED_AT = COL_REF_INITIAL_POINTER + 1; - static final int COL_REF_EXTENDED_INFO = COL_REF_CREATED_AT + 1; - static final int COL_VALUE_CONTENT_ID = COL_REF_EXTENDED_INFO + 1; - static final int COL_VALUE_PAYLOAD = COL_VALUE_CONTENT_ID + 1; - static final int COL_VALUE_DATA = COL_VALUE_PAYLOAD + 1; - static final int COL_SEGMENTS_STRIPES = COL_VALUE_DATA + 1; - static final int COL_INDEX_INDEX = COL_SEGMENTS_STRIPES + 1; - static final int COL_TAG_MESSAGE = COL_INDEX_INDEX + 1; - static final int COL_TAG_HEADERS = COL_TAG_MESSAGE + 1; - static final int COL_TAG_SIGNATURE = COL_TAG_HEADERS + 1; - static final int COL_STRING_CONTENT_TYPE = COL_TAG_SIGNATURE + 1; - static final int COL_STRING_COMPRESSION = COL_STRING_CONTENT_TYPE + 1; - static final int COL_STRING_FILENAME = COL_STRING_COMPRESSION + 1; - static final int COL_STRING_PREDECESSORS = COL_STRING_FILENAME + 1; - static final int COL_STRING_TEXT = COL_STRING_PREDECESSORS + 1; static final String FETCH_OBJ_TYPE = "SELECT " @@ -266,7 +232,7 @@ final class CassandraConstants { static final String FIND_OBJS = "SELECT " - + COLS_OBJS_ALL + + COLS_OBJS_ALL.stream().map(CqlColumn::name).collect(Collectors.joining(", ")) + " FROM %s." + TABLE_OBJS + " WHERE " @@ -277,7 +243,7 @@ final class CassandraConstants { static final String SCAN_OBJS = "SELECT " - + COLS_OBJS_ALL + + COLS_OBJS_ALL.stream().map(CqlColumn::name).collect(Collectors.joining(", ")) + " FROM %s." + TABLE_OBJS + " WHERE " diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraPersist.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraPersist.java index b0811b7ef52..410beadfc80 100644 --- a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraPersist.java +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraPersist.java @@ -16,117 +16,59 @@ package org.projectnessie.versioned.storage.cassandra; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; -import static org.projectnessie.nessie.relocated.protobuf.UnsafeByteOperations.unsafeWrap; +import static java.util.Objects.requireNonNull; import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.ADD_REFERENCE; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_COMMIT_CREATED; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_COMMIT_HEADERS; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_COMMIT_INCOMPLETE_INDEX; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_COMMIT_INCREMENTAL_INDEX; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_COMMIT_MESSAGE; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_COMMIT_REFERENCE_INDEX; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_COMMIT_REFERENCE_INDEX_STRIPES; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_COMMIT_SECONDARY_PARENTS; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_COMMIT_SEQ; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_COMMIT_TAIL; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_COMMIT_TYPE; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_INDEX_INDEX; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_REF_CREATED_AT; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_REF_EXTENDED_INFO; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_REF_INITIAL_POINTER; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_REF_NAME; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_SEGMENTS_STRIPES; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_STRING_COMPRESSION; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_STRING_CONTENT_TYPE; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_STRING_FILENAME; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_STRING_PREDECESSORS; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_STRING_TEXT; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_TAG_HEADERS; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_TAG_MESSAGE; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_TAG_SIGNATURE; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_VALUE_CONTENT_ID; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_VALUE_DATA; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_VALUE_PAYLOAD; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_OBJ_ID; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_OBJ_TYPE; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.COL_REPO_ID; import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.DELETE_OBJ; import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.FETCH_OBJ_TYPE; import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.FIND_OBJS; import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.FIND_REFERENCES; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_COMMIT; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_INDEX; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_REF; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_SEGMENTS; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_STRING; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_TAG; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_VALUE; import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.MARK_REFERENCE_AS_DELETED; import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.MAX_CONCURRENT_STORES; import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.PURGE_REFERENCE; import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.SCAN_OBJS; -import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.STORE_OBJ_SUFFIX; import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.UPDATE_REFERENCE_POINTER; -import static org.projectnessie.versioned.storage.common.indexes.StoreKey.keyFromString; -import static org.projectnessie.versioned.storage.common.objtypes.ContentValueObj.contentValue; -import static org.projectnessie.versioned.storage.common.objtypes.IndexObj.index; -import static org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj.indexSegments; -import static org.projectnessie.versioned.storage.common.objtypes.IndexStripe.indexStripe; -import static org.projectnessie.versioned.storage.common.objtypes.RefObj.ref; -import static org.projectnessie.versioned.storage.common.objtypes.StringObj.stringData; -import static org.projectnessie.versioned.storage.common.objtypes.TagObj.tag; -import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromByteBuffer; -import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromString; -import static org.projectnessie.versioned.storage.serialize.ProtoSerialization.deserializePreviousPointers; +import static org.projectnessie.versioned.storage.cassandra.CassandraSerde.deserializeObjId; +import static org.projectnessie.versioned.storage.cassandra.CassandraSerde.serializeObjId; import static org.projectnessie.versioned.storage.serialize.ProtoSerialization.serializePreviousPointers; import com.datastax.oss.driver.api.core.DriverException; import com.datastax.oss.driver.api.core.cql.AsyncResultSet; +import com.datastax.oss.driver.api.core.cql.BoundStatement; +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; import com.datastax.oss.driver.api.core.cql.Row; import com.google.common.collect.AbstractIterator; -import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.EnumMap; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.CompletionStage; import java.util.concurrent.atomic.AtomicIntegerArray; -import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import org.projectnessie.nessie.relocated.protobuf.ByteString; import org.projectnessie.versioned.storage.cassandra.CassandraBackend.BatchedQuery; +import org.projectnessie.versioned.storage.cassandra.serializers.ObjSerializer; +import org.projectnessie.versioned.storage.cassandra.serializers.ObjSerializers; import org.projectnessie.versioned.storage.common.config.StoreConfig; import org.projectnessie.versioned.storage.common.exceptions.ObjNotFoundException; import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; import org.projectnessie.versioned.storage.common.exceptions.RefAlreadyExistsException; import org.projectnessie.versioned.storage.common.exceptions.RefConditionFailedException; import org.projectnessie.versioned.storage.common.exceptions.RefNotFoundException; -import org.projectnessie.versioned.storage.common.objtypes.CommitHeaders; -import org.projectnessie.versioned.storage.common.objtypes.CommitObj; -import org.projectnessie.versioned.storage.common.objtypes.CommitType; -import org.projectnessie.versioned.storage.common.objtypes.Compression; -import org.projectnessie.versioned.storage.common.objtypes.ContentValueObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexStripe; -import org.projectnessie.versioned.storage.common.objtypes.RefObj; -import org.projectnessie.versioned.storage.common.objtypes.StringObj; -import org.projectnessie.versioned.storage.common.objtypes.TagObj; import org.projectnessie.versioned.storage.common.persist.CloseableIterator; import org.projectnessie.versioned.storage.common.persist.Obj; import org.projectnessie.versioned.storage.common.persist.ObjId; import org.projectnessie.versioned.storage.common.persist.ObjType; +import org.projectnessie.versioned.storage.common.persist.ObjTypes; import org.projectnessie.versioned.storage.common.persist.Persist; import org.projectnessie.versioned.storage.common.persist.Reference; -import org.projectnessie.versioned.storage.common.proto.StorageTypes.HeaderEntry; -import org.projectnessie.versioned.storage.common.proto.StorageTypes.Headers; -import org.projectnessie.versioned.storage.common.proto.StorageTypes.Stripe; -import org.projectnessie.versioned.storage.common.proto.StorageTypes.Stripes; public class CassandraPersist implements Persist { @@ -163,8 +105,10 @@ public Reference fetchReference(@Nonnull @jakarta.annotation.Nonnull String name public Reference[] fetchReferences(@Nonnull @jakarta.annotation.Nonnull String[] names) { try (BatchedQuery batchedQuery = backend.newBatchedQuery( - keys -> backend.executeAsync(FIND_REFERENCES, config.repositoryId(), keys), - CassandraPersist::deserializeReference, + keys -> + backend.executeAsync( + backend.buildStatement(FIND_REFERENCES, config.repositoryId(), keys)), + CassandraSerde::deserializeReference, Reference::name, names.length, Reference.class)) { @@ -190,15 +134,17 @@ public Reference addReference(@Nonnull @jakarta.annotation.Nonnull Reference ref byte[] serializedPreviousPointers = serializePreviousPointers(reference.previousPointers()); ByteBuffer previous = serializedPreviousPointers != null ? ByteBuffer.wrap(serializedPreviousPointers) : null; - if (backend.executeCas( - ADD_REFERENCE, - config.repositoryId(), - reference.name(), - serializeObjId(reference.pointer()), - reference.deleted(), - reference.createdAtMicros(), - serializeObjId(reference.extendedInfoObj()), - previous)) { + BoundStatement stmt = + backend.buildStatement( + ADD_REFERENCE, + config.repositoryId(), + reference.name(), + serializeObjId(reference.pointer()), + reference.deleted(), + reference.createdAtMicros(), + serializeObjId(reference.extendedInfoObj()), + previous); + if (backend.executeCas(stmt)) { return reference; } throw new RefAlreadyExistsException(fetchReference(reference.name())); @@ -209,15 +155,17 @@ public Reference addReference(@Nonnull @jakarta.annotation.Nonnull Reference ref @Override public Reference markReferenceAsDeleted(@Nonnull @jakarta.annotation.Nonnull Reference reference) throws RefNotFoundException, RefConditionFailedException { - if (backend.executeCas( - MARK_REFERENCE_AS_DELETED, - true, - config().repositoryId(), - reference.name(), - serializeObjId(reference.pointer()), - false, - reference.createdAtMicros(), - serializeObjId(reference.extendedInfoObj()))) { + BoundStatement stmt = + backend.buildStatement( + MARK_REFERENCE_AS_DELETED, + true, + config().repositoryId(), + reference.name(), + serializeObjId(reference.pointer()), + false, + reference.createdAtMicros(), + serializeObjId(reference.extendedInfoObj())); + if (backend.executeCas(stmt)) { return reference.withDeleted(true); } @@ -231,14 +179,16 @@ public Reference markReferenceAsDeleted(@Nonnull @jakarta.annotation.Nonnull Ref @Override public void purgeReference(@Nonnull @jakarta.annotation.Nonnull Reference reference) throws RefNotFoundException, RefConditionFailedException { - if (!backend.executeCas( - PURGE_REFERENCE, - config().repositoryId(), - reference.name(), - serializeObjId(reference.pointer()), - true, - reference.createdAtMicros(), - serializeObjId(reference.extendedInfoObj()))) { + BoundStatement stmt = + backend.buildStatement( + PURGE_REFERENCE, + config().repositoryId(), + reference.name(), + serializeObjId(reference.pointer()), + true, + reference.createdAtMicros(), + serializeObjId(reference.extendedInfoObj())); + if (!backend.executeCas(stmt)) { Reference ref = fetchReference(reference.name()); if (ref == null) { throw new RefNotFoundException(reference); @@ -258,16 +208,18 @@ public Reference updateReferencePointer( byte[] serializedPreviousPointers = serializePreviousPointers(updated.previousPointers()); ByteBuffer previous = serializedPreviousPointers != null ? ByteBuffer.wrap(serializedPreviousPointers) : null; - if (!backend.executeCas( - UPDATE_REFERENCE_POINTER, - serializeObjId(newPointer), - previous, - config().repositoryId(), - reference.name(), - serializeObjId(reference.pointer()), - false, - reference.createdAtMicros(), - serializeObjId(reference.extendedInfoObj()))) { + BoundStatement stmt = + backend.buildStatement( + UPDATE_REFERENCE_POINTER, + serializeObjId(newPointer), + previous, + config().repositoryId(), + reference.name(), + serializeObjId(reference.pointer()), + false, + reference.createdAtMicros(), + serializeObjId(reference.extendedInfoObj())); + if (!backend.executeCas(stmt)) { Reference ref = fetchReference(reference.name()); if (ref == null) { throw new RefNotFoundException(reference); @@ -304,13 +256,13 @@ public Obj fetchObj(@Nonnull @jakarta.annotation.Nonnull ObjId id) throws ObjNot @jakarta.annotation.Nonnull public ObjType fetchObjType(@Nonnull @jakarta.annotation.Nonnull ObjId id) throws ObjNotFoundException { - Row row = - backend - .execute(FETCH_OBJ_TYPE, config.repositoryId(), singletonList(serializeObjId(id))) - .one(); + BoundStatement stmt = + backend.buildStatement( + FETCH_OBJ_TYPE, config.repositoryId(), singletonList(serializeObjId(id))); + Row row = backend.execute(stmt).one(); if (row != null) { - String objType = row.getString(0); - return ObjType.valueOf(objType); + String objType = requireNonNull(row.getString(0)); + return ObjTypes.forName(objType); } throw new ObjNotFoundException(id); } @@ -333,12 +285,15 @@ Obj[] fetchObjs( queryIds -> queryIds.stream().map(ObjId::toString).collect(Collectors.toList()); Function, CompletionStage> queryFunc = - keys -> backend.executeAsync(FIND_OBJS, config.repositoryId(), idsToStrings.apply(keys)); + keys -> + backend.executeAsync( + backend.buildStatement(FIND_OBJS, config.repositoryId(), idsToStrings.apply(keys))); Function rowMapper = row -> { - ObjType objType = ObjType.valueOf(row.getString(1)); - return deserializeObj(row, objType); + ObjType objType = ObjTypes.forName(requireNonNull(row.getString(COL_OBJ_TYPE.name()))); + ObjId id = deserializeObjId(row.getString(COL_OBJ_ID.name())); + return ObjSerializers.forType(objType).deserialize(row, id); }; Obj[] r; @@ -358,7 +313,7 @@ Obj[] fetchObjs( List notFound = null; for (int i = 0; i < ids.length; i++) { ObjId id = ids[i]; - if (id != null && (r[i] == null || (type != null && r[i].type() != type))) { + if (id != null && (r[i] == null || (type != null && !r[i].type().equals(type)))) { if (notFound == null) { notFound = new ArrayList<>(); } @@ -372,23 +327,12 @@ Obj[] fetchObjs( return r; } - private Obj deserializeObj(Row row, ObjType type) { - ObjId id = deserializeObjId(row.getString(0)); - - @SuppressWarnings("rawtypes") - StoreObjDesc objDesc = STORE_OBJ_TYPE.get(type); - checkState(objDesc != null, "Cannot deserialize object type %s", type); - return objDesc.deserialize(row, id); - } - @Override public boolean storeObj( @Nonnull @jakarta.annotation.Nonnull Obj obj, boolean ignoreSoftSizeRestrictions) throws ObjTooLargeException { return writeSingleObj( - obj, - ignoreSoftSizeRestrictions, - (storeObj, values) -> backend.executeCas(storeObj.cql(true), values)); + obj, false, ignoreSoftSizeRestrictions, (serializer, stmt) -> backend.executeCas(stmt)); } @Nonnull @@ -396,23 +340,23 @@ public boolean storeObj( @Override public boolean[] storeObjs(@Nonnull @jakarta.annotation.Nonnull Obj[] objs) throws ObjTooLargeException { - return persistObjs(objs, true); + return persistObjs(objs, false); } @Override public void upsertObj(@Nonnull @jakarta.annotation.Nonnull Obj obj) throws ObjTooLargeException { - writeSingleObj(obj, false, (storeObj, values) -> backend.execute(storeObj.cql(false), values)); + writeSingleObj(obj, true, false, (serializer, stmt) -> backend.execute(stmt)); } @Override public void upsertObjs(@Nonnull @jakarta.annotation.Nonnull Obj[] objs) throws ObjTooLargeException { - persistObjs(objs, false); + persistObjs(objs, true); } @Nonnull @jakarta.annotation.Nonnull - private boolean[] persistObjs(@Nonnull @jakarta.annotation.Nonnull Obj[] objs, boolean insert) + private boolean[] persistObjs(@Nonnull @jakarta.annotation.Nonnull Obj[] objs, boolean upsert) throws ObjTooLargeException { AtomicIntegerArray results = new AtomicIntegerArray(objs.length); @@ -424,11 +368,12 @@ private boolean[] persistObjs(@Nonnull @jakarta.annotation.Nonnull Obj[] objs, b int idx = i; writeSingleObj( o, + upsert, false, - (storeObj, values) -> { + (serializer, stmt) -> { CompletionStage cs = backend - .executeAsync(storeObj.cql(insert), values) + .executeAsync(stmt) .handle( (resultSet, e) -> { if (e != null) { @@ -463,44 +408,42 @@ private boolean[] persistObjs(@Nonnull @jakarta.annotation.Nonnull Obj[] objs, b } @FunctionalInterface - interface WriteSingleObj { - R apply(StoreObjDesc storeObj, Object[] values); + private interface WriteSingleObj { + R apply(ObjSerializer serializer, BoundStatement stmt); } private R writeSingleObj( @Nonnull @jakarta.annotation.Nonnull Obj obj, + boolean upsert, boolean ignoreSoftSizeRestrictions, WriteSingleObj consumer) throws ObjTooLargeException { ObjId id = obj.id(); ObjType type = obj.type(); - StoreObjDesc storeObj = storeObjForObj(type); + ObjSerializer serializer = ObjSerializers.forType(type); - List values = new ArrayList<>(); - values.add(config.repositoryId()); - values.add(serializeObjId(id)); - values.add(type.name()); - storeObj.store( - values::add, + BoundStatementBuilder stmt = + backend + .newBoundStatementBuilder(serializer.insertCql(upsert)) + .setString(COL_REPO_ID.name(), config.repositoryId()) + .setString(COL_OBJ_ID.name(), serializeObjId(id)) + .setString(COL_OBJ_TYPE.name(), type.name()); + + serializer.serialize( obj, + stmt, ignoreSoftSizeRestrictions ? Integer.MAX_VALUE : effectiveIncrementalIndexSizeLimit(), ignoreSoftSizeRestrictions ? Integer.MAX_VALUE : effectiveIndexSegmentSizeLimit()); - return consumer.apply(storeObj, values.toArray(new Object[0])); - } - - @SuppressWarnings("unchecked") - private static StoreObjDesc storeObjForObj(ObjType type) { - @SuppressWarnings("rawtypes") - StoreObjDesc storeObj = STORE_OBJ_TYPE.get(type); - checkArgument(storeObj != null, "Cannot serialize object type %s ", type); - return storeObj; + return consumer.apply(serializer, stmt.build()); } @Override public void deleteObj(@Nonnull @jakarta.annotation.Nonnull ObjId id) { - backend.execute(DELETE_OBJ, config.repositoryId(), serializeObjId(id)); + BoundStatement stmt = + backend.buildStatement(DELETE_OBJ, config.repositoryId(), serializeObjId(id)); + backend.execute(stmt); } @Override @@ -510,7 +453,8 @@ public void deleteObjs(@Nonnull @jakarta.annotation.Nonnull ObjId[] ids) { String repoId = config.repositoryId(); for (ObjId id : ids) { if (id != null) { - requests.submitted(backend.executeAsync(DELETE_OBJ, repoId, serializeObjId(id))); + BoundStatement stmt = backend.buildStatement(DELETE_OBJ, repoId, serializeObjId(id)); + requests.submitted(backend.executeAsync(stmt)); } } } @@ -529,366 +473,6 @@ public CloseableIterator scanAllObjects( return new ScanAllObjectsIterator(returnedObjTypes); } - private abstract static class StoreObjDesc { - private final String insertCql; - - StoreObjDesc(String insertCql) { - this.insertCql = insertCql; - } - - abstract O deserialize(Row row, ObjId id); - - abstract void store( - Consumer values, O obj, int incrementalIndexLimit, int maxSerializedIndexSize) - throws ObjTooLargeException; - - String cql(boolean insert) { - if (insert) { - return insertCql + STORE_OBJ_SUFFIX; - } - return insertCql; - } - } - - private static final Map> STORE_OBJ_TYPE = new EnumMap<>(ObjType.class); - - static { - STORE_OBJ_TYPE.put( - ObjType.COMMIT, - new StoreObjDesc(INSERT_OBJ_COMMIT) { - @Override - void store( - Consumer values, - CommitObj obj, - int incrementalIndexLimit, - int maxSerializedIndexSize) - throws ObjTooLargeException { - values.accept(obj.created()); - values.accept(obj.seq()); - values.accept(obj.message()); - - Headers.Builder hb = Headers.newBuilder(); - for (String h : obj.headers().keySet()) { - hb.addHeaders( - HeaderEntry.newBuilder().setName(h).addAllValues(obj.headers().getAll(h))); - } - values.accept(ByteBuffer.wrap(hb.build().toByteArray())); - - values.accept(serializeObjId(obj.referenceIndex())); - - Stripes.Builder b = Stripes.newBuilder(); - obj.referenceIndexStripes().stream() - .map( - s -> - Stripe.newBuilder() - .setFirstKey(s.firstKey().rawString()) - .setLastKey(s.lastKey().rawString()) - .setSegment(s.segment().asBytes())) - .forEach(b::addStripes); - values.accept(b.build().toByteString().asReadOnlyByteBuffer()); - - values.accept(serializeObjIds(obj.tail())); - values.accept(serializeObjIds(obj.secondaryParents())); - - ByteString index = obj.incrementalIndex(); - if (index.size() > incrementalIndexLimit) { - throw new ObjTooLargeException(index.size(), incrementalIndexLimit); - } - values.accept(index.asReadOnlyByteBuffer()); - - values.accept(obj.incompleteIndex()); - values.accept(obj.commitType().name()); - } - - @Override - CommitObj deserialize(Row row, ObjId id) { - CommitObj.Builder b = - CommitObj.commitBuilder() - .id(id) - .created(row.getLong(COL_COMMIT_CREATED)) - .seq(row.getLong(COL_COMMIT_SEQ)) - .message(row.getString(COL_COMMIT_MESSAGE)) - .referenceIndex(deserializeObjId(row.getString(COL_COMMIT_REFERENCE_INDEX))) - .incrementalIndex(deserializeBytes(row, COL_COMMIT_INCREMENTAL_INDEX)) - .incompleteIndex(row.getBoolean(COL_COMMIT_INCOMPLETE_INDEX)) - .commitType(CommitType.valueOf(row.getString(COL_COMMIT_TYPE))); - deserializeObjIds(row, COL_COMMIT_TAIL, b::addTail); - deserializeObjIds(row, COL_COMMIT_SECONDARY_PARENTS, b::addSecondaryParents); - - try { - CommitHeaders.Builder h = CommitHeaders.newCommitHeaders(); - Headers headers = Headers.parseFrom(row.getByteBuffer(COL_COMMIT_HEADERS)); - for (HeaderEntry e : headers.getHeadersList()) { - for (String v : e.getValuesList()) { - h.add(e.getName(), v); - } - } - b.headers(h.build()); - } catch (IOException e) { - throw new RuntimeException(e); - } - - try { - Stripes stripes = - Stripes.parseFrom(row.getByteBuffer(COL_COMMIT_REFERENCE_INDEX_STRIPES)); - stripes.getStripesList().stream() - .map( - s -> - indexStripe( - keyFromString(s.getFirstKey()), - keyFromString(s.getLastKey()), - objIdFromByteBuffer(s.getSegment().asReadOnlyByteBuffer()))) - .forEach(b::addReferenceIndexStripes); - } catch (IOException e) { - throw new RuntimeException(e); - } - - return b.build(); - } - }); - STORE_OBJ_TYPE.put( - ObjType.REF, - new StoreObjDesc(INSERT_OBJ_REF) { - @Override - void store( - Consumer values, - RefObj obj, - int incrementalIndexLimit, - int maxSerializedIndexSize) { - values.accept(obj.name()); - values.accept(serializeObjId(obj.initialPointer())); - values.accept(obj.createdAtMicros()); - values.accept(serializeObjId(obj.extendedInfoObj())); - } - - @Override - RefObj deserialize(Row row, ObjId id) { - return ref( - id, - row.getString(COL_REF_NAME), - deserializeObjId(row.getString(COL_REF_INITIAL_POINTER)), - row.getLong(COL_REF_CREATED_AT), - deserializeObjId(row.getString(COL_REF_EXTENDED_INFO))); - } - }); - STORE_OBJ_TYPE.put( - ObjType.VALUE, - new StoreObjDesc(INSERT_OBJ_VALUE) { - @Override - void store( - Consumer values, - ContentValueObj obj, - int incrementalIndexLimit, - int maxSerializedIndexSize) { - values.accept(obj.contentId()); - values.accept(obj.payload()); - values.accept(obj.data().asReadOnlyByteBuffer()); - } - - @Override - ContentValueObj deserialize(Row row, ObjId id) { - ByteString value = deserializeBytes(row, COL_VALUE_DATA); - if (value != null) { - return contentValue( - id, row.getString(COL_VALUE_CONTENT_ID), row.getInt(COL_VALUE_PAYLOAD), value); - } - throw new IllegalStateException("Data value of obj " + id + " of type VALUE is null"); - } - }); - STORE_OBJ_TYPE.put( - ObjType.INDEX_SEGMENTS, - new StoreObjDesc(INSERT_OBJ_SEGMENTS) { - @Override - void store( - Consumer values, - IndexSegmentsObj obj, - int incrementalIndexLimit, - int maxSerializedIndexSize) { - Stripes.Builder b = Stripes.newBuilder(); - obj.stripes().stream() - .map( - s -> - Stripe.newBuilder() - .setFirstKey(s.firstKey().rawString()) - .setLastKey(s.lastKey().rawString()) - .setSegment(s.segment().asBytes())) - .forEach(b::addStripes); - values.accept(b.build().toByteString().asReadOnlyByteBuffer()); - } - - @Override - IndexSegmentsObj deserialize(Row row, ObjId id) { - try { - Stripes stripes = Stripes.parseFrom(row.getByteBuffer(COL_SEGMENTS_STRIPES)); - List stripeList = - stripes.getStripesList().stream() - .map( - s -> - indexStripe( - keyFromString(s.getFirstKey()), - keyFromString(s.getLastKey()), - objIdFromByteBuffer(s.getSegment().asReadOnlyByteBuffer()))) - .collect(Collectors.toList()); - return indexSegments(id, stripeList); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }); - STORE_OBJ_TYPE.put( - ObjType.INDEX, - new StoreObjDesc(INSERT_OBJ_INDEX) { - @Override - void store( - Consumer values, - IndexObj obj, - int incrementalIndexLimit, - int maxSerializedIndexSize) - throws ObjTooLargeException { - ByteString index = obj.index(); - if (index.size() > maxSerializedIndexSize) { - throw new ObjTooLargeException(index.size(), maxSerializedIndexSize); - } - values.accept(index.asReadOnlyByteBuffer()); - } - - @Override - IndexObj deserialize(Row row, ObjId id) { - ByteString indexValue = deserializeBytes(row, COL_INDEX_INDEX); - if (indexValue != null) { - return index(id, indexValue); - } - throw new IllegalStateException("Index value of obj " + id + " of type INDEX is null"); - } - }); - STORE_OBJ_TYPE.put( - ObjType.TAG, - new StoreObjDesc(INSERT_OBJ_TAG) { - @Override - void store( - Consumer values, - TagObj obj, - int incrementalIndexLimit, - int maxSerializedIndexSize) { - values.accept(obj.message()); - Headers.Builder hb = Headers.newBuilder(); - CommitHeaders headers = obj.headers(); - if (headers != null) { - for (String h : headers.keySet()) { - hb.addHeaders(HeaderEntry.newBuilder().setName(h).addAllValues(headers.getAll(h))); - } - } - values.accept(ByteBuffer.wrap(hb.build().toByteArray())); - ByteString signature = obj.signature(); - values.accept(signature != null ? signature.asReadOnlyByteBuffer() : null); - } - - @Override - TagObj deserialize(Row row, ObjId id) { - CommitHeaders tagHeaders = null; - try { - Headers headers = Headers.parseFrom(row.getByteBuffer(COL_TAG_HEADERS)); - if (headers.getHeadersCount() > 0) { - CommitHeaders.Builder h = CommitHeaders.newCommitHeaders(); - for (HeaderEntry e : headers.getHeadersList()) { - for (String v : e.getValuesList()) { - h.add(e.getName(), v); - } - } - tagHeaders = h.build(); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - - return tag( - id, - row.getString(COL_TAG_MESSAGE), - tagHeaders, - deserializeBytes(row, COL_TAG_SIGNATURE)); - } - }); - STORE_OBJ_TYPE.put( - ObjType.STRING, - new StoreObjDesc(INSERT_OBJ_STRING) { - @Override - void store( - Consumer values, - StringObj obj, - int incrementalIndexLimit, - int maxSerializedIndexSize) { - values.accept(obj.contentType()); - values.accept(obj.compression().name()); - values.accept(obj.filename()); - values.accept(serializeObjIds(obj.predecessors())); - values.accept(obj.text().asReadOnlyByteBuffer()); - } - - @Override - StringObj deserialize(Row row, ObjId id) { - return stringData( - id, - row.getString(COL_STRING_CONTENT_TYPE), - Compression.valueOf(row.getString(COL_STRING_COMPRESSION)), - row.getString(COL_STRING_FILENAME), - deserializeObjIds(row, COL_STRING_PREDECESSORS), - deserializeBytes(row, COL_STRING_TEXT)); - } - }); - } - - private static ByteString deserializeBytes(Row row, int idx) { - ByteBuffer bytes = row.getByteBuffer(idx); - return bytes != null ? unsafeWrap(bytes) : null; - } - - private static Reference deserializeReference(Row row) { - ByteBuffer previous = row.getByteBuffer(5); - byte[] bytes; - if (previous != null) { - bytes = new byte[previous.remaining()]; - previous.get(bytes); - } else { - bytes = null; - } - return Reference.reference( - row.getString(0), - deserializeObjId(row.getString(1)), - row.getBoolean(2), - row.getLong(3), - deserializeObjId(row.getString(4)), - deserializePreviousPointers(bytes)); - } - - private static ObjId deserializeObjId(String id) { - return id != null ? objIdFromString(id) : null; - } - - private static String serializeObjId(ObjId id) { - return id != null ? id.toString() : null; - } - - @SuppressWarnings("SameParameterValue") - private static List deserializeObjIds(Row row, int col) { - List r = new ArrayList<>(); - deserializeObjIds(row, col, r::add); - return r; - } - - private static void deserializeObjIds(Row row, int col, Consumer consumer) { - List s = row.getList(col, String.class); - if (s == null || s.isEmpty()) { - return; - } - s.stream().map(ObjId::objIdFromString).forEach(consumer); - } - - private static List serializeObjIds(List values) { - return (values != null && !values.isEmpty()) - ? values.stream().map(ObjId::toString).collect(Collectors.toList()) - : null; - } - private class ScanAllObjectsIterator extends AbstractIterator implements CloseableIterator { @@ -897,7 +481,8 @@ private class ScanAllObjectsIterator extends AbstractIterator ScanAllObjectsIterator(Set returnedObjTypes) { this.returnedObjTypes = returnedObjTypes; - rs = backend.execute(SCAN_OBJS, config.repositoryId()).iterator(); + BoundStatement stmt = backend.buildStatement(SCAN_OBJS, config.repositoryId()); + rs = backend.execute(stmt).iterator(); } @Override @@ -913,12 +498,13 @@ protected Obj computeNext() { } Row row = rs.next(); - ObjType type = ObjType.valueOf(row.getString(1)); + ObjType type = ObjTypes.forName(requireNonNull(row.getString(1))); if (!returnedObjTypes.contains(type)) { continue; } - return deserializeObj(row, type); + ObjId id = deserializeObjId(row.getString(COL_OBJ_ID.name())); + return ObjSerializers.forType(type).deserialize(row, id); } } } diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraSerde.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraSerde.java new file mode 100644 index 00000000000..ac47dff38e9 --- /dev/null +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CassandraSerde.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.cassandra; + +import static org.projectnessie.nessie.relocated.protobuf.UnsafeByteOperations.unsafeWrap; +import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromString; +import static org.projectnessie.versioned.storage.serialize.ProtoSerialization.deserializePreviousPointers; + +import com.datastax.oss.driver.api.core.cql.Row; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.common.persist.Reference; + +public final class CassandraSerde { + + private CassandraSerde() {} + + public static ByteString deserializeBytes(Row row, String col) { + ByteBuffer bytes = row.getByteBuffer(col); + return bytes != null ? unsafeWrap(bytes) : null; + } + + public static Reference deserializeReference(Row row) { + ByteBuffer previous = row.getByteBuffer(5); + byte[] bytes; + if (previous != null) { + bytes = new byte[previous.remaining()]; + previous.get(bytes); + } else { + bytes = null; + } + return Reference.reference( + row.getString(0), + deserializeObjId(row.getString(1)), + row.getBoolean(2), + row.getLong(3), + deserializeObjId(row.getString(4)), + deserializePreviousPointers(bytes)); + } + + public static ObjId deserializeObjId(String id) { + return id != null ? objIdFromString(id) : null; + } + + public static String serializeObjId(ObjId id) { + return id != null ? id.toString() : null; + } + + public static List deserializeObjIds(Row row, String col) { + List r = new ArrayList<>(); + deserializeObjIds(row, col, r::add); + return r; + } + + public static void deserializeObjIds(Row row, String col, Consumer consumer) { + List s = row.getList(col, String.class); + if (s == null || s.isEmpty()) { + return; + } + s.stream().map(ObjId::objIdFromString).forEach(consumer); + } + + public static List serializeObjIds(List values) { + return (values != null && !values.isEmpty()) + ? values.stream().map(ObjId::toString).collect(Collectors.toList()) + : null; + } +} diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CqlColumn.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CqlColumn.java new file mode 100644 index 00000000000..c76fa6ac292 --- /dev/null +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CqlColumn.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.cassandra; + +import java.util.Objects; + +public final class CqlColumn { + + private final String name; + + private final CqlColumnType type; + + public CqlColumn(String name, CqlColumnType type) { + this.name = name; + this.type = type; + } + + public String name() { + return name; + } + + public CqlColumnType type() { + return type; + } + + @Override + public String toString() { + return name(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CqlColumn cqlColumn = (CqlColumn) o; + return Objects.equals(name, cqlColumn.name) && type == cqlColumn.type; + } + + @Override + public int hashCode() { + return Objects.hash(name, type); + } +} diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CqlColumnType.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CqlColumnType.java index 834bf98f544..739324d9084 100644 --- a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CqlColumnType.java +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/CqlColumnType.java @@ -15,31 +15,38 @@ */ package org.projectnessie.versioned.storage.cassandra; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.DataTypes; + public enum CqlColumnType { // 0 - NAME("TEXT"), + NAME(DataTypes.TEXT), // 1 - OBJ_ID("ASCII"), + OBJ_ID(DataTypes.ASCII), // 2 - OBJ_ID_LIST("LIST"), + OBJ_ID_LIST(DataTypes.listOf(DataTypes.TEXT)), // 3 - BOOL("BOOLEAN"), + BOOL(DataTypes.BOOLEAN), // 4 - VARBINARY("BLOB"), + VARBINARY(DataTypes.BLOB), // 5 - BIGINT("BIGINT"), + BIGINT(DataTypes.BIGINT), // 6 - VARCHAR("TEXT"), + VARCHAR(DataTypes.TEXT), // 7 - INT("INT"); + INT(DataTypes.INT); + + private final DataType dataType; - private final String type; + CqlColumnType(DataType dataType) { + this.dataType = dataType; + } - CqlColumnType(String type) { - this.type = type; + public String cqlName() { + return dataType.asCql(false, false); } - public String type() { - return type; + public DataType dataType() { + return dataType; } } diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/CommitObjSerializer.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/CommitObjSerializer.java new file mode 100644 index 00000000000..a1830c26b0b --- /dev/null +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/CommitObjSerializer.java @@ -0,0 +1,204 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.cassandra.serializers; + +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_PREFIX; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_VALUES; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.STORE_OBJ_SUFFIX; +import static org.projectnessie.versioned.storage.common.indexes.StoreKey.keyFromString; +import static org.projectnessie.versioned.storage.common.objtypes.IndexStripe.indexStripe; +import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromByteBuffer; + +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; +import com.datastax.oss.driver.api.core.cql.Row; +import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Set; +import java.util.stream.Collectors; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.cassandra.CassandraSerde; +import org.projectnessie.versioned.storage.cassandra.CqlColumn; +import org.projectnessie.versioned.storage.cassandra.CqlColumnType; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.objtypes.CommitHeaders; +import org.projectnessie.versioned.storage.common.objtypes.CommitObj; +import org.projectnessie.versioned.storage.common.objtypes.CommitType; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.HeaderEntry; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.Headers; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.Stripe; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.Stripes; + +public class CommitObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new CommitObjSerializer(); + + private static final CqlColumn COL_COMMIT_CREATED = + new CqlColumn("c_created", CqlColumnType.BIGINT); + private static final CqlColumn COL_COMMIT_SEQ = new CqlColumn("c_seq", CqlColumnType.BIGINT); + private static final CqlColumn COL_COMMIT_MESSAGE = + new CqlColumn("c_message", CqlColumnType.VARCHAR); + private static final CqlColumn COL_COMMIT_HEADERS = + new CqlColumn("c_headers", CqlColumnType.VARBINARY); + private static final CqlColumn COL_COMMIT_REFERENCE_INDEX = + new CqlColumn("c_reference_index", CqlColumnType.OBJ_ID); + private static final CqlColumn COL_COMMIT_REFERENCE_INDEX_STRIPES = + new CqlColumn("c_reference_index_stripes", CqlColumnType.VARBINARY); + private static final CqlColumn COL_COMMIT_TAIL = + new CqlColumn("c_tail", CqlColumnType.OBJ_ID_LIST); + private static final CqlColumn COL_COMMIT_SECONDARY_PARENTS = + new CqlColumn("c_secondary_parents", CqlColumnType.OBJ_ID_LIST); + private static final CqlColumn COL_COMMIT_INCREMENTAL_INDEX = + new CqlColumn("c_incremental_index", CqlColumnType.VARBINARY); + private static final CqlColumn COL_COMMIT_INCOMPLETE_INDEX = + new CqlColumn("c_incomplete_index", CqlColumnType.BOOL); + private static final CqlColumn COL_COMMIT_TYPE = + new CqlColumn("c_commit_type", CqlColumnType.NAME); + + private static final Set COLS = + ImmutableSet.builder() + .add(COL_COMMIT_CREATED) + .add(COL_COMMIT_SEQ) + .add(COL_COMMIT_MESSAGE) + .add(COL_COMMIT_HEADERS) + .add(COL_COMMIT_REFERENCE_INDEX) + .add(COL_COMMIT_REFERENCE_INDEX_STRIPES) + .add(COL_COMMIT_TAIL) + .add(COL_COMMIT_SECONDARY_PARENTS) + .add(COL_COMMIT_INCREMENTAL_INDEX) + .add(COL_COMMIT_INCOMPLETE_INDEX) + .add(COL_COMMIT_TYPE) + .build(); + + private static final String INSERT_CQL = + INSERT_OBJ_PREFIX + + COLS.stream().map(CqlColumn::name).collect(Collectors.joining(",")) + + INSERT_OBJ_VALUES + + COLS.stream().map(c -> ":" + c.name()).collect(Collectors.joining(",")) + + ")"; + + private static final String STORE_CQL = INSERT_CQL + STORE_OBJ_SUFFIX; + + private CommitObjSerializer() {} + + @Override + public Set columns() { + return COLS; + } + + @Override + public String insertCql(boolean upsert) { + return upsert ? INSERT_CQL : STORE_CQL; + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + @Override + public void serialize( + CommitObj obj, + BoundStatementBuilder stmt, + int incrementalIndexLimit, + int maxSerializedIndexSize) + throws ObjTooLargeException { + + stmt.setLong(COL_COMMIT_CREATED.name(), obj.created()); + stmt.setLong(COL_COMMIT_SEQ.name(), obj.seq()); + stmt.setString(COL_COMMIT_MESSAGE.name(), obj.message()); + + Headers.Builder hb = Headers.newBuilder(); + for (String h : obj.headers().keySet()) { + hb.addHeaders(HeaderEntry.newBuilder().setName(h).addAllValues(obj.headers().getAll(h))); + } + stmt.setByteBuffer(COL_COMMIT_HEADERS.name(), ByteBuffer.wrap(hb.build().toByteArray())); + stmt.setString( + COL_COMMIT_REFERENCE_INDEX.name(), CassandraSerde.serializeObjId(obj.referenceIndex())); + + Stripes.Builder b = Stripes.newBuilder(); + obj.referenceIndexStripes().stream() + .map( + s -> + Stripe.newBuilder() + .setFirstKey(s.firstKey().rawString()) + .setLastKey(s.lastKey().rawString()) + .setSegment(s.segment().asBytes())) + .forEach(b::addStripes); + stmt.setByteBuffer( + COL_COMMIT_REFERENCE_INDEX_STRIPES.name(), b.build().toByteString().asReadOnlyByteBuffer()); + + stmt.setList(COL_COMMIT_TAIL.name(), CassandraSerde.serializeObjIds(obj.tail()), String.class); + stmt.setList( + COL_COMMIT_SECONDARY_PARENTS.name(), + CassandraSerde.serializeObjIds(obj.secondaryParents()), + String.class); + + ByteString index = obj.incrementalIndex(); + if (index.size() > incrementalIndexLimit) { + throw new ObjTooLargeException(index.size(), incrementalIndexLimit); + } + stmt.setByteBuffer(COL_COMMIT_INCREMENTAL_INDEX.name(), index.asReadOnlyByteBuffer()); + + stmt.setBoolean(COL_COMMIT_INCOMPLETE_INDEX.name(), obj.incompleteIndex()); + stmt.setString(COL_COMMIT_TYPE.name(), obj.commitType().name()); + } + + @Override + public CommitObj deserialize(Row row, ObjId id) { + CommitObj.Builder b = + CommitObj.commitBuilder() + .id(id) + .created(row.getLong(COL_COMMIT_CREATED.name())) + .seq(row.getLong(COL_COMMIT_SEQ.name())) + .message(row.getString(COL_COMMIT_MESSAGE.name())) + .referenceIndex( + CassandraSerde.deserializeObjId(row.getString(COL_COMMIT_REFERENCE_INDEX.name()))) + .incrementalIndex( + CassandraSerde.deserializeBytes(row, COL_COMMIT_INCREMENTAL_INDEX.name())) + .incompleteIndex(row.getBoolean(COL_COMMIT_INCOMPLETE_INDEX.name())) + .commitType(CommitType.valueOf(row.getString(COL_COMMIT_TYPE.name()))); + CassandraSerde.deserializeObjIds(row, COL_COMMIT_TAIL.name(), b::addTail); + CassandraSerde.deserializeObjIds( + row, COL_COMMIT_SECONDARY_PARENTS.name(), b::addSecondaryParents); + + try { + CommitHeaders.Builder h = CommitHeaders.newCommitHeaders(); + Headers headers = Headers.parseFrom(row.getByteBuffer("c_headers")); + for (HeaderEntry e : headers.getHeadersList()) { + for (String v : e.getValuesList()) { + h.add(e.getName(), v); + } + } + b.headers(h.build()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + try { + Stripes stripes = Stripes.parseFrom(row.getByteBuffer("c_reference_index_stripes")); + stripes.getStripesList().stream() + .map( + s -> + indexStripe( + keyFromString(s.getFirstKey()), + keyFromString(s.getLastKey()), + objIdFromByteBuffer(s.getSegment().asReadOnlyByteBuffer()))) + .forEach(b::addReferenceIndexStripes); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return b.build(); + } +} diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/ContentValueObjSerializer.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/ContentValueObjSerializer.java new file mode 100644 index 00000000000..acefafbc0f3 --- /dev/null +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/ContentValueObjSerializer.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.cassandra.serializers; + +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_PREFIX; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_VALUES; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.STORE_OBJ_SUFFIX; +import static org.projectnessie.versioned.storage.common.objtypes.ContentValueObj.contentValue; + +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; +import com.datastax.oss.driver.api.core.cql.Row; +import com.google.common.collect.ImmutableSet; +import java.util.Set; +import java.util.stream.Collectors; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.cassandra.CassandraSerde; +import org.projectnessie.versioned.storage.cassandra.CqlColumn; +import org.projectnessie.versioned.storage.cassandra.CqlColumnType; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.objtypes.ContentValueObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public class ContentValueObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new ContentValueObjSerializer(); + + private static final CqlColumn COL_VALUE_CONTENT_ID = + new CqlColumn("v_content_id", CqlColumnType.NAME); + private static final CqlColumn COL_VALUE_PAYLOAD = new CqlColumn("v_payload", CqlColumnType.INT); + private static final CqlColumn COL_VALUE_DATA = new CqlColumn("v_data", CqlColumnType.VARBINARY); + + private static final Set COLS = + ImmutableSet.of(COL_VALUE_CONTENT_ID, COL_VALUE_PAYLOAD, COL_VALUE_DATA); + + private static final String INSERT_CQL = + INSERT_OBJ_PREFIX + + COLS.stream().map(CqlColumn::name).collect(Collectors.joining(",")) + + INSERT_OBJ_VALUES + + COLS.stream().map(c -> ":" + c.name()).collect(Collectors.joining(",")) + + ")"; + + private static final String STORE_CQL = INSERT_CQL + STORE_OBJ_SUFFIX; + + private ContentValueObjSerializer() {} + + @Override + public Set columns() { + return COLS; + } + + @Override + public String insertCql(boolean upsert) { + return upsert ? INSERT_CQL : STORE_CQL; + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + @Override + public void serialize( + ContentValueObj obj, + BoundStatementBuilder stmt, + int incrementalIndexLimit, + int maxSerializedIndexSize) + throws ObjTooLargeException { + stmt.setString(COL_VALUE_CONTENT_ID.name(), obj.contentId()); + stmt.setInt(COL_VALUE_PAYLOAD.name(), obj.payload()); + stmt.setByteBuffer(COL_VALUE_DATA.name(), obj.data().asReadOnlyByteBuffer()); + } + + @Override + public ContentValueObj deserialize(Row row, ObjId id) { + ByteString value = CassandraSerde.deserializeBytes(row, COL_VALUE_DATA.name()); + if (value != null) { + return contentValue( + id, + row.getString(COL_VALUE_CONTENT_ID.name()), + row.getInt(COL_VALUE_PAYLOAD.name()), + value); + } + throw new IllegalStateException("Data value of obj " + id + " of type VALUE is null"); + } +} diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/CustomObjSerializer.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/CustomObjSerializer.java new file mode 100644 index 00000000000..e14cf96e735 --- /dev/null +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/CustomObjSerializer.java @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.cassandra.serializers; + +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_PREFIX; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_VALUES; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.STORE_OBJ_SUFFIX; + +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; +import com.datastax.oss.driver.api.core.cql.Row; +import com.google.common.collect.ImmutableSet; +import java.nio.ByteBuffer; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import org.projectnessie.versioned.storage.cassandra.CqlColumn; +import org.projectnessie.versioned.storage.cassandra.CqlColumnType; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.serialize.SmileSerialization; + +public class CustomObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new CustomObjSerializer(); + + private static final CqlColumn COL_CUSTOM_CLASS = new CqlColumn("x_class", CqlColumnType.NAME); + private static final CqlColumn COL_CUSTOM_DATA = new CqlColumn("x_data", CqlColumnType.VARBINARY); + + private static final Set COLS = ImmutableSet.of(COL_CUSTOM_CLASS, COL_CUSTOM_DATA); + + private static final String INSERT_CQL = + INSERT_OBJ_PREFIX + + COLS.stream().map(CqlColumn::name).collect(Collectors.joining(",")) + + INSERT_OBJ_VALUES + + COLS.stream().map(c -> ":" + c.name()).collect(Collectors.joining(",")) + + ")"; + + private static final String STORE_CQL = INSERT_CQL + STORE_OBJ_SUFFIX; + + private CustomObjSerializer() {} + + @Override + public Set columns() { + return COLS; + } + + @Override + public String insertCql(boolean upsert) { + return upsert ? INSERT_CQL : STORE_CQL; + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + @Override + public void serialize( + Obj obj, BoundStatementBuilder stmt, int incrementalIndexLimit, int maxSerializedIndexSize) + throws ObjTooLargeException { + stmt.setString(COL_CUSTOM_CLASS.name(), obj.type().targetClass().getName()); + stmt.setByteBuffer( + COL_CUSTOM_DATA.name(), ByteBuffer.wrap(SmileSerialization.serializeObj(obj))); + } + + @Override + public Obj deserialize(Row row, ObjId id) { + ByteBuffer buffer = Objects.requireNonNull(row.getByteBuffer(COL_CUSTOM_DATA.name())); + return SmileSerialization.deserializeObj(id, buffer, row.getString(COL_CUSTOM_CLASS.name())); + } +} diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/IndexObjSerializer.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/IndexObjSerializer.java new file mode 100644 index 00000000000..a1befb351e4 --- /dev/null +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/IndexObjSerializer.java @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.cassandra.serializers; + +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_PREFIX; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_VALUES; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.STORE_OBJ_SUFFIX; +import static org.projectnessie.versioned.storage.common.objtypes.IndexObj.index; + +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; +import com.datastax.oss.driver.api.core.cql.Row; +import com.google.common.collect.ImmutableSet; +import java.util.Set; +import java.util.stream.Collectors; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.cassandra.CassandraSerde; +import org.projectnessie.versioned.storage.cassandra.CqlColumn; +import org.projectnessie.versioned.storage.cassandra.CqlColumnType; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.objtypes.IndexObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public class IndexObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new IndexObjSerializer(); + + private static final CqlColumn COL_INDEX_INDEX = + new CqlColumn("i_index", CqlColumnType.VARBINARY); + + private static final Set COLS = ImmutableSet.of(COL_INDEX_INDEX); + + private static final String INSERT_CQL = + INSERT_OBJ_PREFIX + + COLS.stream().map(CqlColumn::name).collect(Collectors.joining(",")) + + INSERT_OBJ_VALUES + + COLS.stream().map(c -> ":" + c.name()).collect(Collectors.joining(",")) + + ")"; + + private static final String STORE_CQL = INSERT_CQL + STORE_OBJ_SUFFIX; + + private IndexObjSerializer() {} + + @Override + public Set columns() { + return COLS; + } + + @Override + public String insertCql(boolean upsert) { + return upsert ? INSERT_CQL : STORE_CQL; + } + + @Override + public void serialize( + IndexObj obj, + BoundStatementBuilder stmt, + int incrementalIndexLimit, + int maxSerializedIndexSize) + throws ObjTooLargeException { + ByteString index = obj.index(); + if (index.size() > maxSerializedIndexSize) { + throw new ObjTooLargeException(index.size(), maxSerializedIndexSize); + } + stmt.setByteBuffer(COL_INDEX_INDEX.name(), index.asReadOnlyByteBuffer()); + } + + @Override + public IndexObj deserialize(Row row, ObjId id) { + ByteString indexValue = CassandraSerde.deserializeBytes(row, COL_INDEX_INDEX.name()); + if (indexValue != null) { + return index(id, indexValue); + } + throw new IllegalStateException("Index value of obj " + id + " of type INDEX is null"); + } +} diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/IndexSegmentsObjSerializer.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/IndexSegmentsObjSerializer.java new file mode 100644 index 00000000000..84ba039a4a0 --- /dev/null +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/IndexSegmentsObjSerializer.java @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.cassandra.serializers; + +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_PREFIX; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_VALUES; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.STORE_OBJ_SUFFIX; +import static org.projectnessie.versioned.storage.common.indexes.StoreKey.keyFromString; +import static org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj.indexSegments; +import static org.projectnessie.versioned.storage.common.objtypes.IndexStripe.indexStripe; +import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromByteBuffer; + +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; +import com.datastax.oss.driver.api.core.cql.Row; +import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.projectnessie.versioned.storage.cassandra.CqlColumn; +import org.projectnessie.versioned.storage.cassandra.CqlColumnType; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj; +import org.projectnessie.versioned.storage.common.objtypes.IndexStripe; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.Stripe; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.Stripes; + +public class IndexSegmentsObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new IndexSegmentsObjSerializer(); + + private static final CqlColumn COL_SEGMENTS_STRIPES = + new CqlColumn("i_stripes", CqlColumnType.VARBINARY); + + private static final Set COLS = ImmutableSet.of(COL_SEGMENTS_STRIPES); + + private static final String INSERT_CQL = + INSERT_OBJ_PREFIX + + COLS.stream().map(CqlColumn::name).collect(Collectors.joining(",")) + + INSERT_OBJ_VALUES + + COLS.stream().map(c -> ":" + c.name()).collect(Collectors.joining(",")) + + ")"; + + private static final String STORE_CQL = INSERT_CQL + STORE_OBJ_SUFFIX; + + private IndexSegmentsObjSerializer() {} + + @Override + public Set columns() { + return COLS; + } + + @Override + public String insertCql(boolean upsert) { + return upsert ? INSERT_CQL : STORE_CQL; + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + @Override + public void serialize( + IndexSegmentsObj obj, + BoundStatementBuilder stmt, + int incrementalIndexLimit, + int maxSerializedIndexSize) + throws ObjTooLargeException { + Stripes.Builder b = Stripes.newBuilder(); + obj.stripes().stream() + .map( + s -> + Stripe.newBuilder() + .setFirstKey(s.firstKey().rawString()) + .setLastKey(s.lastKey().rawString()) + .setSegment(s.segment().asBytes())) + .forEach(b::addStripes); + stmt.setByteBuffer( + COL_SEGMENTS_STRIPES.name(), b.build().toByteString().asReadOnlyByteBuffer()); + } + + @Override + public IndexSegmentsObj deserialize(Row row, ObjId id) { + try { + Stripes stripes = Stripes.parseFrom(row.getByteBuffer(COL_SEGMENTS_STRIPES.name())); + List stripeList = + stripes.getStripesList().stream() + .map( + s -> + indexStripe( + keyFromString(s.getFirstKey()), + keyFromString(s.getLastKey()), + objIdFromByteBuffer(s.getSegment().asReadOnlyByteBuffer()))) + .collect(Collectors.toList()); + return indexSegments(id, stripeList); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/ObjSerializer.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/ObjSerializer.java new file mode 100644 index 00000000000..6233296fd0f --- /dev/null +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/ObjSerializer.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.cassandra.serializers; + +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; +import com.datastax.oss.driver.api.core.cql.Row; +import java.util.Set; +import org.projectnessie.versioned.storage.cassandra.CqlColumn; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public interface ObjSerializer { + + Set columns(); + + String insertCql(boolean upsert); + + void serialize( + O obj, BoundStatementBuilder stmt, int incrementalIndexLimit, int maxSerializedIndexSize) + throws ObjTooLargeException; + + O deserialize(Row row, ObjId id); +} diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/ObjSerializers.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/ObjSerializers.java new file mode 100644 index 00000000000..98c43aa72d9 --- /dev/null +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/ObjSerializers.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.cassandra.serializers; + +import java.util.HashSet; +import java.util.Set; +import javax.annotation.Nonnull; +import org.projectnessie.versioned.storage.cassandra.CqlColumn; +import org.projectnessie.versioned.storage.common.objtypes.StandardObjType; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjType; + +public final class ObjSerializers { + + public static final Set> ALL_SERIALIZERS = + Set.of( + CommitObjSerializer.INSTANCE, + ContentValueObjSerializer.INSTANCE, + IndexSegmentsObjSerializer.INSTANCE, + IndexObjSerializer.INSTANCE, + RefObjSerializer.INSTANCE, + StringObjSerializer.INSTANCE, + TagObjSerializer.INSTANCE, + CustomObjSerializer.INSTANCE); + + static { + Set columnNames = new HashSet<>(); + ALL_SERIALIZERS.forEach( + serializer -> { + for (CqlColumn col : serializer.columns()) { + if (!columnNames.add(col.name())) { + throw new IllegalStateException("Duplicate column name: " + col); + } + } + }); + } + + @Nonnull + @jakarta.annotation.Nonnull + public static ObjSerializer forType(@Nonnull @jakarta.annotation.Nonnull ObjType type) { + ObjSerializer serializer = CustomObjSerializer.INSTANCE; + if (type instanceof StandardObjType) { + switch ((StandardObjType) type) { + case COMMIT: + serializer = CommitObjSerializer.INSTANCE; + break; + case INDEX_SEGMENTS: + serializer = IndexSegmentsObjSerializer.INSTANCE; + break; + case INDEX: + serializer = IndexObjSerializer.INSTANCE; + break; + case REF: + serializer = RefObjSerializer.INSTANCE; + break; + case STRING: + serializer = StringObjSerializer.INSTANCE; + break; + case TAG: + serializer = TagObjSerializer.INSTANCE; + break; + case VALUE: + serializer = ContentValueObjSerializer.INSTANCE; + break; + default: + throw new IllegalArgumentException("Unknown standard object type: " + type); + } + } + @SuppressWarnings("unchecked") + ObjSerializer cast = (ObjSerializer) serializer; + return cast; + } +} diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/RefObjSerializer.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/RefObjSerializer.java new file mode 100644 index 00000000000..961c155b448 --- /dev/null +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/RefObjSerializer.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.cassandra.serializers; + +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_PREFIX; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_VALUES; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.STORE_OBJ_SUFFIX; +import static org.projectnessie.versioned.storage.common.objtypes.RefObj.ref; + +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; +import com.datastax.oss.driver.api.core.cql.Row; +import com.google.common.collect.ImmutableSet; +import java.util.Set; +import java.util.stream.Collectors; +import org.projectnessie.versioned.storage.cassandra.CassandraSerde; +import org.projectnessie.versioned.storage.cassandra.CqlColumn; +import org.projectnessie.versioned.storage.cassandra.CqlColumnType; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.objtypes.RefObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public class RefObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new RefObjSerializer(); + + private static final CqlColumn COL_REF_NAME = new CqlColumn("r_name", CqlColumnType.NAME); + private static final CqlColumn COL_REF_INITIAL_POINTER = + new CqlColumn("r_initial_pointer", CqlColumnType.OBJ_ID); + private static final CqlColumn COL_REF_CREATED_AT = + new CqlColumn("r_created_at", CqlColumnType.BIGINT); + private static final CqlColumn COL_REF_EXTENDED_INFO = + new CqlColumn("r_extended_info", CqlColumnType.OBJ_ID); + + private static final Set COLS = + ImmutableSet.of( + COL_REF_NAME, COL_REF_INITIAL_POINTER, COL_REF_CREATED_AT, COL_REF_EXTENDED_INFO); + + private static final String INSERT_CQL = + INSERT_OBJ_PREFIX + + COLS.stream().map(CqlColumn::name).collect(Collectors.joining(",")) + + INSERT_OBJ_VALUES + + COLS.stream().map(c -> ":" + c.name()).collect(Collectors.joining(",")) + + ")"; + + private static final String STORE_CQL = INSERT_CQL + STORE_OBJ_SUFFIX; + + private RefObjSerializer() {} + + @Override + public Set columns() { + return COLS; + } + + @Override + public String insertCql(boolean upsert) { + return upsert ? INSERT_CQL : STORE_CQL; + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + @Override + public void serialize( + RefObj obj, BoundStatementBuilder stmt, int incrementalIndexLimit, int maxSerializedIndexSize) + throws ObjTooLargeException { + stmt.setString(COL_REF_NAME.name(), obj.name()); + stmt.setString( + COL_REF_INITIAL_POINTER.name(), CassandraSerde.serializeObjId(obj.initialPointer())); + stmt.setLong(COL_REF_CREATED_AT.name(), obj.createdAtMicros()); + stmt.setString( + COL_REF_EXTENDED_INFO.name(), CassandraSerde.serializeObjId(obj.extendedInfoObj())); + } + + @Override + public RefObj deserialize(Row row, ObjId id) { + return ref( + id, + row.getString(COL_REF_NAME.name()), + CassandraSerde.deserializeObjId(row.getString(COL_REF_INITIAL_POINTER.name())), + row.getLong(COL_REF_CREATED_AT.name()), + CassandraSerde.deserializeObjId(row.getString(COL_REF_EXTENDED_INFO.name()))); + } +} diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/StringObjSerializer.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/StringObjSerializer.java new file mode 100644 index 00000000000..0871990bea3 --- /dev/null +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/StringObjSerializer.java @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.cassandra.serializers; + +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_PREFIX; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_VALUES; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.STORE_OBJ_SUFFIX; +import static org.projectnessie.versioned.storage.common.objtypes.StringObj.stringData; + +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; +import com.datastax.oss.driver.api.core.cql.Row; +import com.google.common.collect.ImmutableSet; +import java.util.Set; +import java.util.stream.Collectors; +import org.projectnessie.versioned.storage.cassandra.CassandraSerde; +import org.projectnessie.versioned.storage.cassandra.CqlColumn; +import org.projectnessie.versioned.storage.cassandra.CqlColumnType; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.objtypes.Compression; +import org.projectnessie.versioned.storage.common.objtypes.StringObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public class StringObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new StringObjSerializer(); + + private static final CqlColumn COL_STRING_CONTENT_TYPE = + new CqlColumn("s_content_type", CqlColumnType.NAME); + private static final CqlColumn COL_STRING_COMPRESSION = + new CqlColumn("s_compression", CqlColumnType.NAME); + private static final CqlColumn COL_STRING_FILENAME = + new CqlColumn("s_filename", CqlColumnType.NAME); + private static final CqlColumn COL_STRING_PREDECESSORS = + new CqlColumn("s_predecessors", CqlColumnType.OBJ_ID_LIST); + private static final CqlColumn COL_STRING_TEXT = new CqlColumn("s_text", CqlColumnType.VARBINARY); + + private static final Set COLS = + ImmutableSet.of( + COL_STRING_CONTENT_TYPE, + COL_STRING_COMPRESSION, + COL_STRING_FILENAME, + COL_STRING_PREDECESSORS, + COL_STRING_TEXT); + + private static final String INSERT_CQL = + INSERT_OBJ_PREFIX + + COLS.stream().map(CqlColumn::name).collect(Collectors.joining(",")) + + INSERT_OBJ_VALUES + + COLS.stream().map(c -> ":" + c.name()).collect(Collectors.joining(",")) + + ")"; + + private static final String STORE_CQL = INSERT_CQL + STORE_OBJ_SUFFIX; + + private StringObjSerializer() {} + + @Override + public Set columns() { + return COLS; + } + + @Override + public String insertCql(boolean upsert) { + return upsert ? INSERT_CQL : STORE_CQL; + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + @Override + public void serialize( + StringObj obj, + BoundStatementBuilder stmt, + int incrementalIndexLimit, + int maxSerializedIndexSize) + throws ObjTooLargeException { + stmt.setString(COL_STRING_CONTENT_TYPE.name(), obj.contentType()); + stmt.setString(COL_STRING_COMPRESSION.name(), obj.compression().name()); + stmt.setString(COL_STRING_FILENAME.name(), obj.filename()); + stmt.setList( + COL_STRING_PREDECESSORS.name(), + CassandraSerde.serializeObjIds(obj.predecessors()), + String.class); + stmt.setByteBuffer(COL_STRING_TEXT.name(), obj.text().asReadOnlyByteBuffer()); + } + + @Override + public StringObj deserialize(Row row, ObjId id) { + return stringData( + id, + row.getString(COL_STRING_CONTENT_TYPE.name()), + Compression.valueOf(row.getString(COL_STRING_COMPRESSION.name())), + row.getString(COL_STRING_FILENAME.name()), + CassandraSerde.deserializeObjIds(row, COL_STRING_PREDECESSORS.name()), + CassandraSerde.deserializeBytes(row, COL_STRING_TEXT.name())); + } +} diff --git a/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/TagObjSerializer.java b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/TagObjSerializer.java new file mode 100644 index 00000000000..2445b46bc80 --- /dev/null +++ b/versioned/storage/cassandra/src/main/java/org/projectnessie/versioned/storage/cassandra/serializers/TagObjSerializer.java @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.cassandra.serializers; + +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_PREFIX; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.INSERT_OBJ_VALUES; +import static org.projectnessie.versioned.storage.cassandra.CassandraConstants.STORE_OBJ_SUFFIX; +import static org.projectnessie.versioned.storage.common.objtypes.TagObj.tag; + +import com.datastax.oss.driver.api.core.cql.BoundStatementBuilder; +import com.datastax.oss.driver.api.core.cql.Row; +import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Set; +import java.util.stream.Collectors; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.cassandra.CassandraSerde; +import org.projectnessie.versioned.storage.cassandra.CqlColumn; +import org.projectnessie.versioned.storage.cassandra.CqlColumnType; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.objtypes.CommitHeaders; +import org.projectnessie.versioned.storage.common.objtypes.TagObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.HeaderEntry; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.Headers; + +public class TagObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new TagObjSerializer(); + + private static final CqlColumn COL_TAG_MESSAGE = + new CqlColumn("t_message", CqlColumnType.VARCHAR); + private static final CqlColumn COL_TAG_HEADERS = + new CqlColumn("t_headers", CqlColumnType.VARBINARY); + private static final CqlColumn COL_TAG_SIGNATURE = + new CqlColumn("t_signature", CqlColumnType.VARBINARY); + + private static final Set COLS = + ImmutableSet.of(COL_TAG_MESSAGE, COL_TAG_HEADERS, COL_TAG_SIGNATURE); + + private static final String INSERT_CQL = + INSERT_OBJ_PREFIX + + COLS.stream().map(CqlColumn::name).collect(Collectors.joining(",")) + + INSERT_OBJ_VALUES + + COLS.stream().map(c -> ":" + c.name()).collect(Collectors.joining(",")) + + ")"; + + private static final String STORE_CQL = INSERT_CQL + STORE_OBJ_SUFFIX; + + private TagObjSerializer() {} + + @Override + public Set columns() { + return COLS; + } + + @Override + public String insertCql(boolean upsert) { + return upsert ? INSERT_CQL : STORE_CQL; + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + @Override + public void serialize( + TagObj obj, BoundStatementBuilder stmt, int incrementalIndexLimit, int maxSerializedIndexSize) + throws ObjTooLargeException { + stmt.setString(COL_TAG_MESSAGE.name(), obj.message()); + Headers.Builder hb = Headers.newBuilder(); + CommitHeaders headers = obj.headers(); + if (headers != null) { + for (String h : headers.keySet()) { + hb.addHeaders(HeaderEntry.newBuilder().setName(h).addAllValues(headers.getAll(h))); + } + } + stmt.setByteBuffer(COL_TAG_HEADERS.name(), ByteBuffer.wrap(hb.build().toByteArray())); + ByteString signature = obj.signature(); + stmt.setByteBuffer( + COL_TAG_SIGNATURE.name(), signature != null ? signature.asReadOnlyByteBuffer() : null); + } + + @Override + public TagObj deserialize(Row row, ObjId id) { + CommitHeaders tagHeaders = null; + try { + Headers headers = Headers.parseFrom(row.getByteBuffer(COL_TAG_HEADERS.name())); + if (headers.getHeadersCount() > 0) { + CommitHeaders.Builder h = CommitHeaders.newCommitHeaders(); + for (HeaderEntry e : headers.getHeadersList()) { + for (String v : e.getValuesList()) { + h.add(e.getName(), v); + } + } + tagHeaders = h.build(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + + return tag( + id, + row.getString(COL_TAG_MESSAGE.name()), + tagHeaders, + CassandraSerde.deserializeBytes(row, COL_TAG_SIGNATURE.name())); + } +} diff --git a/versioned/storage/common-proto/src/main/proto/org/projectnessie/versioned/storage/common/proto/storage.proto b/versioned/storage/common-proto/src/main/proto/org/projectnessie/versioned/storage/common/proto/storage.proto index 8f9121aaaf1..9c6a72b486d 100644 --- a/versioned/storage/common-proto/src/main/proto/org/projectnessie/versioned/storage/common/proto/storage.proto +++ b/versioned/storage/common-proto/src/main/proto/org/projectnessie/versioned/storage/common/proto/storage.proto @@ -106,6 +106,7 @@ message ObjProto { IndexProto index = 6; StringProto string_data = 7; TagProto tag = 8; + CustomProto custom = 9; } } @@ -131,3 +132,8 @@ message ReferencePreviousProto { bytes pointer = 1; int64 timestamp = 2;; } + +message CustomProto { + string target_class = 1; + bytes data = 2; +} diff --git a/versioned/storage/common-serialize/build.gradle.kts b/versioned/storage/common-serialize/build.gradle.kts index 3fbeb774bb5..8d841364f98 100644 --- a/versioned/storage/common-serialize/build.gradle.kts +++ b/versioned/storage/common-serialize/build.gradle.kts @@ -19,14 +19,17 @@ plugins { id("nessie-jacoco") } -extra["maven.name"] = "Nessie - Storage - Common proto serialization" +extra["maven.name"] = "Nessie - Storage - Common serialization" -description = "Protobuf based serialization for storage objects." +description = "Serialization for storage objects." dependencies { api(project(":nessie-versioned-storage-common")) api(project(":nessie-versioned-storage-common-proto")) + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-smile") + testImplementation(platform(libs.junit.bom)) testImplementation(libs.bundles.junit.testing) } diff --git a/versioned/storage/common-serialize/src/main/java/org/projectnessie/versioned/storage/serialize/ProtoSerialization.java b/versioned/storage/common-serialize/src/main/java/org/projectnessie/versioned/storage/serialize/ProtoSerialization.java index 012669bc678..7f841a62aa0 100644 --- a/versioned/storage/common-serialize/src/main/java/org/projectnessie/versioned/storage/serialize/ProtoSerialization.java +++ b/versioned/storage/common-serialize/src/main/java/org/projectnessie/versioned/storage/serialize/ProtoSerialization.java @@ -46,6 +46,7 @@ import org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj; import org.projectnessie.versioned.storage.common.objtypes.IndexStripe; import org.projectnessie.versioned.storage.common.objtypes.RefObj; +import org.projectnessie.versioned.storage.common.objtypes.StandardObjType; import org.projectnessie.versioned.storage.common.objtypes.StringObj; import org.projectnessie.versioned.storage.common.objtypes.TagObj; import org.projectnessie.versioned.storage.common.persist.ImmutableReference; @@ -57,6 +58,7 @@ import org.projectnessie.versioned.storage.common.proto.StorageTypes.CommitTypeProto; import org.projectnessie.versioned.storage.common.proto.StorageTypes.CompressionProto; import org.projectnessie.versioned.storage.common.proto.StorageTypes.ContentValueProto; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.CustomProto; import org.projectnessie.versioned.storage.common.proto.StorageTypes.HeaderEntry; import org.projectnessie.versioned.storage.common.proto.StorageTypes.IndexProto; import org.projectnessie.versioned.storage.common.proto.StorageTypes.IndexSegmentsProto; @@ -68,6 +70,7 @@ import org.projectnessie.versioned.storage.common.proto.StorageTypes.TagProto; public final class ProtoSerialization { + private ProtoSerialization() {} public static byte[] serializeReference(Reference reference) { @@ -205,29 +208,33 @@ public static byte[] serializeObj(Obj obj, int incrementalIndexSizeLimit, int in return null; } ObjProto.Builder b = ObjProto.newBuilder(); - switch (obj.type()) { - case COMMIT: - return b.setCommit(serializeCommit((CommitObj) obj, incrementalIndexSizeLimit)) - .build() - .toByteArray(); - case VALUE: - return b.setContentValue(serializeContentValue((ContentValueObj) obj)) - .build() - .toByteArray(); - case REF: - return b.setRef(serializeRef((RefObj) obj)).build().toByteArray(); - case INDEX_SEGMENTS: - return b.setIndexSegments(serializeIndexSegments((IndexSegmentsObj) obj)) - .build() - .toByteArray(); - case INDEX: - return b.setIndex(serializeIndex((IndexObj) obj, indexSizeLimit)).build().toByteArray(); - case STRING: - return b.setStringData(serializeStringData((StringObj) obj)).build().toByteArray(); - case TAG: - return b.setTag(serializeTag((TagObj) obj)).build().toByteArray(); - default: - throw new UnsupportedOperationException("Unknown object type " + obj.type()); + if (obj.type() instanceof StandardObjType) { + switch (((StandardObjType) obj.type())) { + case COMMIT: + return b.setCommit(serializeCommit((CommitObj) obj, incrementalIndexSizeLimit)) + .build() + .toByteArray(); + case VALUE: + return b.setContentValue(serializeContentValue((ContentValueObj) obj)) + .build() + .toByteArray(); + case REF: + return b.setRef(serializeRef((RefObj) obj)).build().toByteArray(); + case INDEX_SEGMENTS: + return b.setIndexSegments(serializeIndexSegments((IndexSegmentsObj) obj)) + .build() + .toByteArray(); + case INDEX: + return b.setIndex(serializeIndex((IndexObj) obj, indexSizeLimit)).build().toByteArray(); + case STRING: + return b.setStringData(serializeStringData((StringObj) obj)).build().toByteArray(); + case TAG: + return b.setTag(serializeTag((TagObj) obj)).build().toByteArray(); + default: + throw new UnsupportedOperationException("Unknown standard object type " + obj.type()); + } + } else { + return b.setCustom(serializeCustom(obj)).build().toByteArray(); } } @@ -277,6 +284,9 @@ public static Obj deserializeObjProto(ObjId id, ObjProto obj) { if (obj.hasTag()) { return deserializeTag(id, obj.getTag()); } + if (obj.hasCustom()) { + return deserializeCustom(id, obj.getCustom()); + } throw new UnsupportedOperationException("Cannot deserialize " + obj); } @@ -482,4 +492,15 @@ private static TagProto.Builder serializeTag(TagObj obj) { } return tag; } + + private static Obj deserializeCustom(ObjId id, CustomProto custom) { + return SmileSerialization.deserializeObj( + id, custom.getData().toByteArray(), custom.getTargetClass()); + } + + private static CustomProto.Builder serializeCustom(Obj obj) { + return CustomProto.newBuilder() + .setTargetClass(obj.type().targetClass().getName()) + .setData(ByteString.copyFrom(SmileSerialization.serializeObj(obj))); + } } diff --git a/versioned/storage/common-serialize/src/main/java/org/projectnessie/versioned/storage/serialize/SmileSerialization.java b/versioned/storage/common-serialize/src/main/java/org/projectnessie/versioned/storage/serialize/SmileSerialization.java new file mode 100644 index 00000000000..ea03e3ee0e9 --- /dev/null +++ b/versioned/storage/common-serialize/src/main/java/org/projectnessie/versioned/storage/serialize/SmileSerialization.java @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2022 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.serialize; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.dataformat.smile.databind.SmileMapper; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import org.projectnessie.versioned.storage.common.json.ObjIdHelper; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public final class SmileSerialization { + + private static final ObjectMapper SMILE_MAPPER = new SmileMapper().findAndRegisterModules(); + + private SmileSerialization() {} + + public static Obj deserializeObj(ObjId id, byte[] data, String targetClassFqdn) { + try { + @SuppressWarnings("unchecked") + Class targetClass = (Class) Class.forName(targetClassFqdn); + return deserializeObj(id, data, targetClass); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + public static Obj deserializeObj(ObjId id, ByteBuffer data, String targetClassFqdn) { + try { + @SuppressWarnings("unchecked") + Class targetClass = (Class) Class.forName(targetClassFqdn); + return deserializeObj(id, data, targetClass); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + public static Obj deserializeObj(ObjId id, byte[] data, Class targetClass) { + try { + ObjectReader reader = ObjIdHelper.storeObjIdInContext(SMILE_MAPPER, id); + return reader.readValue(data, targetClass); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static Obj deserializeObj(ObjId id, ByteBuffer data, Class targetClass) { + byte[] bytes = new byte[data.remaining()]; + data.get(bytes); + return deserializeObj(id, bytes, targetClass); + } + + public static byte[] serializeObj(Obj obj) { + try { + return SMILE_MAPPER.writeValueAsBytes(obj); + } catch (JsonProcessingException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/versioned/storage/common-tests/build.gradle.kts b/versioned/storage/common-tests/build.gradle.kts index b931e65cf8f..5c518dc2d07 100644 --- a/versioned/storage/common-tests/build.gradle.kts +++ b/versioned/storage/common-tests/build.gradle.kts @@ -49,8 +49,14 @@ dependencies { annotationProcessor(libs.immutables.value.processor) compileOnly(libs.microprofile.openapi) - compileOnly(platform(libs.jackson.bom)) - compileOnly("com.fasterxml.jackson.core:jackson-annotations") + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.core:jackson-annotations") + + // required for custom object serialization tests + implementation("com.fasterxml.jackson.core:jackson-databind") + implementation("com.fasterxml.jackson.datatype:jackson-datatype-guava") + implementation("com.fasterxml.jackson.datatype:jackson-datatype-jdk8") + implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310") implementation(platform(libs.junit.bom)) implementation(libs.bundles.junit.testing) diff --git a/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractBackendRepositoryTests.java b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractBackendRepositoryTests.java index 376bc3130a5..f05b85cf998 100644 --- a/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractBackendRepositoryTests.java +++ b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractBackendRepositoryTests.java @@ -20,7 +20,6 @@ import static org.projectnessie.versioned.storage.versionstore.RefMapping.REFS_HEADS; import java.util.Collections; -import java.util.EnumSet; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -43,7 +42,7 @@ import org.projectnessie.versioned.storage.common.persist.CloseableIterator; import org.projectnessie.versioned.storage.common.persist.Obj; import org.projectnessie.versioned.storage.common.persist.ObjId; -import org.projectnessie.versioned.storage.common.persist.ObjType; +import org.projectnessie.versioned.storage.common.persist.ObjTypes; import org.projectnessie.versioned.storage.common.persist.Persist; import org.projectnessie.versioned.storage.common.persist.PersistFactory; import org.projectnessie.versioned.storage.common.persist.Reference; @@ -81,7 +80,7 @@ public void createEraseRepoViaPersist() throws Exception { .toArray(Obj[]::new))) .hasSize(objs) .doesNotContain(false); - try (CloseableIterator scan = repo1.scanAllObjects(EnumSet.allOf(ObjType.class))) { + try (CloseableIterator scan = repo1.scanAllObjects(ObjTypes.allObjTypes())) { soft.assertThat(scan) .toIterable() .filteredOn( @@ -91,7 +90,7 @@ public void createEraseRepoViaPersist() throws Exception { repo1.erase(); soft.assertThat(repositoryLogic.repositoryExists()).isFalse(); - try (CloseableIterator scan = repo1.scanAllObjects(EnumSet.allOf(ObjType.class))) { + try (CloseableIterator scan = repo1.scanAllObjects(ObjTypes.allObjTypes())) { soft.assertThat(scan).isExhausted(); } } @@ -110,7 +109,7 @@ public void createEraseManyRepos(int numRepos) { soft.assertThat(repos) .noneMatch( r -> { - try (CloseableIterator scan = r.scanAllObjects(EnumSet.allOf(ObjType.class))) { + try (CloseableIterator scan = r.scanAllObjects(ObjTypes.allObjTypes())) { return scan.hasNext(); } }); @@ -153,7 +152,7 @@ public void createEraseSomeRepos() throws ObjTooLargeException, RefAlreadyExists soft.assertThat(toDelete) .noneMatch( r -> { - try (CloseableIterator scan = r.scanAllObjects(EnumSet.allOf(ObjType.class))) { + try (CloseableIterator scan = r.scanAllObjects(ObjTypes.allObjTypes())) { return scan.hasNext(); } }); @@ -163,7 +162,7 @@ public void createEraseSomeRepos() throws ObjTooLargeException, RefAlreadyExists for (int i = 0; i < refs; i++) { soft.assertThat(repo.fetchReference(REFS_HEADS + "reference-" + i)).isNotNull(); } - try (CloseableIterator scan = repo.scanAllObjects(EnumSet.allOf(ObjType.class))) { + try (CloseableIterator scan = repo.scanAllObjects(ObjTypes.allObjTypes())) { soft.assertThat(scan) .toIterable() .filteredOn( diff --git a/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractBasePersistTests.java b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractBasePersistTests.java index d42d51f198b..07e492b1acf 100644 --- a/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractBasePersistTests.java +++ b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractBasePersistTests.java @@ -48,24 +48,27 @@ import static org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj.indexSegments; import static org.projectnessie.versioned.storage.common.objtypes.IndexStripe.indexStripe; import static org.projectnessie.versioned.storage.common.objtypes.RefObj.ref; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.COMMIT; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.INDEX; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.INDEX_SEGMENTS; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.REF; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.STRING; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.TAG; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.VALUE; import static org.projectnessie.versioned.storage.common.objtypes.StringObj.stringData; import static org.projectnessie.versioned.storage.common.objtypes.TagObj.tag; import static org.projectnessie.versioned.storage.common.persist.ObjId.EMPTY_OBJ_ID; import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromString; import static org.projectnessie.versioned.storage.common.persist.ObjId.randomObjId; -import static org.projectnessie.versioned.storage.common.persist.ObjType.COMMIT; -import static org.projectnessie.versioned.storage.common.persist.ObjType.INDEX; -import static org.projectnessie.versioned.storage.common.persist.ObjType.INDEX_SEGMENTS; -import static org.projectnessie.versioned.storage.common.persist.ObjType.REF; -import static org.projectnessie.versioned.storage.common.persist.ObjType.STRING; -import static org.projectnessie.versioned.storage.common.persist.ObjType.TAG; -import static org.projectnessie.versioned.storage.common.persist.ObjType.VALUE; import static org.projectnessie.versioned.storage.common.persist.Reference.reference; import com.google.common.collect.Lists; +import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.EnumSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ThreadLocalRandom; import java.util.function.IntFunction; @@ -94,9 +97,7 @@ import org.projectnessie.versioned.storage.common.objtypes.CommitType; import org.projectnessie.versioned.storage.common.objtypes.Compression; import org.projectnessie.versioned.storage.common.objtypes.ContentValueObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj; -import org.projectnessie.versioned.storage.common.objtypes.RefObj; +import org.projectnessie.versioned.storage.common.objtypes.StandardObjType; import org.projectnessie.versioned.storage.common.objtypes.StringObj; import org.projectnessie.versioned.storage.common.objtypes.TagObj; import org.projectnessie.versioned.storage.common.persist.CloseableIterator; @@ -104,8 +105,11 @@ import org.projectnessie.versioned.storage.common.persist.Obj; import org.projectnessie.versioned.storage.common.persist.ObjId; import org.projectnessie.versioned.storage.common.persist.ObjType; +import org.projectnessie.versioned.storage.common.persist.ObjTypes; import org.projectnessie.versioned.storage.common.persist.Persist; import org.projectnessie.versioned.storage.common.persist.Reference; +import org.projectnessie.versioned.storage.commontests.objtypes.SimpleCustomObj; +import org.projectnessie.versioned.storage.commontests.objtypes.SimpleCustomObjType; import org.projectnessie.versioned.storage.testextension.NessiePersist; import org.projectnessie.versioned.storage.testextension.NessieStoreConfig; import org.projectnessie.versioned.storage.testextension.PersistExtension; @@ -435,50 +439,46 @@ public static Stream allObjectTypeSamples() { objIdFromString("0000000000000000")), copyFromUtf8("This is not a markdown")), ref(randomObjId(), "foo", randomObjId(), 123L, null), - ref(randomObjId(), "bar", randomObjId(), 456L, randomObjId())); + ref(randomObjId(), "bar", randomObjId(), 456L, randomObjId()), + // custom object types + SimpleCustomObj.builder() + .id(randomObjId()) + .parent(randomObjId()) + .text("foo") + .number(42.42d) + .map(Map.of("k1", "v1", "k2", "v2")) + .list(List.of("a", "b", "c")) + .optional("optional") + .instant(Instant.ofEpochMilli(1234567890L)) + .build(), + SimpleCustomObj.builder().id(randomObjId()).build()); } - @SuppressWarnings("rawtypes") - static Class classForType(ObjType type) { - switch (type) { - case COMMIT: - return CommitObj.class; - case VALUE: - return ContentValueObj.class; - case INDEX_SEGMENTS: - return IndexSegmentsObj.class; - case INDEX: - return IndexObj.class; - case REF: - return RefObj.class; - case TAG: - return TagObj.class; - case STRING: - return StringObj.class; - default: - throw new IllegalArgumentException(type.name()); + static StandardObjType typeDifferentThan(ObjType type) { + if (type instanceof StandardObjType) { + switch (((StandardObjType) type)) { + case COMMIT: + return VALUE; + case VALUE: + return COMMIT; + case INDEX_SEGMENTS: + return TAG; + case INDEX: + return REF; + case REF: + return INDEX; + case TAG: + return STRING; + case STRING: + return INDEX_SEGMENTS; + default: + // fall through + } } - } - - static ObjType typeDifferentThan(ObjType type) { - switch (type) { - case COMMIT: - return VALUE; - case VALUE: - return COMMIT; - case INDEX_SEGMENTS: - return TAG; - case INDEX: - return REF; - case REF: - return INDEX; - case TAG: - return STRING; - case STRING: - return INDEX_SEGMENTS; - default: - throw new IllegalArgumentException(type.name()); + if (type instanceof SimpleCustomObjType) { + return StandardObjType.COMMIT; } + throw new IllegalArgumentException(type.name()); } @SuppressWarnings("unchecked") @@ -490,7 +490,7 @@ public void singleObjectCreateDelete(Obj obj) throws Exception { soft.assertThatThrownBy(() -> persist.fetchObjType(obj.id())) .isInstanceOf(ObjNotFoundException.class); soft.assertThatThrownBy( - () -> persist.fetchTypedObj(obj.id(), obj.type(), classForType(obj.type()))) + () -> persist.fetchTypedObj(obj.id(), obj.type(), obj.type().targetClass())) .isInstanceOf(ObjNotFoundException.class); soft.assertThatThrownBy(() -> persist.fetchObjs(new ObjId[] {obj.id()})) .isInstanceOf(ObjNotFoundException.class); @@ -499,11 +499,11 @@ public void singleObjectCreateDelete(Obj obj) throws Exception { soft.assertThat(persist.fetchObj(obj.id())).isEqualTo(obj); soft.assertThat(persist.fetchObjType(obj.id())).isEqualTo(obj.type()); - soft.assertThat(persist.fetchTypedObj(obj.id(), obj.type(), classForType(obj.type()))) + soft.assertThat(persist.fetchTypedObj(obj.id(), obj.type(), obj.type().targetClass())) .isEqualTo(obj); - ObjType otherType = typeDifferentThan(obj.type()); + StandardObjType otherType = typeDifferentThan(obj.type()); soft.assertThatThrownBy( - () -> persist.fetchTypedObj(obj.id(), otherType, classForType(otherType))) + () -> persist.fetchTypedObj(obj.id(), otherType, otherType.targetClass())) .isInstanceOf(ObjNotFoundException.class); soft.assertThat(persist.fetchObjs(new ObjId[] {obj.id()})).containsExactly(obj); @@ -516,7 +516,7 @@ public void singleObjectCreateDelete(Obj obj) throws Exception { soft.assertThatThrownBy(() -> persist.fetchObjType(obj.id())) .isInstanceOf(ObjNotFoundException.class); soft.assertThatThrownBy( - () -> persist.fetchTypedObj(obj.id(), obj.type(), classForType(obj.type()))) + () -> persist.fetchTypedObj(obj.id(), obj.type(), obj.type().targetClass())) .isInstanceOf(ObjNotFoundException.class); soft.assertThatThrownBy(() -> persist.fetchObjs(new ObjId[] {obj.id()})) .isInstanceOf(ObjNotFoundException.class); @@ -721,20 +721,20 @@ public void scanAllObjects( .isNotEqualTo(otherRepo.config().repositoryId()); ArrayList list1; - try (CloseableIterator iter = persist.scanAllObjects(EnumSet.allOf(ObjType.class))) { + try (CloseableIterator iter = persist.scanAllObjects(ObjTypes.allObjTypes())) { list1 = newArrayList(iter); } ArrayList list2; - try (CloseableIterator iter = otherRepo.scanAllObjects(EnumSet.allOf(ObjType.class))) { + try (CloseableIterator iter = otherRepo.scanAllObjects(ObjTypes.allObjTypes())) { list2 = newArrayList(iter); } soft.assertThat(list1).isNotEmpty().doesNotContainAnyElementsOf(list2); soft.assertThat(list2).isNotEmpty().doesNotContainAnyElementsOf(list1); - try (CloseableIterator iter = otherRepo.scanAllObjects(EnumSet.of(COMMIT))) { + try (CloseableIterator iter = otherRepo.scanAllObjects(Set.of(COMMIT))) { soft.assertThat(newArrayList(iter)).isNotEmpty().allMatch(o -> o.type() == COMMIT); } - try (CloseableIterator iter = otherRepo.scanAllObjects(EnumSet.of(COMMIT))) { + try (CloseableIterator iter = otherRepo.scanAllObjects(Set.of(COMMIT))) { soft.assertThat(newArrayList(iter)).isNotEmpty().allMatch(o -> o.type() == COMMIT); } } @@ -847,61 +847,65 @@ public void updateMultipleObjects() throws Exception { } public static Obj updateObjChange(Obj obj) { - Obj newObj; - switch (obj.type()) { - case COMMIT: - StoreIndex index = newStoreIndex(COMMIT_OP_SERIALIZER); - index.add(indexElement(key("updated", "added", "key"), commitOp(ADD, 123, randomObjId()))); - index.add( - indexElement(key("updated", "removed", "key"), commitOp(REMOVE, 123, randomObjId()))); - CommitObj c = ((CommitObj) obj); - newObj = - commitBuilder() - .id(obj.id()) - .created(123123L) - .headers(newCommitHeaders().add("update", "that").build()) - .message("updated commit") - .incrementalIndex(index.serialize()) - .commitType(CommitType.values()[c.commitType().ordinal() ^ 1]) - .seq(1231231253L) - .incompleteIndex(!c.incompleteIndex()) - .build(); - break; - case VALUE: - newObj = contentValue(obj.id(), randomContentId(), 123, copyFromUtf8("updated stuff")); - break; - case REF: - newObj = ref(obj.id(), "hello", randomObjId(), 42L, randomObjId()); - break; - case INDEX: - index = newStoreIndex(COMMIT_OP_SERIALIZER); - index.add(indexElement(key("updated", "added", "key"), commitOp(ADD, 123, randomObjId()))); - index.add( - indexElement(key("updated", "removed", "key"), commitOp(REMOVE, 123, randomObjId()))); - newObj = index(obj.id(), index.serialize()); - break; - case INDEX_SEGMENTS: - newObj = - indexSegments( - obj.id(), singletonList(indexStripe(key("abc"), key("def"), randomObjId()))); - break; - case TAG: - newObj = tag(obj.id(), null, null, copyFromUtf8("updated-tag")); - break; - case STRING: - newObj = - stringData( - obj.id(), - "text/plain", - Compression.LZ4, - "filename", - asList(randomObjId(), randomObjId(), randomObjId(), randomObjId()), - ByteString.copyFrom(new byte[123])); - break; - default: - throw new UnsupportedOperationException("Unknown object type " + obj.type()); + ObjType type = obj.type(); + if (type instanceof StandardObjType) { + switch (((StandardObjType) type)) { + case COMMIT: + StoreIndex index = newStoreIndex(COMMIT_OP_SERIALIZER); + index.add( + indexElement(key("updated", "added", "key"), commitOp(ADD, 123, randomObjId()))); + index.add( + indexElement(key("updated", "removed", "key"), commitOp(REMOVE, 123, randomObjId()))); + CommitObj c = ((CommitObj) obj); + return commitBuilder() + .id(obj.id()) + .created(123123L) + .headers(newCommitHeaders().add("update", "that").build()) + .message("updated commit") + .incrementalIndex(index.serialize()) + .commitType(CommitType.values()[c.commitType().ordinal() ^ 1]) + .seq(1231231253L) + .incompleteIndex(!c.incompleteIndex()) + .build(); + case VALUE: + return contentValue(obj.id(), randomContentId(), 123, copyFromUtf8("updated stuff")); + case REF: + return ref(obj.id(), "hello", randomObjId(), 42L, randomObjId()); + case INDEX: + index = newStoreIndex(COMMIT_OP_SERIALIZER); + index.add( + indexElement(key("updated", "added", "key"), commitOp(ADD, 123, randomObjId()))); + index.add( + indexElement(key("updated", "removed", "key"), commitOp(REMOVE, 123, randomObjId()))); + return index(obj.id(), index.serialize()); + case INDEX_SEGMENTS: + return indexSegments( + obj.id(), singletonList(indexStripe(key("abc"), key("def"), randomObjId()))); + case TAG: + return tag(obj.id(), null, null, copyFromUtf8("updated-tag")); + case STRING: + return stringData( + obj.id(), + "text/plain", + Compression.LZ4, + "filename", + asList(randomObjId(), randomObjId(), randomObjId(), randomObjId()), + ByteString.copyFrom(new byte[123])); + default: + // fall through + } + } + if (obj instanceof SimpleCustomObj) { + return SimpleCustomObj.builder() + .id(obj.id()) + .parent(randomObjId()) + .text("updated") + .number(43.43d) + .map(Map.of("k2", "v2", "k3", "v3")) + .list(List.of("b", "c", "d")) + .build(); } - return newObj; + throw new UnsupportedOperationException("Unknown object type " + type); } @ParameterizedTest @@ -943,32 +947,32 @@ public void scanAllObjects(int numObjs) throws Exception { storedAssert.containsOnly(true); } - try (CloseableIterator scan = persist.scanAllObjects(EnumSet.allOf(ObjType.class))) { + try (CloseableIterator scan = persist.scanAllObjects(ObjTypes.allObjTypes())) { soft.assertThat(Lists.newArrayList(scan)) .hasSize(3 * numObjs) .contains(values) .contains(strings) .contains(commits); } - try (CloseableIterator scan = persist.scanAllObjects(EnumSet.of(VALUE, STRING))) { + try (CloseableIterator scan = persist.scanAllObjects(Set.of(VALUE, STRING))) { soft.assertThat(Lists.newArrayList(scan)) .hasSize(2 * numObjs) .contains(values) .contains(strings); } - try (CloseableIterator scan = persist.scanAllObjects(EnumSet.of(VALUE, COMMIT))) { + try (CloseableIterator scan = persist.scanAllObjects(Set.of(VALUE, COMMIT))) { soft.assertThat(Lists.newArrayList(scan)) .hasSize(2 * numObjs) .contains(values) .contains(commits); } - try (CloseableIterator scan = persist.scanAllObjects(EnumSet.of(COMMIT))) { + try (CloseableIterator scan = persist.scanAllObjects(Set.of(COMMIT))) { soft.assertThat(Lists.newArrayList(scan)).containsExactlyInAnyOrder(commits); } - try (CloseableIterator scan = persist.scanAllObjects(EnumSet.of(COMMIT, TAG, INDEX))) { + try (CloseableIterator scan = persist.scanAllObjects(Set.of(COMMIT, TAG, INDEX))) { soft.assertThat(Lists.newArrayList(scan)).containsExactlyInAnyOrder(commits); } - try (CloseableIterator scan = persist.scanAllObjects(EnumSet.of(TAG, INDEX))) { + try (CloseableIterator scan = persist.scanAllObjects(Set.of(TAG, INDEX))) { soft.assertThat(Lists.newArrayList(scan)).isEmpty(); } } @@ -981,6 +985,10 @@ public void scanAllObjects(int numObjs) throws Exception { @Test public void createObjectsWithUpsertThenFetchAndScan() throws Exception { Obj[] objs = allObjectTypeSamples().toArray(Obj[]::new); + Obj[] standardObjs = + Arrays.stream(objs).filter(o -> o.type() instanceof StandardObjType).toArray(Obj[]::new); + Obj[] testObjs = + Arrays.stream(objs).filter(o -> o instanceof SimpleCustomObj).toArray(Obj[]::new); persist.erase(); @@ -992,10 +1000,18 @@ public void createObjectsWithUpsertThenFetchAndScan() throws Exception { soft.assertThat(persist.fetchObjs(stream(objs).map(Obj::id).toArray(ObjId[]::new))) .containsExactly(objs); - try (CloseableIterator scan = persist.scanAllObjects(EnumSet.allOf(ObjType.class))) { + try (CloseableIterator scan = persist.scanAllObjects(ObjTypes.allObjTypes())) { soft.assertThat(Lists.newArrayList(scan)).containsExactlyInAnyOrder(objs); } - try (CloseableIterator scan = persist.scanAllObjects(EnumSet.of(COMMIT))) { + try (CloseableIterator scan = + persist.scanAllObjects(Set.copyOf(EnumSet.allOf(StandardObjType.class)))) { + soft.assertThat(Lists.newArrayList(scan)).containsExactlyInAnyOrder(standardObjs); + } + try (CloseableIterator scan = + persist.scanAllObjects(Set.of(SimpleCustomObjType.INSTANCE))) { + soft.assertThat(Lists.newArrayList(scan)).containsExactlyInAnyOrder(testObjs); + } + try (CloseableIterator scan = persist.scanAllObjects(Set.of(COMMIT))) { Obj[] expected = stream(objs).filter(c -> c.type() == COMMIT).toArray(Obj[]::new); soft.assertThat(newArrayList(scan)).containsExactlyInAnyOrder(expected); } diff --git a/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractConsistencyLogicTests.java b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractConsistencyLogicTests.java index 3eed2d503c5..8dc8ce21dad 100644 --- a/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractConsistencyLogicTests.java +++ b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractConsistencyLogicTests.java @@ -97,9 +97,9 @@ import org.projectnessie.versioned.storage.common.objtypes.CommitObj; import org.projectnessie.versioned.storage.common.objtypes.CommitOp; import org.projectnessie.versioned.storage.common.objtypes.ContentValueObj; +import org.projectnessie.versioned.storage.common.objtypes.StandardObjType; import org.projectnessie.versioned.storage.common.persist.Obj; import org.projectnessie.versioned.storage.common.persist.ObjId; -import org.projectnessie.versioned.storage.common.persist.ObjType; import org.projectnessie.versioned.storage.common.persist.Persist; import org.projectnessie.versioned.storage.common.persist.Reference; import org.projectnessie.versioned.storage.testextension.NessiePersist; @@ -1023,7 +1023,8 @@ public void corruptIntRefsIndex( refRefs = persist.fetchReference(InternalRef.REF_REFS.name()); refRefsHead = - persist.fetchTypedObj(requireNonNull(refRefs).pointer(), ObjType.COMMIT, CommitObj.class); + persist.fetchTypedObj( + requireNonNull(refRefs).pointer(), StandardObjType.COMMIT, CommitObj.class); if (refRefsHead.referenceIndex() != null) { refsUntilRefIndex = refRefsHead.referenceIndex(); break; @@ -1044,7 +1045,8 @@ public void corruptIntRefsIndex( refRefs = persist.fetchReference(InternalRef.REF_REFS.name()); refRefsHead = - persist.fetchTypedObj(requireNonNull(refRefs).pointer(), ObjType.COMMIT, CommitObj.class); + persist.fetchTypedObj( + requireNonNull(refRefs).pointer(), StandardObjType.COMMIT, CommitObj.class); if (!Objects.equals(refsUntilRefIndex, refRefsHead.referenceIndex())) { lostReferences.add(ref); break; diff --git a/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractIndexesLogicTests.java b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractIndexesLogicTests.java index 92c6eefb06e..c2bc8b7f4b7 100644 --- a/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractIndexesLogicTests.java +++ b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractIndexesLogicTests.java @@ -60,8 +60,8 @@ import org.projectnessie.versioned.storage.common.logic.SuppliedCommitIndex; import org.projectnessie.versioned.storage.common.objtypes.CommitObj; import org.projectnessie.versioned.storage.common.objtypes.CommitOp; +import org.projectnessie.versioned.storage.common.objtypes.StandardObjType; import org.projectnessie.versioned.storage.common.persist.ObjId; -import org.projectnessie.versioned.storage.common.persist.ObjType; import org.projectnessie.versioned.storage.common.persist.Persist; import org.projectnessie.versioned.storage.testextension.NessiePersist; import org.projectnessie.versioned.storage.testextension.PersistExtension; @@ -362,7 +362,7 @@ void completeIndexesInCommitChainWithStripes( indexesLogic.completeIndexesInCommitChain(currentId, () -> {}); for (int i = 5; i > 0; i--) { - CommitObj current = persist.fetchTypedObj(currentId, ObjType.COMMIT, CommitObj.class); + CommitObj current = persist.fetchTypedObj(currentId, StandardObjType.COMMIT, CommitObj.class); if (i > 1) { // All commits except commit 1 should have stripes soft.assertThat(current.referenceIndexStripes()).isNotEmpty(); diff --git a/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractRepositoryLogicTests.java b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractRepositoryLogicTests.java index b496e117f60..3439197746e 100644 --- a/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractRepositoryLogicTests.java +++ b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/AbstractRepositoryLogicTests.java @@ -27,13 +27,12 @@ import static org.projectnessie.versioned.storage.common.logic.Logics.repositoryLogic; import static org.projectnessie.versioned.storage.common.logic.ReferencesQuery.referencesQuery; import static org.projectnessie.versioned.storage.common.objtypes.CommitHeaders.EMPTY_COMMIT_HEADERS; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.COMMIT; import static org.projectnessie.versioned.storage.common.persist.ObjId.EMPTY_OBJ_ID; import static org.projectnessie.versioned.storage.common.persist.ObjId.randomObjId; -import static org.projectnessie.versioned.storage.common.persist.ObjType.COMMIT; import static org.projectnessie.versioned.storage.common.persist.Reference.reference; import java.time.Instant; -import java.util.EnumSet; import java.util.Set; import java.util.function.Consumer; import java.util.stream.Stream; @@ -58,7 +57,7 @@ import org.projectnessie.versioned.storage.common.persist.CloseableIterator; import org.projectnessie.versioned.storage.common.persist.Obj; import org.projectnessie.versioned.storage.common.persist.ObjId; -import org.projectnessie.versioned.storage.common.persist.ObjType; +import org.projectnessie.versioned.storage.common.persist.ObjTypes; import org.projectnessie.versioned.storage.common.persist.Persist; import org.projectnessie.versioned.storage.common.persist.Reference; import org.projectnessie.versioned.storage.testextension.NessiePersist; @@ -83,7 +82,7 @@ public void repositoryExists() { persist.erase(); soft.assertThat(repositoryLogic.repositoryExists()).isFalse(); - try (CloseableIterator iter = persist.scanAllObjects(EnumSet.allOf(ObjType.class))) { + try (CloseableIterator iter = persist.scanAllObjects(ObjTypes.allObjTypes())) { soft.assertThat(iter).isExhausted(); } } diff --git a/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObj.java b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObj.java new file mode 100644 index 00000000000..f30b847668c --- /dev/null +++ b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObj.java @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.commontests.objtypes; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import javax.annotation.Nullable; +import org.immutables.value.Value; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.common.persist.ObjType; + +@Value.Immutable +@JsonSerialize(as = ImmutableSimpleCustomObj.class) +@JsonDeserialize(using = SimpleCustomObjDeserializer.class) +public interface SimpleCustomObj extends Obj { + + @Override + default ObjType type() { + return SimpleCustomObjType.INSTANCE; + } + + @Nullable + @jakarta.annotation.Nullable + ObjId parent(); + + @Nullable + @jakarta.annotation.Nullable + String text(); + + @Nullable + @jakarta.annotation.Nullable + Number number(); + + @Nullable + @jakarta.annotation.Nullable + Map map(); + + @Nullable + @jakarta.annotation.Nullable + List list(); + + @Nullable + @jakarta.annotation.Nullable + Instant instant(); + + Optional optional(); + + static ImmutableSimpleCustomObj.Builder builder() { + return ImmutableSimpleCustomObj.builder(); + } +} diff --git a/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObjDeserializer.java b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObjDeserializer.java new file mode 100644 index 00000000000..b1c5e520cc2 --- /dev/null +++ b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObjDeserializer.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.commontests.objtypes; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import java.io.IOException; +import org.projectnessie.versioned.storage.common.json.ObjIdHelper; + +public class SimpleCustomObjDeserializer extends JsonDeserializer { + + @SuppressWarnings("deprecation") + @Override + public SimpleCustomObj deserialize(JsonParser p, DeserializationContext ctx) throws IOException { + ImmutableSimpleCustomObj.Json json = p.readValueAs(ImmutableSimpleCustomObj.Json.class); + json.setId(ObjIdHelper.retrieveObjIdFromContext(ctx)); + return ImmutableSimpleCustomObj.fromJson(json); + } +} diff --git a/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObjType.java b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObjType.java new file mode 100644 index 00000000000..1dd7e832689 --- /dev/null +++ b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObjType.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.commontests.objtypes; + +import org.projectnessie.versioned.storage.common.persist.ObjType; + +public final class SimpleCustomObjType implements ObjType { + + public static final SimpleCustomObjType INSTANCE = new SimpleCustomObjType(); + + private SimpleCustomObjType() {} + + @Override + public String name() { + return "test"; + } + + @Override + public String shortName() { + return "tt"; + } + + @Override + public Class targetClass() { + return SimpleCustomObj.class; + } +} diff --git a/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObjTypeBundle.java b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObjTypeBundle.java new file mode 100644 index 00000000000..16bcd351ac2 --- /dev/null +++ b/versioned/storage/common-tests/src/main/java/org/projectnessie/versioned/storage/commontests/objtypes/SimpleCustomObjTypeBundle.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.commontests.objtypes; + +import java.util.function.Consumer; +import org.projectnessie.versioned.storage.common.persist.ObjType; +import org.projectnessie.versioned.storage.common.persist.ObjTypeBundle; + +public class SimpleCustomObjTypeBundle implements ObjTypeBundle { + + @Override + public void register(Consumer registrar) { + registrar.accept(SimpleCustomObjType.INSTANCE); + } +} diff --git a/versioned/storage/common-tests/src/main/resources/META-INF/services/org.projectnessie.versioned.storage.common.persist.ObjTypeBundle b/versioned/storage/common-tests/src/main/resources/META-INF/services/org.projectnessie.versioned.storage.common.persist.ObjTypeBundle new file mode 100644 index 00000000000..7cd4a5fcbde --- /dev/null +++ b/versioned/storage/common-tests/src/main/resources/META-INF/services/org.projectnessie.versioned.storage.common.persist.ObjTypeBundle @@ -0,0 +1,17 @@ +# +# Copyright (C) 2023 Dremio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.projectnessie.versioned.storage.commontests.objtypes.SimpleCustomObjTypeBundle diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/ObjIdDeserializer.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/ObjIdDeserializer.java new file mode 100644 index 00000000000..0c295572b2b --- /dev/null +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/ObjIdDeserializer.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.common.json; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import java.io.IOException; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public class ObjIdDeserializer extends StdDeserializer { + + public ObjIdDeserializer() { + super(ObjId.class); + } + + @Override + public ObjId deserialize(JsonParser p, DeserializationContext ctx) throws IOException { + return ObjId.objIdFromByteArray(p.getBinaryValue()); + } +} diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/ObjIdHelper.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/ObjIdHelper.java new file mode 100644 index 00000000000..eac66c28fd7 --- /dev/null +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/ObjIdHelper.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.common.json; + +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectReader; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public final class ObjIdHelper { + + private static final String DESERIALIZATION_CTX_ATTR_KEY = "nessie.storage.ObjId"; + + /** Retrieves an {@link ObjId} from the {@link DeserializationContext}. */ + public static ObjId retrieveObjIdFromContext(DeserializationContext ctx) { + return (ObjId) ctx.getAttribute(DESERIALIZATION_CTX_ATTR_KEY); + } + + /** Returns an {@link ObjectReader} having the {@link ObjId} stored in its context attributes. */ + public static ObjectReader storeObjIdInContext(ObjectMapper mapper, ObjId id) { + return mapper.reader().withAttribute(DESERIALIZATION_CTX_ATTR_KEY, id); + } + + private ObjIdHelper() {} +} diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/ObjIdSerializer.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/ObjIdSerializer.java new file mode 100644 index 00000000000..ca3c006df85 --- /dev/null +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/ObjIdSerializer.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.common.json; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import java.io.IOException; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public class ObjIdSerializer extends StdSerializer { + + public ObjIdSerializer() { + super(ObjId.class); + } + + @Override + public void serialize(ObjId value, JsonGenerator gen, SerializerProvider provider) + throws IOException { + gen.writeBinary(value.asByteArray()); + } +} diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/PersistModule.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/PersistModule.java new file mode 100644 index 00000000000..4c8385829df --- /dev/null +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/json/PersistModule.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.common.json; + +import com.fasterxml.jackson.databind.module.SimpleModule; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public class PersistModule extends SimpleModule { + + private static final long serialVersionUID = 1L; + + public PersistModule() { + addSerializer(new ObjIdSerializer()); + addDeserializer(ObjId.class, new ObjIdDeserializer()); + } +} diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/CommitLogicImpl.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/CommitLogicImpl.java index ef25c2be6ae..8cfcdeecca6 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/CommitLogicImpl.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/CommitLogicImpl.java @@ -51,14 +51,14 @@ import static org.projectnessie.versioned.storage.common.objtypes.Hashes.hashAsObjId; import static org.projectnessie.versioned.storage.common.objtypes.Hashes.hashCommitHeaders; import static org.projectnessie.versioned.storage.common.objtypes.Hashes.newHasher; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.COMMIT; import static org.projectnessie.versioned.storage.common.persist.ObjId.EMPTY_OBJ_ID; -import static org.projectnessie.versioned.storage.common.persist.ObjType.COMMIT; import com.google.common.collect.AbstractIterator; import com.google.common.hash.Hasher; import java.util.ArrayList; import java.util.Arrays; -import java.util.EnumSet; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -1115,7 +1115,7 @@ public HeadsAndForkPoints identifyAllHeadsAndForkPoints( // scanAllCommitLogEntries() returns all commits in no specific order, parents may be scanned // before or after their children. - try (CloseableIterator scan = persist.scanAllObjects(EnumSet.of(COMMIT))) { + try (CloseableIterator scan = persist.scanAllObjects(Collections.singleton(COMMIT))) { while (scan.hasNext()) { CommitObj commit = (CommitObj) scan.next(); diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/ConsistencyLogicImpl.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/ConsistencyLogicImpl.java index 80f902571d8..4810096a36b 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/ConsistencyLogicImpl.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/ConsistencyLogicImpl.java @@ -28,8 +28,8 @@ import org.projectnessie.versioned.storage.common.objtypes.CommitObj; import org.projectnessie.versioned.storage.common.objtypes.CommitOp; import org.projectnessie.versioned.storage.common.objtypes.IndexStripe; +import org.projectnessie.versioned.storage.common.objtypes.StandardObjType; import org.projectnessie.versioned.storage.common.persist.ObjId; -import org.projectnessie.versioned.storage.common.persist.ObjType; import org.projectnessie.versioned.storage.common.persist.Persist; import org.projectnessie.versioned.storage.common.persist.Reference; @@ -60,7 +60,7 @@ public R checkCommit(ObjId commitId, CommitStatusCallback callback) { Set test = new HashSet<>(); CommitObj commitObj = null; try { - commitObj = persist.fetchTypedObj(commitId, ObjType.COMMIT, CommitObj.class); + commitObj = persist.fetchTypedObj(commitId, StandardObjType.COMMIT, CommitObj.class); if (commitObj.referenceIndex() != null) { test.add(commitObj.referenceIndex()); diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/IndexesLogicImpl.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/IndexesLogicImpl.java index 131f2f0590f..2394ccc9939 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/IndexesLogicImpl.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/IndexesLogicImpl.java @@ -35,9 +35,9 @@ import static org.projectnessie.versioned.storage.common.objtypes.IndexObj.index; import static org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj.indexSegments; import static org.projectnessie.versioned.storage.common.objtypes.IndexStripe.indexStripe; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.COMMIT; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.INDEX; import static org.projectnessie.versioned.storage.common.persist.ObjId.EMPTY_OBJ_ID; -import static org.projectnessie.versioned.storage.common.persist.ObjType.COMMIT; -import static org.projectnessie.versioned.storage.common.persist.ObjType.INDEX; import static org.projectnessie.versioned.storage.common.util.SupplyOnce.memoize; import com.google.common.annotations.VisibleForTesting; @@ -65,6 +65,7 @@ import org.projectnessie.versioned.storage.common.objtypes.IndexObj; import org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj; import org.projectnessie.versioned.storage.common.objtypes.IndexStripe; +import org.projectnessie.versioned.storage.common.objtypes.StandardObjType; import org.projectnessie.versioned.storage.common.persist.Obj; import org.projectnessie.versioned.storage.common.persist.ObjId; import org.projectnessie.versioned.storage.common.persist.ObjType; @@ -243,7 +244,6 @@ public StoreIndex buildReferenceIndexOnly( private StoreIndex loadReferenceIndex( @Nonnull @jakarta.annotation.Nonnull ObjId indexId, @Nonnull @jakarta.annotation.Nonnull ObjId commitId) { - StoreIndex referenceIndex; Obj keyIndex; try { keyIndex = persist.fetchObj(indexId); @@ -252,21 +252,21 @@ private StoreIndex loadReferenceIndex( format("Commit %s references a reference index, which does not exist", indexId)); } ObjType indexType = keyIndex.type(); - switch (indexType) { - case INDEX_SEGMENTS: - IndexSegmentsObj split = (IndexSegmentsObj) keyIndex; - List indexStripes = split.stripes(); - referenceIndex = referenceIndexFromStripes(indexStripes, commitId); - break; - case INDEX: - referenceIndex = deserializeIndex(((IndexObj) keyIndex).index()).setObjId(keyIndex.id()); - break; - default: - throw new IllegalStateException( - "Commit %s references a reference index, which is of unsupported key index type " - + indexType); + if (indexType instanceof StandardObjType) { + switch ((StandardObjType) indexType) { + case INDEX_SEGMENTS: + IndexSegmentsObj split = (IndexSegmentsObj) keyIndex; + List indexStripes = split.stripes(); + return referenceIndexFromStripes(indexStripes, commitId); + case INDEX: + return deserializeIndex(((IndexObj) keyIndex).index()).setObjId(keyIndex.id()); + default: + // fall through + } } - return referenceIndex; + throw new IllegalStateException( + "Commit %s references a reference index, which is of unsupported key index type " + + indexType); } static StoreIndex deserializeIndex(ByteString serialized) { diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/ReferenceLogicImpl.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/ReferenceLogicImpl.java index 830afd79a55..83f305bec07 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/ReferenceLogicImpl.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/ReferenceLogicImpl.java @@ -39,9 +39,9 @@ import static org.projectnessie.versioned.storage.common.logic.ReferenceLogicImpl.CommitReferenceResult.Kind.REF_ROW_MISSING; import static org.projectnessie.versioned.storage.common.objtypes.CommitHeaders.newCommitHeaders; import static org.projectnessie.versioned.storage.common.objtypes.RefObj.ref; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.COMMIT; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.REF; import static org.projectnessie.versioned.storage.common.persist.ObjId.EMPTY_OBJ_ID; -import static org.projectnessie.versioned.storage.common.persist.ObjType.COMMIT; -import static org.projectnessie.versioned.storage.common.persist.ObjType.REF; import static org.projectnessie.versioned.storage.common.persist.Reference.INTERNAL_PREFIX; import static org.projectnessie.versioned.storage.common.persist.Reference.isInternalReferenceName; import static org.projectnessie.versioned.storage.common.persist.Reference.reference; diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/StringLogicImpl.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/StringLogicImpl.java index 421a32a734b..19b31266bbc 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/StringLogicImpl.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/logic/StringLogicImpl.java @@ -24,8 +24,8 @@ import static org.projectnessie.versioned.storage.common.logic.Logics.commitLogic; import static org.projectnessie.versioned.storage.common.logic.Logics.indexesLogic; import static org.projectnessie.versioned.storage.common.objtypes.CommitHeaders.EMPTY_COMMIT_HEADERS; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.STRING; import static org.projectnessie.versioned.storage.common.objtypes.StringObj.stringData; -import static org.projectnessie.versioned.storage.common.persist.ObjType.STRING; import java.nio.charset.StandardCharsets; import java.util.UUID; diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/CommitObj.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/CommitObj.java index 1c6060aa452..f306e65579d 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/CommitObj.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/CommitObj.java @@ -77,7 +77,7 @@ public interface CommitObj extends Obj { @Override default ObjType type() { - return ObjType.COMMIT; + return StandardObjType.COMMIT; } static Builder commitBuilder() { @@ -184,17 +184,17 @@ default boolean incompleteIndex() { /** * Pointer to the reference {@link StoreKey}-to-{@link CommitOp} index for this commit. * - *

This value, if not {@code null}, can point to {@link ObjType#INDEX_SEGMENTS}, in which case - * the reference index is already big and organized in multiple segments, or to {@link - * ObjType#INDEX} when the full index is small enough to fit into a single segment and indirection - * is not necessary. + *

This value, if not {@code null}, can point to {@link StandardObjType#INDEX_SEGMENTS}, in + * which case the reference index is already big and organized in multiple segments, or to {@link + * StandardObjType#INDEX} when the full index is small enough to fit into a single segment and + * indirection is not necessary. * *

A {@code null} value means that the "embedded" {@link #incrementalIndex()} was never big * enough, a "reference index" does not exist and {@link #incrementalIndex()} contains everything. * - *

An external {@link ObjType#INDEX_SEGMENTS} object will only be created, if the number of - * stripes is higher than {@link StoreConfig#maxReferenceStripesPerCommit()}, otherwise the - * stripes for the reference index will be stored {@link #referenceIndexStripes() inside} the + *

An external {@link StandardObjType#INDEX_SEGMENTS} object will only be created, if the + * number of stripes is higher than {@link StoreConfig#maxReferenceStripesPerCommit()}, otherwise + * the stripes for the reference index will be stored {@link #referenceIndexStripes() inside} the * commit. * * @see #incrementalIndex() @@ -207,9 +207,9 @@ default boolean incompleteIndex() { /** * Pointers to the composite reference index stripes, an "embedded" version of {@link - * ObjType#INDEX_SEGMENTS}. Commits that require to "externalize" index elements to a reference - * index, which requires up to {@link StoreConfig#maxReferenceStripesPerCommit()} will be kept - * here and not create another indirection via a {@link IndexSegmentsObj}. + * StandardObjType#INDEX_SEGMENTS}. Commits that require to "externalize" index elements to a + * reference index, which requires up to {@link StoreConfig#maxReferenceStripesPerCommit()} will + * be kept here and not create another indirection via a {@link IndexSegmentsObj}. * * @see #incrementalIndex() * @see #incompleteIndex() diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/ContentValueObj.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/ContentValueObj.java index ddd701e68f7..90d30bd2169 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/ContentValueObj.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/ContentValueObj.java @@ -29,7 +29,7 @@ public interface ContentValueObj extends Obj { @Override default ObjType type() { - return ObjType.VALUE; + return StandardObjType.VALUE; } @Override diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/Hashes.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/Hashes.java index 568930dd28b..e0f3daea079 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/Hashes.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/Hashes.java @@ -16,12 +16,12 @@ package org.projectnessie.versioned.storage.common.objtypes; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.projectnessie.versioned.storage.common.persist.ObjType.INDEX; -import static org.projectnessie.versioned.storage.common.persist.ObjType.INDEX_SEGMENTS; -import static org.projectnessie.versioned.storage.common.persist.ObjType.REF; -import static org.projectnessie.versioned.storage.common.persist.ObjType.STRING; -import static org.projectnessie.versioned.storage.common.persist.ObjType.TAG; -import static org.projectnessie.versioned.storage.common.persist.ObjType.VALUE; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.INDEX; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.INDEX_SEGMENTS; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.REF; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.STRING; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.TAG; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.VALUE; import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/IndexObj.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/IndexObj.java index bc1327c1530..398bfce5de2 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/IndexObj.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/IndexObj.java @@ -30,7 +30,7 @@ public interface IndexObj extends Obj { @Override default ObjType type() { - return ObjType.INDEX; + return StandardObjType.INDEX; } @Override diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/IndexSegmentsObj.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/IndexSegmentsObj.java index 3c8429a486e..1143af156f0 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/IndexSegmentsObj.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/IndexSegmentsObj.java @@ -30,7 +30,7 @@ public interface IndexSegmentsObj extends Obj { @Override default ObjType type() { - return ObjType.INDEX_SEGMENTS; + return StandardObjType.INDEX_SEGMENTS; } @Override diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/RefObj.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/RefObj.java index ee708718b4a..888271f2499 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/RefObj.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/RefObj.java @@ -35,7 +35,7 @@ public interface RefObj extends Obj { @Override default ObjType type() { - return ObjType.REF; + return StandardObjType.REF; } @Value.Parameter(order = 1) diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/StandardObjType.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/StandardObjType.java new file mode 100644 index 00000000000..ad5bf4a9d49 --- /dev/null +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/StandardObjType.java @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.common.objtypes; + +import org.projectnessie.versioned.storage.common.logic.InternalRef; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjType; + +public enum StandardObjType implements ObjType { + /** + * Identifies a named reference and contains the initial referencee. + * + *

Managed in the well-known internal reference {@link InternalRef#REF_REFS}. + * + *

{@link Obj} is a {@link RefObj}. + */ + REF("r", RefObj.class), + + /** {@link Obj} is a {@link CommitObj}. */ + COMMIT("c", CommitObj.class), + + /** {@link Obj} is a {@link TagObj}. */ + TAG("t", TagObj.class), + + /** {@link Obj} is a {@link ContentValueObj}. */ + VALUE("v", ContentValueObj.class), + + /** {@link Obj} is a {@link StringObj}. */ + STRING("s", StringObj.class), + + /** {@link Obj} is a {@link IndexSegmentsObj}. */ + INDEX_SEGMENTS("I", IndexSegmentsObj.class), + + /** {@link Obj} is a {@link IndexObj}. */ + INDEX("i", IndexObj.class); + + private final String shortName; + + private final Class targetClass; + + StandardObjType(String shortName, Class targetClass) { + this.shortName = shortName; + this.targetClass = targetClass; + } + + @Override + public String shortName() { + return shortName; + } + + @Override + public Class targetClass() { + return targetClass; + } +} diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/StandardObjTypeBundle.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/StandardObjTypeBundle.java new file mode 100644 index 00000000000..a34e1372a57 --- /dev/null +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/StandardObjTypeBundle.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.common.objtypes; + +import java.util.function.Consumer; +import org.projectnessie.versioned.storage.common.persist.ObjType; +import org.projectnessie.versioned.storage.common.persist.ObjTypeBundle; + +public class StandardObjTypeBundle implements ObjTypeBundle { + + @Override + public void register(Consumer registrar) { + for (StandardObjType objType : StandardObjType.values()) { + registrar.accept(objType); + } + } +} diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/StringObj.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/StringObj.java index 1366277c6d9..bc0efad0006 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/StringObj.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/StringObj.java @@ -30,7 +30,7 @@ public interface StringObj extends Obj { @Override default ObjType type() { - return ObjType.STRING; + return StandardObjType.STRING; } @Override diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/TagObj.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/TagObj.java index ea64deca564..0f8ff3c9cae 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/TagObj.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/objtypes/TagObj.java @@ -29,7 +29,7 @@ public interface TagObj extends Obj { @Override default ObjType type() { - return ObjType.TAG; + return StandardObjType.TAG; } @Override diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/Obj.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/Obj.java index ca65dc428ce..07547735a3b 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/Obj.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/Obj.java @@ -15,9 +15,12 @@ */ package org.projectnessie.versioned.storage.common.persist; +import com.fasterxml.jackson.annotation.JsonIgnore; + public interface Obj { /** The ID of this object. */ + @JsonIgnore ObjId id(); ObjType type(); diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/ObjType.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/ObjType.java index 1d2b20392f2..70808f6d65f 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/ObjType.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/ObjType.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 Dremio + * Copyright (C) 2023 Dremio * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,61 +15,14 @@ */ package org.projectnessie.versioned.storage.common.persist; -import org.projectnessie.versioned.storage.common.logic.InternalRef; -import org.projectnessie.versioned.storage.common.objtypes.CommitObj; -import org.projectnessie.versioned.storage.common.objtypes.ContentValueObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj; -import org.projectnessie.versioned.storage.common.objtypes.RefObj; -import org.projectnessie.versioned.storage.common.objtypes.StringObj; -import org.projectnessie.versioned.storage.common.objtypes.TagObj; +public interface ObjType { -public enum ObjType { - /** - * Identifies a named reference and contains the initial referencee. - * - *

Managed in the well-known internal reference {@link InternalRef#REF_REFS}. - * - *

{@link Obj} is a {@link RefObj}. - */ - REF("r"), + /** Must be unique among all registered object types. */ + String name(); - /** {@link Obj} is a {@link CommitObj}. */ - COMMIT("c"), + /** Must be unique among all registered object types. */ + String shortName(); - /** {@link Obj} is a {@link TagObj}. */ - TAG("t"), - - /** {@link Obj} is a {@link ContentValueObj}. */ - VALUE("v"), - - /** {@link Obj} is a {@link StringObj}. */ - STRING("s"), - - /** {@link Obj} is a {@link IndexSegmentsObj}. */ - INDEX_SEGMENTS("I"), - - /** {@link Obj} is a {@link IndexObj}. */ - INDEX("i"); - - private static final ObjType[] ALL_OBJ_TYPES = ObjType.values(); - - private final String shortName; - - ObjType(String shortName) { - this.shortName = shortName; - } - - public static ObjType fromShortName(String shortName) { - for (ObjType type : ALL_OBJ_TYPES) { - if (type.shortName().equals(shortName)) { - return type; - } - } - throw new IllegalStateException("Unknown object short type name " + shortName); - } - - public String shortName() { - return shortName; - } + /** The target class that objects of this type should be serialized from and deserialized to. */ + Class targetClass(); } diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/ObjTypeBundle.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/ObjTypeBundle.java new file mode 100644 index 00000000000..a6df06ea414 --- /dev/null +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/ObjTypeBundle.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.common.persist; + +import java.util.function.Consumer; + +public interface ObjTypeBundle { + + void register(Consumer registrar); +} diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/ObjTypes.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/ObjTypes.java new file mode 100644 index 00000000000..3820696deac --- /dev/null +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/ObjTypes.java @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.common.persist; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.Set; +import java.util.TreeMap; +import javax.annotation.Nonnull; + +public final class ObjTypes { + + /** Returns all registered {@link ObjType}s. */ + @Nonnull + @jakarta.annotation.Nonnull + public static Set allObjTypes() { + return Registry.OBJ_TYPES; + } + + @Nonnull + @jakarta.annotation.Nonnull + public static ObjType forName(@Nonnull @jakarta.annotation.Nonnull String name) { + ObjType type = Registry.BY_NAME.get(name); + checkArgument(type != null, "Unknown object type name: %s", name); + return type; + } + + @Nonnull + @jakarta.annotation.Nonnull + public static ObjType forShortName(@Nonnull @jakarta.annotation.Nonnull String shortName) { + ObjType type = Registry.BY_SHORT_NAME.get(shortName); + checkArgument(type != null, "Unknown object type short name: %s", shortName); + return type; + } + + private static final class Registry { + private static final Map BY_NAME; + private static final Map BY_SHORT_NAME; + private static final Set OBJ_TYPES; + + static { + Map byName = new TreeMap<>(); + Map byShortName = new HashMap<>(); + for (ObjTypeBundle bundle : ServiceLoader.load(ObjTypeBundle.class)) { + bundle.register( + objType -> { + if (byName.put(objType.name(), objType) != null) { + throw new IllegalStateException("Duplicate object type name: " + objType.name()); + } + if (byShortName.put(objType.shortName(), objType) != null) { + throw new IllegalStateException( + "Duplicate object type short name: " + objType.shortName()); + } + }); + } + BY_NAME = Collections.unmodifiableMap(byName); + BY_SHORT_NAME = Collections.unmodifiableMap(byShortName); + OBJ_TYPES = Set.copyOf(byName.values()); + } + } +} diff --git a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/Reference.java b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/Reference.java index 874283e9c07..9e85ef442fc 100644 --- a/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/Reference.java +++ b/versioned/storage/common/src/main/java/org/projectnessie/versioned/storage/common/persist/Reference.java @@ -24,15 +24,16 @@ import org.immutables.value.Value; import org.projectnessie.versioned.storage.common.config.StoreConfig; import org.projectnessie.versioned.storage.common.logic.InternalRef; +import org.projectnessie.versioned.storage.common.objtypes.StandardObjType; /** * Reference is a generic named pointer. * *

    *
  • Branches: reference names for branches start with {@code refs/heads/} and points to {@link - * ObjType#COMMIT}. + * StandardObjType#COMMIT}. *
  • Tags: reference names for branches start with {@code refs/tags/} and points to {@link - * ObjType#TAG} or {@link ObjType#COMMIT}. + * StandardObjType#TAG} or {@link StandardObjType#COMMIT}. *
  • Internal references, not publicly exposed, see {@link InternalRef}. *
*/ diff --git a/versioned/storage/common/src/main/resources/META-INF/services/com.fasterxml.jackson.databind.Module b/versioned/storage/common/src/main/resources/META-INF/services/com.fasterxml.jackson.databind.Module new file mode 100644 index 00000000000..98d4864e830 --- /dev/null +++ b/versioned/storage/common/src/main/resources/META-INF/services/com.fasterxml.jackson.databind.Module @@ -0,0 +1,17 @@ +# +# Copyright (C) 2023 Dremio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.projectnessie.versioned.storage.common.json.PersistModule diff --git a/versioned/storage/common/src/main/resources/META-INF/services/org.projectnessie.versioned.storage.common.persist.ObjTypeBundle b/versioned/storage/common/src/main/resources/META-INF/services/org.projectnessie.versioned.storage.common.persist.ObjTypeBundle new file mode 100644 index 00000000000..618978c03d8 --- /dev/null +++ b/versioned/storage/common/src/main/resources/META-INF/services/org.projectnessie.versioned.storage.common.persist.ObjTypeBundle @@ -0,0 +1,17 @@ +# +# Copyright (C) 2023 Dremio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.projectnessie.versioned.storage.common.objtypes.StandardObjTypeBundle diff --git a/versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/json/TestObjIdSerialization.java b/versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/json/TestObjIdSerialization.java new file mode 100644 index 00000000000..1aa82a6ed36 --- /dev/null +++ b/versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/json/TestObjIdSerialization.java @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.common.json; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +class TestObjIdSerialization { + + final ObjectMapper objectMapper = new ObjectMapper().registerModule(new PersistModule()); + + static Stream objectIds() { + return Stream.of( + ObjId.EMPTY_OBJ_ID, + // generic + ObjId.objIdFromString("cafebabe"), + // SHA-256 + ObjId.objIdFromString("cafebabecafebabecafebabecafebabecafebabecafebabecafebabecafebabe")); + } + + @ParameterizedTest + @MethodSource("objectIds") + void testSerializeDeserialize(ObjId id) throws IOException { + byte[] bytes = objectMapper.writeValueAsBytes(id); + ObjId actual = objectMapper.readValue(bytes, ObjId.class); + assertThat(actual).isEqualTo(id); + } +} diff --git a/versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/logic/TestReferenceIndexes.java b/versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/logic/TestReferenceIndexes.java index acba09e097c..bf751863d8e 100644 --- a/versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/logic/TestReferenceIndexes.java +++ b/versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/logic/TestReferenceIndexes.java @@ -47,11 +47,11 @@ import static org.projectnessie.versioned.storage.common.objtypes.CommitOp.Action.REMOVE; import static org.projectnessie.versioned.storage.common.objtypes.CommitOp.COMMIT_OP_SERIALIZER; import static org.projectnessie.versioned.storage.common.objtypes.ContentValueObj.contentValue; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.COMMIT; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.INDEX; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.INDEX_SEGMENTS; import static org.projectnessie.versioned.storage.common.persist.ObjId.EMPTY_OBJ_ID; import static org.projectnessie.versioned.storage.common.persist.ObjId.randomObjId; -import static org.projectnessie.versioned.storage.common.persist.ObjType.COMMIT; -import static org.projectnessie.versioned.storage.common.persist.ObjType.INDEX; -import static org.projectnessie.versioned.storage.common.persist.ObjType.INDEX_SEGMENTS; import static org.projectnessie.versioned.storage.commontests.AbstractCommitLogicTests.stdCommit; import com.google.common.base.Strings; @@ -227,7 +227,7 @@ public void referenceIndexLifecycle( // Adds new keys, one per commits. // Loop until the amount of allowed reference index stripes per commit is exceeded and - // a reference index lookup object is created (ObjType.INDEX_SEGMENTS). + // a reference index lookup object is created (StandardObjType.INDEX_SEGMENTS). for (num++; ; num++) { head = commitWithValues(persist, commitLogic, add, contents, num, head); @@ -283,7 +283,7 @@ public void referenceIndexLifecycle( } // Remove keys now... until we're back at "embedded" reference segments list (no longer need - // ObjType.INDEX_SEGMENTS)... + // StandardObjType.INDEX_SEGMENTS)... for (num--; num >= 0; num--) { head = requireNonNull(commitLogic.doCommit(remove.apply(head, num), emptyList())).id(); diff --git a/versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/objtypes/TestObjTypes.java b/versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/objtypes/TestStandardObjTypes.java similarity index 92% rename from versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/objtypes/TestObjTypes.java rename to versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/objtypes/TestStandardObjTypes.java index f00902f8595..b2b97af1d23 100644 --- a/versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/objtypes/TestObjTypes.java +++ b/versioned/storage/common/src/test/java/org/projectnessie/versioned/storage/common/objtypes/TestStandardObjTypes.java @@ -36,18 +36,18 @@ import static org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj.indexSegments; import static org.projectnessie.versioned.storage.common.objtypes.IndexStripe.indexStripe; import static org.projectnessie.versioned.storage.common.objtypes.RefObj.ref; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.COMMIT; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.INDEX; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.INDEX_SEGMENTS; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.REF; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.STRING; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.TAG; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.VALUE; import static org.projectnessie.versioned.storage.common.objtypes.StringObj.stringData; import static org.projectnessie.versioned.storage.common.objtypes.TagObj.tag; import static org.projectnessie.versioned.storage.common.persist.ObjId.EMPTY_OBJ_ID; import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromString; import static org.projectnessie.versioned.storage.common.persist.ObjId.randomObjId; -import static org.projectnessie.versioned.storage.common.persist.ObjType.COMMIT; -import static org.projectnessie.versioned.storage.common.persist.ObjType.INDEX; -import static org.projectnessie.versioned.storage.common.persist.ObjType.INDEX_SEGMENTS; -import static org.projectnessie.versioned.storage.common.persist.ObjType.REF; -import static org.projectnessie.versioned.storage.common.persist.ObjType.STRING; -import static org.projectnessie.versioned.storage.common.persist.ObjType.TAG; -import static org.projectnessie.versioned.storage.common.persist.ObjType.VALUE; import java.util.List; import java.util.stream.Collectors; @@ -66,10 +66,10 @@ import org.projectnessie.versioned.storage.common.persist.ObjType; @ExtendWith(SoftAssertionsExtension.class) -public class TestObjTypes { +public class TestStandardObjTypes { @InjectSoftAssertions protected SoftAssertions soft; - static Stream objTypes() { + static Stream standardObjTypes() { return Stream.of( arguments(ref(randomObjId(), "hello", randomObjId(), 42L, randomObjId()), REF), arguments( @@ -105,8 +105,8 @@ static Stream objTypes() { } @ParameterizedTest - @MethodSource("objTypes") - public void objTypes(Obj obj, ObjType type) { + @MethodSource("standardObjTypes") + public void standardObjTypes(Obj obj, ObjType type) { soft.assertThat(obj).extracting(Obj::type).isSameAs(type); } diff --git a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/DynamoDBConstants.java b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/DynamoDBConstants.java index 0637f602b6f..065816f794e 100644 --- a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/DynamoDBConstants.java +++ b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/DynamoDBConstants.java @@ -37,55 +37,9 @@ final class DynamoDBConstants { static final String KEY_NAME = "k"; static final String COL_OBJ_TYPE = "y"; - static final String COL_COMMIT = "c"; - static final String COL_REF = "e"; - static final String COL_VALUE = "v"; - static final String COL_SEGMENTS = "I"; - static final String COL_INDEX = "i"; - static final String COL_TAG = "t"; - static final String COL_STRING = "s"; static final String CONDITION_STORE_REF = "attribute_not_exists(" + COL_REFERENCES_POINTER + ")"; static final String CONDITION_STORE_OBJ = "attribute_not_exists(" + COL_OBJ_TYPE + ")"; - static final String COL_COMMIT_CREATED = "c"; - static final String COL_COMMIT_SEQ = "q"; - static final String COL_COMMIT_MESSAGE = "m"; - static final String COL_COMMIT_HEADERS = "h"; - static final String COL_COMMIT_REFERENCE_INDEX = "x"; - static final String COL_COMMIT_REFERENCE_INDEX_STRIPES = "r"; - static final String COL_COMMIT_TAIL = "t"; - static final String COL_COMMIT_SECONDARY_PARENTS = "s"; - static final String COL_COMMIT_INCREMENTAL_INDEX = "i"; - static final String COL_COMMIT_INCOMPLETE_INDEX = "n"; - static final String COL_COMMIT_TYPE = "y"; - - static final String COL_REF_NAME = "n"; - static final String COL_REF_INITIAL_POINTER = "p"; - static final String COL_REF_CREATED_AT = "c"; - static final String COL_REF_EXTENDED_INFO = "e"; - - static final String COL_VALUE_CONTENT_ID = "i"; - static final String COL_VALUE_PAYLOAD = "p"; - static final String COL_VALUE_DATA = "d"; - - static final String COL_SEGMENTS_STRIPES = "s"; - static final String COL_STRIPES_FIRST_KEY = "f"; - static final String COL_STRIPES_LAST_KEY = "l"; - static final String COL_STRIPES_SEGMENT = "s"; - - static final String COL_INDEX_INDEX = "i"; - - static final String COL_TAG_COMMIT_ID = "i"; - static final String COL_TAG_MESSAGE = "m"; - static final String COL_TAG_HEADERS = "h"; - static final String COL_TAG_SIGNATURE = "s"; - - static final String COL_STRING_CONTENT_TYPE = "y"; - static final String COL_STRING_COMPRESSION = "c"; - static final String COL_STRING_FILENAME = "f"; - static final String COL_STRING_PREDECESSORS = "p"; - static final String COL_STRING_TEXT = "t"; - private DynamoDBConstants() {} } diff --git a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/DynamoDBPersist.java b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/DynamoDBPersist.java index 274044d92a5..045bfa04d05 100644 --- a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/DynamoDBPersist.java +++ b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/DynamoDBPersist.java @@ -16,73 +16,22 @@ package org.projectnessie.versioned.storage.dynamodb; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; import static java.util.Collections.emptyList; import static java.util.Collections.emptyListIterator; import static java.util.Collections.singleton; import static java.util.Collections.singletonMap; -import static org.projectnessie.nessie.relocated.protobuf.UnsafeByteOperations.unsafeWrap; -import static org.projectnessie.versioned.storage.common.indexes.StoreKey.keyFromString; -import static org.projectnessie.versioned.storage.common.objtypes.CommitHeaders.newCommitHeaders; -import static org.projectnessie.versioned.storage.common.objtypes.CommitObj.commitBuilder; -import static org.projectnessie.versioned.storage.common.objtypes.ContentValueObj.contentValue; -import static org.projectnessie.versioned.storage.common.objtypes.IndexObj.index; -import static org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj.indexSegments; -import static org.projectnessie.versioned.storage.common.objtypes.IndexStripe.indexStripe; -import static org.projectnessie.versioned.storage.common.objtypes.RefObj.ref; -import static org.projectnessie.versioned.storage.common.objtypes.StringObj.stringData; -import static org.projectnessie.versioned.storage.common.objtypes.TagObj.tag; -import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromByteBuffer; import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromString; import static org.projectnessie.versioned.storage.common.persist.Reference.reference; import static org.projectnessie.versioned.storage.dynamodb.DynamoDBBackend.condition; import static org.projectnessie.versioned.storage.dynamodb.DynamoDBBackend.keyPrefix; import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.BATCH_GET_LIMIT; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_COMMIT; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_COMMIT_CREATED; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_COMMIT_HEADERS; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_COMMIT_INCOMPLETE_INDEX; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_COMMIT_INCREMENTAL_INDEX; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_COMMIT_MESSAGE; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_COMMIT_REFERENCE_INDEX; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_COMMIT_REFERENCE_INDEX_STRIPES; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_COMMIT_SECONDARY_PARENTS; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_COMMIT_SEQ; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_COMMIT_TAIL; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_COMMIT_TYPE; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_INDEX; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_INDEX_INDEX; import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_OBJ_TYPE; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_REF; import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_REFERENCES_CONDITION_COMMON; import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_REFERENCES_CREATED_AT; import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_REFERENCES_DELETED; import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_REFERENCES_EXTENDED_INFO; import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_REFERENCES_POINTER; import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_REFERENCES_PREVIOUS; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_REF_CREATED_AT; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_REF_EXTENDED_INFO; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_REF_INITIAL_POINTER; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_REF_NAME; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_SEGMENTS; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_SEGMENTS_STRIPES; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_STRING; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_STRING_COMPRESSION; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_STRING_CONTENT_TYPE; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_STRING_FILENAME; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_STRING_PREDECESSORS; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_STRING_TEXT; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_STRIPES_FIRST_KEY; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_STRIPES_LAST_KEY; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_STRIPES_SEGMENT; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_TAG; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_TAG_HEADERS; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_TAG_MESSAGE; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_TAG_SIGNATURE; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_VALUE; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_VALUE_CONTENT_ID; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_VALUE_DATA; -import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.COL_VALUE_PAYLOAD; import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.CONDITION_STORE_OBJ; import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.CONDITION_STORE_REF; import static org.projectnessie.versioned.storage.dynamodb.DynamoDBConstants.ITEM_SIZE_LIMIT; @@ -90,10 +39,8 @@ import static org.projectnessie.versioned.storage.serialize.ProtoSerialization.deserializePreviousPointers; import static org.projectnessie.versioned.storage.serialize.ProtoSerialization.serializePreviousPointers; import static software.amazon.awssdk.core.SdkBytes.fromByteArray; -import static software.amazon.awssdk.core.SdkBytes.fromByteBuffer; import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromB; import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromBool; -import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromL; import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromM; import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromS; import static software.amazon.awssdk.services.dynamodb.model.ComparisonOperator.BEGINS_WITH; @@ -101,45 +48,32 @@ import com.google.common.collect.AbstractIterator; import java.util.ArrayList; -import java.util.EnumMap; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; -import java.util.function.Consumer; -import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.agrona.collections.Hashing; import org.agrona.collections.Object2IntHashMap; -import org.projectnessie.nessie.relocated.protobuf.ByteString; import org.projectnessie.versioned.storage.common.config.StoreConfig; import org.projectnessie.versioned.storage.common.exceptions.ObjNotFoundException; import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; import org.projectnessie.versioned.storage.common.exceptions.RefAlreadyExistsException; import org.projectnessie.versioned.storage.common.exceptions.RefConditionFailedException; import org.projectnessie.versioned.storage.common.exceptions.RefNotFoundException; -import org.projectnessie.versioned.storage.common.objtypes.CommitHeaders; -import org.projectnessie.versioned.storage.common.objtypes.CommitObj; -import org.projectnessie.versioned.storage.common.objtypes.CommitType; -import org.projectnessie.versioned.storage.common.objtypes.Compression; -import org.projectnessie.versioned.storage.common.objtypes.ContentValueObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexStripe; -import org.projectnessie.versioned.storage.common.objtypes.RefObj; -import org.projectnessie.versioned.storage.common.objtypes.StringObj; -import org.projectnessie.versioned.storage.common.objtypes.TagObj; import org.projectnessie.versioned.storage.common.persist.CloseableIterator; import org.projectnessie.versioned.storage.common.persist.Obj; import org.projectnessie.versioned.storage.common.persist.ObjId; import org.projectnessie.versioned.storage.common.persist.ObjType; +import org.projectnessie.versioned.storage.common.persist.ObjTypes; import org.projectnessie.versioned.storage.common.persist.Persist; import org.projectnessie.versioned.storage.common.persist.Reference; +import org.projectnessie.versioned.storage.dynamodb.serializers.ObjSerializer; +import org.projectnessie.versioned.storage.dynamodb.serializers.ObjSerializers; import software.amazon.awssdk.awscore.exception.AwsErrorDetails; -import software.amazon.awssdk.core.SdkBytes; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.BatchGetItemResponse; import software.amazon.awssdk.services.dynamodb.model.Condition; @@ -151,8 +85,6 @@ public class DynamoDBPersist implements Persist { - private static final Map> STORE_OBJ_TYPE = new EnumMap<>(ObjType.class); - private final DynamoDBBackend backend; private final StoreConfig config; private final String keyPrefix; @@ -281,14 +213,14 @@ public Reference fetchReference(@Nonnull @jakarta.annotation.Nonnull String name Map i = item.item(); - String createdAtStr = attributeToString(i, COL_REFERENCES_CREATED_AT); + String createdAtStr = DynamoDBSerde.attributeToString(i, COL_REFERENCES_CREATED_AT); long createdAt = createdAtStr != null ? Long.parseLong(createdAtStr) : 0L; return reference( name, - attributeToObjId(i, COL_REFERENCES_POINTER), - attributeToBool(i, COL_REFERENCES_DELETED), + DynamoDBSerde.attributeToObjId(i, COL_REFERENCES_POINTER), + DynamoDBSerde.attributeToBool(i, COL_REFERENCES_DELETED), createdAt, - attributeToObjId(i, COL_REFERENCES_EXTENDED_INFO), + DynamoDBSerde.attributeToObjId(i, COL_REFERENCES_EXTENDED_INFO), attributeToPreviousPointers(i)); } @@ -338,15 +270,16 @@ private void findReferencesPage( .forEach( item -> { String name = item.get(KEY_NAME).s().substring(keyPrefix.length()); - String createdAtStr = attributeToString(item, COL_REFERENCES_CREATED_AT); + String createdAtStr = + DynamoDBSerde.attributeToString(item, COL_REFERENCES_CREATED_AT); long createdAt = createdAtStr != null ? Long.parseLong(createdAtStr) : 0L; Reference reference = reference( name, - attributeToObjId(item, COL_REFERENCES_POINTER), - attributeToBool(item, COL_REFERENCES_DELETED), + DynamoDBSerde.attributeToObjId(item, COL_REFERENCES_POINTER), + DynamoDBSerde.attributeToBool(item, COL_REFERENCES_DELETED), createdAt, - attributeToObjId(item, COL_REFERENCES_EXTENDED_INFO), + DynamoDBSerde.attributeToObjId(item, COL_REFERENCES_EXTENDED_INFO), attributeToPreviousPointers(item)); int idx = nameToIndex.getValue(name); if (idx >= 0) { @@ -374,7 +307,7 @@ public Obj fetchObj(@Nonnull @jakarta.annotation.Nonnull ObjId id) throws ObjNot throw new ObjNotFoundException(id); } - return decomposeObj(item.item()); + return itemToObj(item.item()); } @Override @@ -389,8 +322,8 @@ public T fetchTypedObj( throw new ObjNotFoundException(id); } - Obj obj = decomposeObj(item.item()); - if (obj.type() != type) { + Obj obj = itemToObj(item.item()); + if (!obj.type().equals(type)) { throw new ObjNotFoundException(id); } @@ -416,7 +349,7 @@ public ObjType fetchObjType(@Nonnull @jakarta.annotation.Nonnull ObjId id) throw new ObjNotFoundException(id); } - return objTypeFromItem(item.item()); + return ObjTypes.forShortName(item.item().get(COL_OBJ_TYPE).s()); } @Nonnull @@ -477,7 +410,7 @@ private void fetchObjsPage( .get(backend.tableObjs) .forEach( item -> { - Obj obj = decomposeObj(item); + Obj obj = itemToObj(item); int idx = idToIndex.getValue(obj.id()); if (idx != -1) { r[idx] = obj; @@ -598,36 +531,22 @@ public void erase() { backend.eraseRepositories(singleton(config().repositoryId())); } - private ObjType objTypeFromItem(Map item) { - return objTypeFromItem(item.get(COL_OBJ_TYPE)); - } - - private ObjType objTypeFromItem(AttributeValue attributeValue) { - String shortType = attributeValue.s(); - return ObjType.fromShortName(shortType); - } - - @SuppressWarnings("unchecked") - private Obj decomposeObj(Map item) { + private Obj itemToObj(Map item) { ObjId id = objIdFromString(item.get(KEY_NAME).s().substring(keyPrefix.length())); - ObjType type = objTypeFromItem(item); - @SuppressWarnings("rawtypes") - StoreObjDesc storeObj = STORE_OBJ_TYPE.get(type); - checkState(storeObj != null, "Cannot deserialize object type %s", type); - Map inner = item.get(storeObj.typeName).m(); - return storeObj.fromMap(id, inner); + AttributeValue attributeValue = item.get(COL_OBJ_TYPE); + ObjType type = ObjTypes.forShortName(attributeValue.s()); + ObjSerializer serializer = ObjSerializers.forType(type); + Map inner = item.get(serializer.attributeName()).m(); + return serializer.fromMap(id, inner); } - @SuppressWarnings("unchecked") @Nonnull @jakarta.annotation.Nonnull private Map objToItem( @Nonnull @jakarta.annotation.Nonnull Obj obj, ObjId id, boolean ignoreSoftSizeRestrictions) throws ObjTooLargeException { ObjType type = obj.type(); - @SuppressWarnings("rawtypes") - StoreObjDesc storeObj = STORE_OBJ_TYPE.get(type); - checkArgument(storeObj != null, "Cannot serialize object type %s ", type); + ObjSerializer serializer = ObjSerializers.forType(type); Map item = new HashMap<>(); Map inner = new HashMap<>(); @@ -637,321 +556,11 @@ private Map objToItem( ignoreSoftSizeRestrictions ? Integer.MAX_VALUE : effectiveIncrementalIndexSizeLimit(); int indexSizeLimit = ignoreSoftSizeRestrictions ? Integer.MAX_VALUE : effectiveIndexSegmentSizeLimit(); - storeObj.toMap(obj, inner, incrementalIndexSizeLimit, indexSizeLimit); - item.put(storeObj.typeName, fromM(inner)); + serializer.toMap(obj, inner, incrementalIndexSizeLimit, indexSizeLimit); + item.put(serializer.attributeName(), fromM(inner)); return item; } - private abstract static class StoreObjDesc { - final String typeName; - - StoreObjDesc(String typeName) { - this.typeName = typeName; - } - - abstract void toMap( - O obj, Map i, int incrementalIndexSize, int maxSerializedIndexSize) - throws ObjTooLargeException; - - abstract O fromMap(ObjId id, Map i); - } - - static { - STORE_OBJ_TYPE.put( - ObjType.COMMIT, - new StoreObjDesc(COL_COMMIT) { - @Override - void toMap( - CommitObj obj, - Map i, - int incrementalIndexSize, - int maxSerializedIndexSize) - throws ObjTooLargeException { - i.put(COL_COMMIT_SEQ, fromS(Long.toString(obj.seq()))); - i.put(COL_COMMIT_CREATED, fromS(Long.toString(obj.created()))); - ObjId referenceIndex = obj.referenceIndex(); - if (referenceIndex != null) { - objIdToAttribute(i, COL_COMMIT_REFERENCE_INDEX, referenceIndex); - } - i.put(COL_COMMIT_MESSAGE, fromS(obj.message())); - objIdsAttribute(i, COL_COMMIT_TAIL, obj.tail()); - objIdsAttribute(i, COL_COMMIT_SECONDARY_PARENTS, obj.secondaryParents()); - - ByteString index = obj.incrementalIndex(); - if (index.size() > incrementalIndexSize) { - throw new ObjTooLargeException(index.size(), incrementalIndexSize); - } - bytesAttribute(i, COL_COMMIT_INCREMENTAL_INDEX, index); - - if (!obj.referenceIndexStripes().isEmpty()) { - i.put( - COL_COMMIT_REFERENCE_INDEX_STRIPES, stripesAttrList(obj.referenceIndexStripes())); - } - - Map headerMap = new HashMap<>(); - CommitHeaders headers = obj.headers(); - for (String s : headers.keySet()) { - headerMap.put( - s, - fromL( - headers.getAll(s).stream() - .map(AttributeValue::fromS) - .collect(Collectors.toList()))); - } - if (!headerMap.isEmpty()) { - i.put(COL_COMMIT_HEADERS, fromM(headerMap)); - } - i.put(COL_COMMIT_INCOMPLETE_INDEX, fromBool(obj.incompleteIndex())); - i.put(COL_COMMIT_TYPE, fromS(obj.commitType().shortName())); - } - - @Override - CommitObj fromMap(ObjId id, Map i) { - CommitObj.Builder b = - commitBuilder() - .id(id) - .seq(Long.parseLong(attributeToString(i, COL_COMMIT_SEQ))) - .created(Long.parseLong(attributeToString(i, COL_COMMIT_CREATED))) - .message(attributeToString(i, COL_COMMIT_MESSAGE)) - .incrementalIndex(attributeToBytes(i, COL_COMMIT_INCREMENTAL_INDEX)) - .incompleteIndex(attributeToBool(i, COL_COMMIT_INCOMPLETE_INDEX)) - .commitType(CommitType.fromShortName(attributeToString(i, COL_COMMIT_TYPE))); - AttributeValue v = i.get(COL_COMMIT_REFERENCE_INDEX); - if (v != null) { - b.referenceIndex(attributeToObjId(v)); - } - - fromStripesAttrList( - i.get(COL_COMMIT_REFERENCE_INDEX_STRIPES), b::addReferenceIndexStripes); - - attributeToObjIds(i, COL_COMMIT_TAIL, b::addTail); - attributeToObjIds(i, COL_COMMIT_SECONDARY_PARENTS, b::addSecondaryParents); - - CommitHeaders.Builder headers = newCommitHeaders(); - AttributeValue headerMap = i.get(COL_COMMIT_HEADERS); - if (headerMap != null) { - headerMap.m().forEach((k, l) -> l.l().forEach(hv -> headers.add(k, hv.s()))); - } - b.headers(headers.build()); - - return b.build(); - } - }); - - STORE_OBJ_TYPE.put( - ObjType.REF, - new StoreObjDesc(COL_REF) { - @Override - void toMap( - RefObj obj, - Map i, - int incrementalIndexSize, - int maxSerializedIndexSize) { - i.put(COL_REF_NAME, fromS(obj.name())); - i.put(COL_REF_CREATED_AT, fromS(Long.toString(obj.createdAtMicros()))); - objIdToAttribute(i, COL_REF_INITIAL_POINTER, obj.initialPointer()); - ObjId extendedInfoObj = obj.extendedInfoObj(); - if (extendedInfoObj != null) { - objIdToAttribute(i, COL_REF_EXTENDED_INFO, extendedInfoObj); - } - } - - @Override - RefObj fromMap(ObjId id, Map i) { - String createdAtStr = attributeToString(i, COL_REFERENCES_CREATED_AT); - long createdAt = createdAtStr != null ? Long.parseLong(createdAtStr) : 0L; - return ref( - id, - attributeToString(i, COL_REF_NAME), - attributeToObjId(i, COL_REF_INITIAL_POINTER), - createdAt, - attributeToObjId(i, COL_REF_EXTENDED_INFO)); - } - }); - - STORE_OBJ_TYPE.put( - ObjType.VALUE, - new StoreObjDesc(COL_VALUE) { - @Override - void toMap( - ContentValueObj obj, - Map i, - int incrementalIndexSize, - int maxSerializedIndexSize) { - i.put(COL_VALUE_CONTENT_ID, fromS(obj.contentId())); - i.put(COL_VALUE_PAYLOAD, fromS(Integer.toString(obj.payload()))); - bytesAttribute(i, COL_VALUE_DATA, obj.data()); - } - - @Override - ContentValueObj fromMap(ObjId id, Map i) { - return contentValue( - id, - attributeToString(i, COL_VALUE_CONTENT_ID), - Integer.parseInt(attributeToString(i, COL_VALUE_PAYLOAD)), - attributeToBytes(i, COL_VALUE_DATA)); - } - }); - - STORE_OBJ_TYPE.put( - ObjType.INDEX_SEGMENTS, - new StoreObjDesc(COL_SEGMENTS) { - @Override - void toMap( - IndexSegmentsObj obj, - Map i, - int incrementalIndexSize, - int maxSerializedIndexSize) { - i.put(COL_SEGMENTS_STRIPES, stripesAttrList(obj.stripes())); - } - - @Override - IndexSegmentsObj fromMap(ObjId id, Map i) { - List stripes = new ArrayList<>(); - fromStripesAttrList(i.get(COL_SEGMENTS_STRIPES), stripes::add); - return indexSegments(id, stripes); - } - }); - - STORE_OBJ_TYPE.put( - ObjType.INDEX, - new StoreObjDesc(COL_INDEX) { - @Override - void toMap( - IndexObj obj, - Map i, - int incrementalIndexSize, - int maxSerializedIndexSize) - throws ObjTooLargeException { - ByteString index = obj.index(); - if (index.size() > maxSerializedIndexSize) { - throw new ObjTooLargeException(index.size(), maxSerializedIndexSize); - } - bytesAttribute(i, COL_INDEX_INDEX, index); - } - - @Override - IndexObj fromMap(ObjId id, Map i) { - return index(id, attributeToBytes(i, COL_INDEX_INDEX)); - } - }); - - STORE_OBJ_TYPE.put( - ObjType.TAG, - new StoreObjDesc(COL_TAG) { - @Override - void toMap( - TagObj obj, - Map i, - int incrementalIndexSize, - int maxSerializedIndexSize) { - String message = obj.message(); - if (message != null) { - i.put(COL_TAG_MESSAGE, fromS(message)); - } - - Map headerMap = new HashMap<>(); - CommitHeaders headers = obj.headers(); - if (headers != null) { - for (String s : headers.keySet()) { - headerMap.put( - s, - fromL( - headers.getAll(s).stream() - .map(AttributeValue::fromS) - .collect(Collectors.toList()))); - } - if (!headerMap.isEmpty()) { - i.put(COL_TAG_HEADERS, fromM(headerMap)); - } - } - - ByteString signature = obj.signature(); - if (signature != null) { - bytesAttribute(i, COL_TAG_SIGNATURE, signature); - } - } - - @Override - TagObj fromMap(ObjId id, Map i) { - CommitHeaders tagHeaders = null; - AttributeValue headerMap = i.get(COL_COMMIT_HEADERS); - if (headerMap != null) { - CommitHeaders.Builder headers = newCommitHeaders(); - headerMap.m().forEach((k, l) -> l.l().forEach(hv -> headers.add(k, hv.s()))); - tagHeaders = headers.build(); - } - - return tag( - id, - attributeToString(i, COL_TAG_MESSAGE), - tagHeaders, - attributeToBytes(i, COL_TAG_SIGNATURE)); - } - }); - - STORE_OBJ_TYPE.put( - ObjType.STRING, - new StoreObjDesc(COL_STRING) { - @Override - void toMap( - StringObj obj, - Map i, - int incrementalIndexSize, - int maxSerializedIndexSize) { - String s = obj.contentType(); - if (s != null && !s.isEmpty()) { - i.put(COL_STRING_CONTENT_TYPE, fromS(s)); - } - i.put(COL_STRING_COMPRESSION, fromS(obj.compression().name())); - s = obj.filename(); - if (s != null && !s.isEmpty()) { - i.put(COL_STRING_FILENAME, fromS(s)); - } - objIdsAttribute(i, COL_STRING_PREDECESSORS, obj.predecessors()); - bytesAttribute(i, COL_STRING_TEXT, obj.text()); - } - - @Override - StringObj fromMap(ObjId id, Map i) { - List predecessors = new ArrayList<>(); - attributeToObjIds(i, COL_STRING_PREDECESSORS, predecessors::add); - return stringData( - id, - attributeToString(i, COL_STRING_CONTENT_TYPE), - Compression.valueOf(attributeToString(i, COL_STRING_COMPRESSION)), - attributeToString(i, COL_STRING_FILENAME), - predecessors, - attributeToBytes(i, COL_STRING_TEXT)); - } - }); - } - - private static void fromStripesAttrList(AttributeValue attrList, Consumer consumer) { - if (attrList != null) { - for (AttributeValue seg : attrList.l()) { - Map m = seg.m(); - consumer.accept( - indexStripe( - keyFromString(attributeToString(m, COL_STRIPES_FIRST_KEY)), - keyFromString(attributeToString(m, COL_STRIPES_LAST_KEY)), - attributeToObjId(m, COL_STRIPES_SEGMENT))); - } - } - } - - private static AttributeValue stripesAttrList(List stripes) { - List stripeAttr = new ArrayList<>(); - for (IndexStripe stripe : stripes) { - Map sv = new HashMap<>(); - sv.put(COL_STRIPES_FIRST_KEY, fromS(stripe.firstKey().rawString())); - sv.put(COL_STRIPES_LAST_KEY, fromS(stripe.lastKey().rawString())); - objIdToAttribute(sv, COL_STRIPES_SEGMENT, stripe.segment()); - stripeAttr.add(fromM(sv)); - } - return fromL(stripeAttr); - } - private static boolean checkItemSizeExceeded(AwsErrorDetails errorDetails) { return "DynamoDb".equals(errorDetails.serviceName()) && "ValidationException".equals(errorDetails.errorCode()) @@ -964,10 +573,10 @@ private Map referenceAttributeValues( @Nonnull @jakarta.annotation.Nonnull Reference reference) { Map item = new HashMap<>(); item.put(KEY_NAME, referenceKey(reference.name())); - objIdToAttribute(item, COL_REFERENCES_POINTER, reference.pointer()); + DynamoDBSerde.objIdToAttribute(item, COL_REFERENCES_POINTER, reference.pointer()); item.put(COL_REFERENCES_DELETED, fromBool(reference.deleted())); item.put(COL_REFERENCES_CREATED_AT, referencesCreatedAt(reference)); - objIdToAttribute(item, COL_REFERENCES_EXTENDED_INFO, reference.extendedInfoObj()); + DynamoDBSerde.objIdToAttribute(item, COL_REFERENCES_EXTENDED_INFO, reference.extendedInfoObj()); byte[] previousPointers = serializePreviousPointers(reference.previousPointers()); if (previousPointers != null) { @@ -994,10 +603,10 @@ private void conditionalReferencePut( private static Map referenceConditionAttributes(Reference reference) { Map values = new HashMap<>(); - objIdToAttribute(values, ":pointer", reference.pointer()); + DynamoDBSerde.objIdToAttribute(values, ":pointer", reference.pointer()); values.put(":deleted", fromBool(reference.deleted())); values.put(":createdAt", referencesCreatedAt(reference)); - objIdToAttribute(values, ":extendedInfo", reference.extendedInfoObj()); + DynamoDBSerde.objIdToAttribute(values, ":extendedInfo", reference.extendedInfoObj()); return values; } @@ -1041,63 +650,6 @@ private Map objKeyMap(@Nonnull @jakarta.annotation.Nonnu return singletonMap(KEY_NAME, objKey(id)); } - private static void attributeToObjIds( - Map i, String n, Consumer receiver) { - AttributeValue v = i.get(n); - if (v != null) { - v.l().stream().map(el -> objIdFromByteBuffer(el.b().asByteBuffer())).forEach(receiver); - } - } - - private static String attributeToString(Map i, String n) { - AttributeValue v = i.get(n); - return v != null ? v.s() : null; - } - - private static ByteString attributeToBytes(Map i, String n) { - AttributeValue v = i.get(n); - return v != null ? unsafeWrap(v.b().asByteArrayUnsafe()) : null; - } - - private static boolean attributeToBool(Map i, String n) { - AttributeValue v = i.get(n); - if (v == null) { - return false; - } - Boolean b = v.bool(); - return b != null && b; - } - - private static ObjId attributeToObjId(Map i, String n) { - return attributeToObjId(i.get(n)); - } - - private static ObjId attributeToObjId(AttributeValue v) { - return v == null ? null : objIdFromByteBuffer(v.b().asByteBuffer()); - } - - private static void objIdToAttribute(Map i, String n, ObjId id) { - i.put(n, id != null ? fromB(fromByteBuffer(id.asByteBuffer())) : null); - } - - private static void objIdsAttribute(Map i, String n, List l) { - if (l == null || l.isEmpty()) { - return; - } - i.put( - n, - fromL( - l.stream() - .map(ObjId::asByteBuffer) - .map(SdkBytes::fromByteBuffer) - .map(AttributeValue::fromB) - .collect(Collectors.toList()))); - } - - private static void bytesAttribute(Map i, String n, ByteString b) { - i.put(n, fromB(fromByteBuffer(b.asReadOnlyByteBuffer()))); - } - private class ScanAllObjectsIterator extends AbstractIterator implements CloseableIterator { @@ -1135,7 +687,7 @@ protected Obj computeNext() { } Map item = pageIter.next(); - return decomposeObj(item); + return itemToObj(item); } } diff --git a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/DynamoDBSerde.java b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/DynamoDBSerde.java new file mode 100644 index 00000000000..f3fc42832f9 --- /dev/null +++ b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/DynamoDBSerde.java @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.dynamodb; + +import static org.projectnessie.nessie.relocated.protobuf.UnsafeByteOperations.unsafeWrap; +import static org.projectnessie.versioned.storage.common.indexes.StoreKey.keyFromString; +import static org.projectnessie.versioned.storage.common.objtypes.IndexStripe.indexStripe; +import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromByteBuffer; +import static software.amazon.awssdk.core.SdkBytes.fromByteBuffer; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromB; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromL; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromM; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromS; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.common.objtypes.IndexStripe; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +public final class DynamoDBSerde { + + private static final String COL_STRIPES_FIRST_KEY = "f"; + private static final String COL_STRIPES_LAST_KEY = "l"; + private static final String COL_STRIPES_SEGMENT = "s"; + + private DynamoDBSerde() {} + + public static void attributeToObjIds( + Map i, String n, Consumer receiver) { + AttributeValue v = i.get(n); + if (v != null) { + v.l().stream().map(el -> objIdFromByteBuffer(el.b().asByteBuffer())).forEach(receiver); + } + } + + public static String attributeToString(Map i, String n) { + AttributeValue v = i.get(n); + return v != null ? v.s() : null; + } + + public static ByteString attributeToBytes(Map i, String n) { + AttributeValue v = i.get(n); + return v != null ? unsafeWrap(v.b().asByteArrayUnsafe()) : null; + } + + public static boolean attributeToBool(Map i, String n) { + AttributeValue v = i.get(n); + if (v == null) { + return false; + } + Boolean b = v.bool(); + return b != null && b; + } + + public static ObjId attributeToObjId(Map i, String n) { + return attributeToObjId(i.get(n)); + } + + public static ObjId attributeToObjId(AttributeValue v) { + return v == null ? null : objIdFromByteBuffer(v.b().asByteBuffer()); + } + + public static void objIdToAttribute(Map i, String n, ObjId id) { + i.put(n, id != null ? fromB(fromByteBuffer(id.asByteBuffer())) : null); + } + + public static void objIdsAttribute(Map i, String n, List l) { + if (l == null || l.isEmpty()) { + return; + } + i.put( + n, + fromL( + l.stream() + .map(ObjId::asByteBuffer) + .map(SdkBytes::fromByteBuffer) + .map(AttributeValue::fromB) + .collect(Collectors.toList()))); + } + + public static void bytesAttribute(Map i, String n, ByteString b) { + i.put(n, fromB(fromByteBuffer(b.asReadOnlyByteBuffer()))); + } + + public static void fromStripesAttrList(AttributeValue attrList, Consumer consumer) { + if (attrList != null) { + for (AttributeValue seg : attrList.l()) { + Map m = seg.m(); + consumer.accept( + indexStripe( + keyFromString(DynamoDBSerde.attributeToString(m, COL_STRIPES_FIRST_KEY)), + keyFromString(DynamoDBSerde.attributeToString(m, COL_STRIPES_LAST_KEY)), + DynamoDBSerde.attributeToObjId(m, COL_STRIPES_SEGMENT))); + } + } + } + + public static AttributeValue stripesAttrList(List stripes) { + List stripeAttr = new ArrayList<>(); + for (IndexStripe stripe : stripes) { + Map sv = new HashMap<>(); + sv.put(COL_STRIPES_FIRST_KEY, fromS(stripe.firstKey().rawString())); + sv.put(COL_STRIPES_LAST_KEY, fromS(stripe.lastKey().rawString())); + DynamoDBSerde.objIdToAttribute(sv, COL_STRIPES_SEGMENT, stripe.segment()); + stripeAttr.add(fromM(sv)); + } + return fromL(stripeAttr); + } +} diff --git a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/CommitObjSerializer.java b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/CommitObjSerializer.java new file mode 100644 index 00000000000..b78435be75d --- /dev/null +++ b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/CommitObjSerializer.java @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.dynamodb.serializers; + +import static java.util.Objects.requireNonNull; +import static org.projectnessie.versioned.storage.common.objtypes.CommitHeaders.newCommitHeaders; +import static org.projectnessie.versioned.storage.common.objtypes.CommitObj.commitBuilder; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToBool; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToBytes; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToObjId; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToObjIds; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToString; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.bytesAttribute; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.fromStripesAttrList; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.objIdToAttribute; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.objIdsAttribute; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.stripesAttrList; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromBool; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromL; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromM; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromS; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.objtypes.CommitHeaders; +import org.projectnessie.versioned.storage.common.objtypes.CommitObj; +import org.projectnessie.versioned.storage.common.objtypes.CommitType; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +public class CommitObjSerializer implements ObjSerializer { + + public static final CommitObjSerializer INSTANCE = new CommitObjSerializer(); + + private static final String COL_COMMIT = "c"; + + private static final String COL_COMMIT_CREATED = "c"; + private static final String COL_COMMIT_SEQ = "q"; + private static final String COL_COMMIT_MESSAGE = "m"; + private static final String COL_COMMIT_HEADERS = "h"; + private static final String COL_COMMIT_REFERENCE_INDEX = "x"; + private static final String COL_COMMIT_REFERENCE_INDEX_STRIPES = "r"; + private static final String COL_COMMIT_TAIL = "t"; + private static final String COL_COMMIT_SECONDARY_PARENTS = "s"; + private static final String COL_COMMIT_INCREMENTAL_INDEX = "i"; + private static final String COL_COMMIT_INCOMPLETE_INDEX = "n"; + private static final String COL_COMMIT_TYPE = "y"; + + private CommitObjSerializer() {} + + @Override + public String attributeName() { + return COL_COMMIT; + } + + @Override + public void toMap( + CommitObj obj, + Map i, + int incrementalIndexSize, + int maxSerializedIndexSize) + throws ObjTooLargeException { + i.put(COL_COMMIT_SEQ, fromS(Long.toString(obj.seq()))); + i.put(COL_COMMIT_CREATED, fromS(Long.toString(obj.created()))); + ObjId referenceIndex = obj.referenceIndex(); + if (referenceIndex != null) { + objIdToAttribute(i, COL_COMMIT_REFERENCE_INDEX, referenceIndex); + } + i.put(COL_COMMIT_MESSAGE, fromS(obj.message())); + objIdsAttribute(i, COL_COMMIT_TAIL, obj.tail()); + objIdsAttribute(i, COL_COMMIT_SECONDARY_PARENTS, obj.secondaryParents()); + + ByteString index = obj.incrementalIndex(); + if (index.size() > incrementalIndexSize) { + throw new ObjTooLargeException(index.size(), incrementalIndexSize); + } + bytesAttribute(i, COL_COMMIT_INCREMENTAL_INDEX, index); + + if (!obj.referenceIndexStripes().isEmpty()) { + i.put(COL_COMMIT_REFERENCE_INDEX_STRIPES, stripesAttrList(obj.referenceIndexStripes())); + } + + Map headerMap = new HashMap<>(); + CommitHeaders headers = obj.headers(); + for (String s : headers.keySet()) { + headerMap.put( + s, + fromL( + headers.getAll(s).stream().map(AttributeValue::fromS).collect(Collectors.toList()))); + } + if (!headerMap.isEmpty()) { + i.put(COL_COMMIT_HEADERS, fromM(headerMap)); + } + i.put(COL_COMMIT_INCOMPLETE_INDEX, fromBool(obj.incompleteIndex())); + i.put(COL_COMMIT_TYPE, fromS(obj.commitType().shortName())); + } + + @Override + public CommitObj fromMap(ObjId id, Map i) { + CommitObj.Builder b = + commitBuilder() + .id(id) + .seq(Long.parseLong(requireNonNull(attributeToString(i, COL_COMMIT_SEQ)))) + .created(Long.parseLong(requireNonNull(attributeToString(i, COL_COMMIT_CREATED)))) + .message(attributeToString(i, COL_COMMIT_MESSAGE)) + .incrementalIndex(attributeToBytes(i, COL_COMMIT_INCREMENTAL_INDEX)) + .incompleteIndex(attributeToBool(i, COL_COMMIT_INCOMPLETE_INDEX)) + .commitType(CommitType.fromShortName(attributeToString(i, COL_COMMIT_TYPE))); + AttributeValue v = i.get(COL_COMMIT_REFERENCE_INDEX); + if (v != null) { + b.referenceIndex(attributeToObjId(v)); + } + + fromStripesAttrList(i.get(COL_COMMIT_REFERENCE_INDEX_STRIPES), b::addReferenceIndexStripes); + + attributeToObjIds(i, COL_COMMIT_TAIL, b::addTail); + attributeToObjIds(i, COL_COMMIT_SECONDARY_PARENTS, b::addSecondaryParents); + + CommitHeaders.Builder headers = newCommitHeaders(); + AttributeValue headerMap = i.get(COL_COMMIT_HEADERS); + if (headerMap != null) { + headerMap.m().forEach((k, l) -> l.l().forEach(hv -> headers.add(k, hv.s()))); + } + b.headers(headers.build()); + + return b.build(); + } +} diff --git a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/ContentValueObjSerializer.java b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/ContentValueObjSerializer.java new file mode 100644 index 00000000000..bf0067910b0 --- /dev/null +++ b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/ContentValueObjSerializer.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.dynamodb.serializers; + +import static java.util.Objects.requireNonNull; +import static org.projectnessie.versioned.storage.common.objtypes.ContentValueObj.contentValue; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToBytes; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToString; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.bytesAttribute; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromS; + +import java.util.Map; +import org.projectnessie.versioned.storage.common.objtypes.ContentValueObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +public class ContentValueObjSerializer implements ObjSerializer { + + public static final ContentValueObjSerializer INSTANCE = new ContentValueObjSerializer(); + + private static final String COL_VALUE = "v"; + + private static final String COL_VALUE_CONTENT_ID = "i"; + private static final String COL_VALUE_PAYLOAD = "p"; + private static final String COL_VALUE_DATA = "d"; + + private ContentValueObjSerializer() {} + + @Override + public String attributeName() { + return COL_VALUE; + } + + @Override + public void toMap( + ContentValueObj obj, + Map i, + int incrementalIndexSize, + int maxSerializedIndexSize) { + i.put(COL_VALUE_CONTENT_ID, fromS(obj.contentId())); + i.put(COL_VALUE_PAYLOAD, fromS(Integer.toString(obj.payload()))); + bytesAttribute(i, COL_VALUE_DATA, obj.data()); + } + + @Override + public ContentValueObj fromMap(ObjId id, Map i) { + return contentValue( + id, + attributeToString(i, COL_VALUE_CONTENT_ID), + Integer.parseInt(requireNonNull(attributeToString(i, COL_VALUE_PAYLOAD))), + attributeToBytes(i, COL_VALUE_DATA)); + } +} diff --git a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/CustomObjSerializer.java b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/CustomObjSerializer.java new file mode 100644 index 00000000000..bf5d72e426e --- /dev/null +++ b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/CustomObjSerializer.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.dynamodb.serializers; + +import static java.util.Objects.requireNonNull; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToBytes; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToString; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.bytesAttribute; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromS; + +import java.nio.ByteBuffer; +import java.util.Map; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.serialize.SmileSerialization; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +public class CustomObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new CustomObjSerializer(); + + private static final String COL_CUSTOM = "x"; + + private static final String COL_CUSTOM_CLASS = "x_class"; + private static final String COL_CUSTOM_DATA = "x_data"; + + private CustomObjSerializer() {} + + @Override + public String attributeName() { + return COL_CUSTOM; + } + + @Override + public void toMap( + Obj obj, Map i, int incrementalIndexSize, int maxSerializedIndexSize) + throws ObjTooLargeException { + i.put(COL_CUSTOM_CLASS, fromS(obj.type().targetClass().getName())); + bytesAttribute(i, COL_CUSTOM_DATA, ByteString.copyFrom(SmileSerialization.serializeObj(obj))); + } + + @Override + public Obj fromMap(ObjId id, Map i) { + ByteBuffer buffer = requireNonNull(attributeToBytes(i, COL_CUSTOM_DATA)).asReadOnlyByteBuffer(); + return SmileSerialization.deserializeObj(id, buffer, attributeToString(i, COL_CUSTOM_CLASS)); + } +} diff --git a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/IndexObjSerializer.java b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/IndexObjSerializer.java new file mode 100644 index 00000000000..0a90e613cd9 --- /dev/null +++ b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/IndexObjSerializer.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.dynamodb.serializers; + +import static java.util.Objects.requireNonNull; +import static org.projectnessie.versioned.storage.common.objtypes.IndexObj.index; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToBytes; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.bytesAttribute; + +import java.util.Map; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.objtypes.IndexObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +public class IndexObjSerializer implements ObjSerializer { + + public static final IndexObjSerializer INSTANCE = new IndexObjSerializer(); + + private static final String COL_INDEX = "i"; + private static final String COL_INDEX_INDEX = "i"; + + private IndexObjSerializer() {} + + @Override + public String attributeName() { + return COL_INDEX; + } + + @Override + public void toMap( + IndexObj obj, + Map i, + int incrementalIndexSize, + int maxSerializedIndexSize) + throws ObjTooLargeException { + ByteString index = obj.index(); + if (index.size() > maxSerializedIndexSize) { + throw new ObjTooLargeException(index.size(), maxSerializedIndexSize); + } + bytesAttribute(i, COL_INDEX_INDEX, index); + } + + @Override + public IndexObj fromMap(ObjId id, Map i) { + return index(id, requireNonNull(attributeToBytes(i, COL_INDEX_INDEX))); + } +} diff --git a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/IndexSegmentsObjSerializer.java b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/IndexSegmentsObjSerializer.java new file mode 100644 index 00000000000..c6cd99ac06f --- /dev/null +++ b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/IndexSegmentsObjSerializer.java @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.dynamodb.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj.indexSegments; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.fromStripesAttrList; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.stripesAttrList; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj; +import org.projectnessie.versioned.storage.common.objtypes.IndexStripe; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +public class IndexSegmentsObjSerializer implements ObjSerializer { + + public static final IndexSegmentsObjSerializer INSTANCE = new IndexSegmentsObjSerializer(); + + private static final String COL_SEGMENTS = "I"; + private static final String COL_SEGMENTS_STRIPES = "s"; + + private IndexSegmentsObjSerializer() {} + + @Override + public String attributeName() { + return COL_SEGMENTS; + } + + @Override + public void toMap( + IndexSegmentsObj obj, + Map i, + int incrementalIndexSize, + int maxSerializedIndexSize) { + i.put(COL_SEGMENTS_STRIPES, stripesAttrList(obj.stripes())); + } + + @Override + public IndexSegmentsObj fromMap(ObjId id, Map i) { + List stripes = new ArrayList<>(); + fromStripesAttrList(i.get(COL_SEGMENTS_STRIPES), stripes::add); + return indexSegments(id, stripes); + } +} diff --git a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/ObjSerializer.java b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/ObjSerializer.java new file mode 100644 index 00000000000..fe0400cf56f --- /dev/null +++ b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/ObjSerializer.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.dynamodb.serializers; + +import java.util.Map; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.common.persist.ObjType; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +public interface ObjSerializer { + + /** + * The name of the document's top-level attribute that will contain the serialized object. Note + * that this is generally equal to {@link ObjType#shortName()}, but not always, for historical + * reasons. + */ + String attributeName(); + + void toMap( + O obj, Map i, int incrementalIndexSize, int maxSerializedIndexSize) + throws ObjTooLargeException; + + O fromMap(ObjId id, Map i); +} diff --git a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/ObjSerializers.java b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/ObjSerializers.java new file mode 100644 index 00000000000..4cabd428a29 --- /dev/null +++ b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/ObjSerializers.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.dynamodb.serializers; + +import static java.util.Objects.requireNonNull; + +import java.util.HashSet; +import java.util.Set; +import javax.annotation.Nonnull; +import org.projectnessie.versioned.storage.common.objtypes.StandardObjType; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjType; + +public final class ObjSerializers { + + public static final Set> ALL_SERIALIZERS = + Set.of( + CommitObjSerializer.INSTANCE, + ContentValueObjSerializer.INSTANCE, + IndexSegmentsObjSerializer.INSTANCE, + IndexObjSerializer.INSTANCE, + RefObjSerializer.INSTANCE, + StringObjSerializer.INSTANCE, + TagObjSerializer.INSTANCE, + CustomObjSerializer.INSTANCE); + + static { + Set attributeNames = new HashSet<>(); + ALL_SERIALIZERS.forEach( + serializer -> { + String fieldName = requireNonNull(serializer.attributeName()); + if (!attributeNames.add(fieldName)) { + throw new IllegalStateException("Duplicate attribute name: " + fieldName); + } + }); + } + + @Nonnull + @jakarta.annotation.Nonnull + public static ObjSerializer forType(@Nonnull @jakarta.annotation.Nonnull ObjType type) { + ObjSerializer serializer = CustomObjSerializer.INSTANCE; + if (type instanceof StandardObjType) { + switch ((StandardObjType) type) { + case COMMIT: + serializer = CommitObjSerializer.INSTANCE; + break; + case INDEX_SEGMENTS: + serializer = IndexSegmentsObjSerializer.INSTANCE; + break; + case INDEX: + serializer = IndexObjSerializer.INSTANCE; + break; + case REF: + serializer = RefObjSerializer.INSTANCE; + break; + case STRING: + serializer = StringObjSerializer.INSTANCE; + break; + case TAG: + serializer = TagObjSerializer.INSTANCE; + break; + case VALUE: + serializer = ContentValueObjSerializer.INSTANCE; + break; + default: + throw new IllegalArgumentException("Unknown standard object type: " + type); + } + } + @SuppressWarnings("unchecked") + ObjSerializer cast = (ObjSerializer) serializer; + return cast; + } +} diff --git a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/RefObjSerializer.java b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/RefObjSerializer.java new file mode 100644 index 00000000000..4c9c617e122 --- /dev/null +++ b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/RefObjSerializer.java @@ -0,0 +1,73 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.dynamodb.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.RefObj.ref; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToObjId; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToString; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.objIdToAttribute; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromS; + +import java.util.Map; +import org.projectnessie.versioned.storage.common.objtypes.RefObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +public class RefObjSerializer implements ObjSerializer { + + public static final RefObjSerializer INSTANCE = new RefObjSerializer(); + + private static final String COL_REF = "e"; + + private static final String COL_REF_NAME = "n"; + private static final String COL_REF_INITIAL_POINTER = "p"; + private static final String COL_REF_CREATED_AT = "c"; + private static final String COL_REF_EXTENDED_INFO = "e"; + + private RefObjSerializer() {} + + @Override + public String attributeName() { + return COL_REF; + } + + @Override + public void toMap( + RefObj obj, + Map i, + int incrementalIndexSize, + int maxSerializedIndexSize) { + i.put(COL_REF_NAME, fromS(obj.name())); + i.put(COL_REF_CREATED_AT, fromS(Long.toString(obj.createdAtMicros()))); + objIdToAttribute(i, COL_REF_INITIAL_POINTER, obj.initialPointer()); + ObjId extendedInfoObj = obj.extendedInfoObj(); + if (extendedInfoObj != null) { + objIdToAttribute(i, COL_REF_EXTENDED_INFO, extendedInfoObj); + } + } + + @Override + public RefObj fromMap(ObjId id, Map i) { + String createdAtStr = attributeToString(i, COL_REF_CREATED_AT); + long createdAt = createdAtStr != null ? Long.parseLong(createdAtStr) : 0L; + return ref( + id, + attributeToString(i, COL_REF_NAME), + attributeToObjId(i, COL_REF_INITIAL_POINTER), + createdAt, + attributeToObjId(i, COL_REF_EXTENDED_INFO)); + } +} diff --git a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/StringObjSerializer.java b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/StringObjSerializer.java new file mode 100644 index 00000000000..861456865db --- /dev/null +++ b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/StringObjSerializer.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.dynamodb.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.StringObj.stringData; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToBytes; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToObjIds; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToString; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.bytesAttribute; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.objIdsAttribute; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromS; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.projectnessie.versioned.storage.common.objtypes.Compression; +import org.projectnessie.versioned.storage.common.objtypes.StringObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +public class StringObjSerializer implements ObjSerializer { + + public static final StringObjSerializer INSTANCE = new StringObjSerializer(); + + private static final String COL_STRING = "s"; + + private static final String COL_STRING_CONTENT_TYPE = "y"; + private static final String COL_STRING_COMPRESSION = "c"; + private static final String COL_STRING_FILENAME = "f"; + private static final String COL_STRING_PREDECESSORS = "p"; + private static final String COL_STRING_TEXT = "t"; + + private StringObjSerializer() {} + + @Override + public String attributeName() { + return COL_STRING; + } + + @Override + public void toMap( + StringObj obj, + Map i, + int incrementalIndexSize, + int maxSerializedIndexSize) { + String s = obj.contentType(); + if (s != null && !s.isEmpty()) { + i.put(COL_STRING_CONTENT_TYPE, fromS(s)); + } + i.put(COL_STRING_COMPRESSION, fromS(obj.compression().name())); + s = obj.filename(); + if (s != null && !s.isEmpty()) { + i.put(COL_STRING_FILENAME, fromS(s)); + } + objIdsAttribute(i, COL_STRING_PREDECESSORS, obj.predecessors()); + bytesAttribute(i, COL_STRING_TEXT, obj.text()); + } + + @Override + public StringObj fromMap(ObjId id, Map i) { + List predecessors = new ArrayList<>(); + attributeToObjIds(i, COL_STRING_PREDECESSORS, predecessors::add); + return stringData( + id, + attributeToString(i, COL_STRING_CONTENT_TYPE), + Compression.valueOf(attributeToString(i, COL_STRING_COMPRESSION)), + attributeToString(i, COL_STRING_FILENAME), + predecessors, + attributeToBytes(i, COL_STRING_TEXT)); + } +} diff --git a/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/TagObjSerializer.java b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/TagObjSerializer.java new file mode 100644 index 00000000000..9f8dc2274b1 --- /dev/null +++ b/versioned/storage/dynamodb/src/main/java/org/projectnessie/versioned/storage/dynamodb/serializers/TagObjSerializer.java @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.dynamodb.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.CommitHeaders.newCommitHeaders; +import static org.projectnessie.versioned.storage.common.objtypes.TagObj.tag; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToBytes; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.attributeToString; +import static org.projectnessie.versioned.storage.dynamodb.DynamoDBSerde.bytesAttribute; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromL; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromM; +import static software.amazon.awssdk.services.dynamodb.model.AttributeValue.fromS; + +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.common.objtypes.CommitHeaders; +import org.projectnessie.versioned.storage.common.objtypes.TagObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +public class TagObjSerializer implements ObjSerializer { + + public static final TagObjSerializer INSTANCE = new TagObjSerializer(); + + private static final String COL_TAG = "t"; + + private static final String COL_TAG_MESSAGE = "m"; + private static final String COL_TAG_HEADERS = "h"; + private static final String COL_TAG_SIGNATURE = "s"; + + private TagObjSerializer() {} + + @Override + public String attributeName() { + return COL_TAG; + } + + @Override + public void toMap( + TagObj obj, + Map i, + int incrementalIndexSize, + int maxSerializedIndexSize) { + String message = obj.message(); + if (message != null) { + i.put(COL_TAG_MESSAGE, fromS(message)); + } + + Map headerMap = new HashMap<>(); + CommitHeaders headers = obj.headers(); + if (headers != null) { + for (String s : headers.keySet()) { + headerMap.put( + s, + fromL( + headers.getAll(s).stream() + .map(AttributeValue::fromS) + .collect(Collectors.toList()))); + } + if (!headerMap.isEmpty()) { + i.put(COL_TAG_HEADERS, fromM(headerMap)); + } + } + + ByteString signature = obj.signature(); + if (signature != null) { + bytesAttribute(i, COL_TAG_SIGNATURE, signature); + } + } + + @Override + public TagObj fromMap(ObjId id, Map i) { + CommitHeaders tagHeaders = null; + AttributeValue headerMap = i.get(COL_TAG_HEADERS); + if (headerMap != null) { + CommitHeaders.Builder headers = newCommitHeaders(); + headerMap.m().forEach((k, l) -> l.l().forEach(hv -> headers.add(k, hv.s()))); + tagHeaders = headers.build(); + } + + return tag( + id, + attributeToString(i, COL_TAG_MESSAGE), + tagHeaders, + attributeToBytes(i, COL_TAG_SIGNATURE)); + } +} diff --git a/versioned/storage/inmemory/src/main/java/org/projectnessie/versioned/storage/inmemory/InmemoryPersist.java b/versioned/storage/inmemory/src/main/java/org/projectnessie/versioned/storage/inmemory/InmemoryPersist.java index 299f1fbc237..c5b25c3ec9e 100644 --- a/versioned/storage/inmemory/src/main/java/org/projectnessie/versioned/storage/inmemory/InmemoryPersist.java +++ b/versioned/storage/inmemory/src/main/java/org/projectnessie/versioned/storage/inmemory/InmemoryPersist.java @@ -206,7 +206,7 @@ public T fetchTypedObj( @Nonnull @jakarta.annotation.Nonnull ObjId id, ObjType type, Class typeClass) throws ObjNotFoundException { Obj obj = inmemory.objects.get(compositeKey(id)); - if (obj == null || obj.type() != type) { + if (obj == null || !obj.type().equals(type)) { throw new ObjNotFoundException(id); } @SuppressWarnings("unchecked") diff --git a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/AbstractJdbcPersist.java b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/AbstractJdbcPersist.java index c1eadd33588..4fbb1eb9e7f 100644 --- a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/AbstractJdbcPersist.java +++ b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/AbstractJdbcPersist.java @@ -16,52 +16,17 @@ package org.projectnessie.versioned.storage.jdbc; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; import static java.util.Arrays.stream; -import static java.util.Collections.emptyList; -import static java.util.Objects.requireNonNull; -import static org.projectnessie.nessie.relocated.protobuf.UnsafeByteOperations.unsafeWrap; -import static org.projectnessie.versioned.storage.common.indexes.StoreKey.keyFromString; -import static org.projectnessie.versioned.storage.common.objtypes.ContentValueObj.contentValue; -import static org.projectnessie.versioned.storage.common.objtypes.IndexObj.index; -import static org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj.indexSegments; -import static org.projectnessie.versioned.storage.common.objtypes.IndexStripe.indexStripe; -import static org.projectnessie.versioned.storage.common.objtypes.RefObj.ref; -import static org.projectnessie.versioned.storage.common.objtypes.StringObj.stringData; -import static org.projectnessie.versioned.storage.common.objtypes.TagObj.tag; -import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromByteBuffer; -import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromString; +import static java.util.stream.Collectors.joining; import static org.projectnessie.versioned.storage.common.util.Closing.closeMultiple; import static org.projectnessie.versioned.storage.jdbc.JdbcBackend.unhandledSQLException; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.deserializeObjId; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.serializeObjId; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.ADD_REFERENCE; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_COMMIT_CREATED; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_COMMIT_HEADERS; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_COMMIT_INCOMPLETE_INDEX; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_COMMIT_INCREMENTAL_INDEX; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_COMMIT_MESSAGE; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_COMMIT_REFERENCE_INDEX; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_COMMIT_REFERENCE_INDEX_STRIPES; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_COMMIT_SECONDARY_PARENTS; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_COMMIT_SEQ; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_COMMIT_TAIL; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_COMMIT_TYPE; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_INDEX_INDEX; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_REF_CREATED_AT; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_REF_EXTENDED_INFO; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_REF_INITIAL_POINTER; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_REF_NAME; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_SEGMENTS_STRIPES; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_STRING_COMPRESSION; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_STRING_CONTENT_TYPE; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_STRING_FILENAME; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_STRING_PREDECESSORS; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_STRING_TEXT; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_TAG_HEADERS; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_TAG_MESSAGE; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_TAG_SIGNATURE; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_VALUE_CONTENT_ID; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_VALUE_DATA; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_VALUE_PAYLOAD; +import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COLS_OBJS_ALL; +import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_OBJ_ID; +import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_OBJ_TYPE; +import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_REPO_ID; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.DELETE_OBJ; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.FETCH_OBJ_TYPE; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.FIND_OBJS; @@ -73,70 +38,78 @@ import static org.projectnessie.versioned.storage.jdbc.SqlConstants.REFS_CREATED_AT_COND; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.REFS_EXTENDED_INFO_COND; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.SCAN_OBJS; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.STORE_OBJ; +import static org.projectnessie.versioned.storage.jdbc.SqlConstants.TABLE_OBJS; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.UPDATE_REFERENCE_POINTER; -import static org.projectnessie.versioned.storage.serialize.ProtoSerialization.deserializePreviousPointers; import static org.projectnessie.versioned.storage.serialize.ProtoSerialization.serializePreviousPointers; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.AbstractIterator; -import java.io.IOException; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Types; import java.util.ArrayList; -import java.util.EnumMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.function.Consumer; -import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.agrona.collections.Hashing; import org.agrona.collections.Int2IntHashMap; import org.agrona.collections.Object2IntHashMap; -import org.projectnessie.nessie.relocated.protobuf.ByteString; import org.projectnessie.versioned.storage.common.config.StoreConfig; import org.projectnessie.versioned.storage.common.exceptions.ObjNotFoundException; import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; import org.projectnessie.versioned.storage.common.exceptions.RefAlreadyExistsException; import org.projectnessie.versioned.storage.common.exceptions.RefConditionFailedException; import org.projectnessie.versioned.storage.common.exceptions.RefNotFoundException; -import org.projectnessie.versioned.storage.common.objtypes.CommitHeaders; -import org.projectnessie.versioned.storage.common.objtypes.CommitObj; -import org.projectnessie.versioned.storage.common.objtypes.CommitType; -import org.projectnessie.versioned.storage.common.objtypes.Compression; -import org.projectnessie.versioned.storage.common.objtypes.ContentValueObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexStripe; -import org.projectnessie.versioned.storage.common.objtypes.RefObj; -import org.projectnessie.versioned.storage.common.objtypes.StringObj; -import org.projectnessie.versioned.storage.common.objtypes.TagObj; import org.projectnessie.versioned.storage.common.persist.CloseableIterator; import org.projectnessie.versioned.storage.common.persist.Obj; import org.projectnessie.versioned.storage.common.persist.ObjId; import org.projectnessie.versioned.storage.common.persist.ObjType; +import org.projectnessie.versioned.storage.common.persist.ObjTypes; import org.projectnessie.versioned.storage.common.persist.Persist; import org.projectnessie.versioned.storage.common.persist.Reference; -import org.projectnessie.versioned.storage.common.proto.StorageTypes.HeaderEntry; -import org.projectnessie.versioned.storage.common.proto.StorageTypes.Headers; -import org.projectnessie.versioned.storage.common.proto.StorageTypes.Stripe; -import org.projectnessie.versioned.storage.common.proto.StorageTypes.Stripes; +import org.projectnessie.versioned.storage.jdbc.serializers.ObjSerializer; +import org.projectnessie.versioned.storage.jdbc.serializers.ObjSerializers; @SuppressWarnings({"SqlDialectInspection", "SqlNoDataSourceInspection"}) abstract class AbstractJdbcPersist implements Persist { private final StoreConfig config; private final DatabaseSpecific databaseSpecific; + private final String storeObjSql; + private final Map storeObjSqlParams; AbstractJdbcPersist(DatabaseSpecific databaseSpecific, StoreConfig config) { this.config = config; this.databaseSpecific = databaseSpecific; + this.storeObjSqlParams = buildStoreObjSqlParams(); + this.storeObjSql = buildStoreObjSql(); + } + + private Map buildStoreObjSqlParams() { + Builder params = ImmutableMap.builder(); + int i = 1; + params.put(COL_REPO_ID, i++); + for (String col : COLS_OBJS_ALL.keySet()) { + params.put(col, i++); + } + return params.build(); + } + + private String buildStoreObjSql() { + return "INSERT INTO " + + TABLE_OBJS + + " (" + + String.join(", ", storeObjSqlParams.keySet()) + + ") VALUES (" + + storeObjSqlParams.keySet().stream().map(c -> "?").collect(joining(", ")) + + ")"; } @Nonnull @@ -189,7 +162,7 @@ protected final Reference[] findReferences( } try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { - Reference ref = deserializeReference(rs); + Reference ref = JdbcSerde.deserializeReference(rs); int i = nameToIndex.getValue(ref.name()); if (i != -1) { r[i] = ref; @@ -214,14 +187,14 @@ protected final Reference addReference( try (PreparedStatement ps = conn.prepareStatement(sql)) { ps.setString(1, config.repositoryId()); ps.setString(2, reference.name()); - serializeObjId(ps, 3, reference.pointer()); + serializeObjId(ps, 3, reference.pointer(), databaseSpecific); ps.setBoolean(4, reference.deleted()); if (reference.createdAtMicros() != 0L) { ps.setLong(5, reference.createdAtMicros()); } else { ps.setNull(5, Types.BIGINT); } - serializeObjId(ps, 6, reference.extendedInfoObj()); + serializeObjId(ps, 6, reference.extendedInfoObj(), databaseSpecific); byte[] previous = serializePreviousPointers(reference.previousPointers()); if (previous != null) { ps.setBytes(7, previous); @@ -254,7 +227,7 @@ protected final Reference markReferenceAsDeleted( ps.setBoolean(idx++, true); ps.setString(idx++, config().repositoryId()); ps.setString(idx++, reference.name()); - serializeObjId(ps, idx++, reference.pointer()); + serializeObjId(ps, idx++, reference.pointer(), databaseSpecific); ps.setBoolean(idx++, false); long createdAtMicros = reference.createdAtMicros(); if (createdAtMicros != 0L) { @@ -262,7 +235,7 @@ protected final Reference markReferenceAsDeleted( } ObjId extendedInfoObj = reference.extendedInfoObj(); if (extendedInfoObj != null) { - serializeObjId(ps, idx, extendedInfoObj); + serializeObjId(ps, idx, extendedInfoObj, databaseSpecific); } if (ps.executeUpdate() != 1) { @@ -287,7 +260,7 @@ protected final void purgeReference( int idx = 1; ps.setString(idx++, config().repositoryId()); ps.setString(idx++, reference.name()); - serializeObjId(ps, idx++, reference.pointer()); + serializeObjId(ps, idx++, reference.pointer(), databaseSpecific); ps.setBoolean(idx++, true); long createdAtMicros = reference.createdAtMicros(); if (createdAtMicros != 0L) { @@ -295,7 +268,7 @@ protected final void purgeReference( } ObjId extendedInfoObj = reference.extendedInfoObj(); if (extendedInfoObj != null) { - serializeObjId(ps, idx, extendedInfoObj); + serializeObjId(ps, idx, extendedInfoObj, databaseSpecific); } if (ps.executeUpdate() != 1) { @@ -320,7 +293,7 @@ protected final Reference updateReferencePointer( try (PreparedStatement ps = conn.prepareStatement(referencesDml(UPDATE_REFERENCE_POINTER, reference))) { int idx = 1; - serializeObjId(ps, idx++, newPointer); + serializeObjId(ps, idx++, newPointer, databaseSpecific); Reference updated = reference.forNewPointer(newPointer, config); byte[] previous = serializePreviousPointers(updated.previousPointers()); if (previous != null) { @@ -331,7 +304,7 @@ protected final Reference updateReferencePointer( ps.setString(idx++, config().repositoryId()); ps.setString(idx++, reference.name()); - serializeObjId(ps, idx++, reference.pointer()); + serializeObjId(ps, idx++, reference.pointer(), databaseSpecific); ps.setBoolean(idx++, false); long createdAtMicros = reference.createdAtMicros(); if (createdAtMicros != 0L) { @@ -339,7 +312,7 @@ protected final Reference updateReferencePointer( } ObjId extendedInfoObj = reference.extendedInfoObj(); if (extendedInfoObj != null) { - serializeObjId(ps, idx, extendedInfoObj); + serializeObjId(ps, idx, extendedInfoObj, databaseSpecific); } if (ps.executeUpdate() != 1) { @@ -363,9 +336,9 @@ private String referencesDml(String sql, Reference reference) { .replace(REFS_EXTENDED_INFO_COND, extendedInfoCond); } - @SuppressWarnings("unused") protected T fetchTypedObj( - Connection conn, ObjId id, ObjType type, Class typeClass) throws ObjNotFoundException { + Connection conn, ObjId id, ObjType type, @SuppressWarnings("unused") Class typeClass) + throws ObjNotFoundException { Obj obj = fetchObjs(conn, new ObjId[] {id}, type)[0]; @SuppressWarnings("unchecked") @@ -386,11 +359,11 @@ protected ObjType fetchObjType( throws ObjNotFoundException { try (PreparedStatement ps = conn.prepareStatement(sqlSelectMultiple(FETCH_OBJ_TYPE, 1))) { ps.setString(1, config.repositoryId()); - serializeObjId(ps, 2, id); + serializeObjId(ps, 2, id, databaseSpecific); try (ResultSet rs = ps.executeQuery()) { if (rs.next()) { String objType = rs.getString(1); - return ObjType.valueOf(objType); + return ObjTypes.forName(objType); } } throw new ObjNotFoundException(id); @@ -438,7 +411,7 @@ protected final Obj[] fetchObjs( int idx = 1; ps.setString(idx++, config.repositoryId()); for (ObjId key : keys) { - serializeObjId(ps, idx++, key); + serializeObjId(ps, idx++, key, databaseSpecific); } if (type != null) { ps.setString(idx, type.name()); @@ -475,14 +448,11 @@ protected final Obj[] fetchObjs( } private Obj deserializeObj(ResultSet rs) throws SQLException { - ObjId id = deserializeObjId(rs, 1); - String objType = rs.getString(2); - ObjType type = ObjType.valueOf(objType); - - @SuppressWarnings("rawtypes") - StoreObjDesc objDesc = STORE_OBJ_TYPE.get(type); - checkState(objDesc != null, "Cannot deserialize object type %s", objType); - return objDesc.deserialize(rs, id); + ObjId id = deserializeObjId(rs, COL_OBJ_ID); + String objType = rs.getString(COL_OBJ_TYPE); + ObjType type = ObjTypes.forName(objType); + ObjSerializer serializer = ObjSerializers.forType(type); + return serializer.deserialize(rs, id); } protected final boolean storeObj( @@ -518,7 +488,6 @@ protected final Void updateObjs( return null; } - @SuppressWarnings("unchecked") @Nonnull @jakarta.annotation.Nonnull private boolean[] upsertObjs( @@ -535,7 +504,7 @@ private boolean[] upsertObjs( conn, stream(objs).map(obj -> obj == null ? null : obj.id()).toArray(ObjId[]::new)); } - try (PreparedStatement ps = conn.prepareStatement(databaseSpecific.wrapInsert(STORE_OBJ))) { + try (PreparedStatement ps = conn.prepareStatement(databaseSpecific.wrapInsert(storeObjSql))) { boolean[] r = new boolean[objs.length]; Int2IntHashMap batchIndexToObjIndex = @@ -567,22 +536,22 @@ private boolean[] upsertObjs( checkArgument(id != null, "Obj to store must have a non-null ID"); - checkArgument(STORE_OBJ_TYPE.containsKey(type), "Cannot serialize object type %s ", type); - - // - - int idx = 1; - ps.setString(idx++, config.repositoryId()); - serializeObjId(ps, idx++, id); - ps.setString(idx++, type.name()); - - for (Entry> e : STORE_OBJ_TYPE.entrySet()) { - if (e.getKey() == type) { - @SuppressWarnings("rawtypes") - StoreObjDesc storeType = e.getValue(); - idx = storeType.store(ps, idx, obj, incrementalIndexSizeLimit, indexSizeLimit); - } else { - idx = e.getValue().storeNone(ps, idx); + ps.setString(storeObjSqlParams.get(COL_REPO_ID), config.repositoryId()); + serializeObjId(ps, storeObjSqlParams.get(COL_OBJ_ID), id, databaseSpecific); + ps.setString(storeObjSqlParams.get(COL_OBJ_TYPE), type.name()); + + ObjSerializer serializer = ObjSerializers.forType(type); + serializer.serialize( + ps, + obj, + incrementalIndexSizeLimit, + indexSizeLimit, + storeObjSqlParams::get, + databaseSpecific); + + for (ObjSerializer other : ObjSerializers.ALL_SERIALIZERS) { + if (serializer != other) { + other.setNull(ps, storeObjSqlParams::get, databaseSpecific); } } @@ -616,7 +585,7 @@ protected final void deleteObj( @Nonnull @jakarta.annotation.Nonnull ObjId id) { try (PreparedStatement ps = conn.prepareStatement(DELETE_OBJ)) { ps.setString(1, config.repositoryId()); - serializeObjId(ps, 2, id); + serializeObjId(ps, 2, id, databaseSpecific); ps.executeUpdate(); } catch (SQLException e) { @@ -639,7 +608,7 @@ protected final void deleteObjs( continue; } ps.setString(1, config.repositoryId()); - serializeObjId(ps, 2, id); + serializeObjId(ps, 2, id, databaseSpecific); ps.addBatch(); if (++batchSize == MAX_BATCH_SIZE) { @@ -665,467 +634,6 @@ protected CloseableIterator scanAllObjects(Connection conn, Set re return new ScanAllObjectsIterator(conn, returnedObjTypes); } - private abstract static class StoreObjDesc { - - abstract O deserialize(ResultSet rs, ObjId id) throws SQLException; - - abstract int storeNone(PreparedStatement ps, int idx) throws SQLException; - - abstract int store( - PreparedStatement ps, int idx, O obj, int incrementalIndexLimit, int maxSerializedIndexSize) - throws SQLException, ObjTooLargeException; - } - - private static final Map> STORE_OBJ_TYPE = new EnumMap<>(ObjType.class); - - static { - STORE_OBJ_TYPE.put( - ObjType.COMMIT, - new StoreObjDesc() { - @Override - int storeNone(PreparedStatement ps, int idx) throws SQLException { - ps.setNull(idx++, Types.BIGINT); - ps.setNull(idx++, Types.BIGINT); - ps.setNull(idx++, Types.VARCHAR); - ps.setNull(idx++, Types.BINARY); - ps.setNull(idx++, Types.VARCHAR); - ps.setNull(idx++, Types.BINARY); - - ps.setNull(idx++, Types.VARCHAR); - ps.setNull(idx++, Types.VARCHAR); - - ps.setNull(idx++, Types.BINARY); - - ps.setNull(idx++, Types.BOOLEAN); - ps.setNull(idx++, Types.VARCHAR); - return idx; - } - - @Override - int store( - PreparedStatement ps, - int idx, - CommitObj obj, - int incrementalIndexLimit, - int maxSerializedIndexSize) - throws SQLException, ObjTooLargeException { - ps.setLong(idx++, obj.created()); - ps.setLong(idx++, obj.seq()); - ps.setString(idx++, obj.message()); - - obj.headers(); - Headers.Builder hb = Headers.newBuilder(); - for (String h : obj.headers().keySet()) { - hb.addHeaders( - HeaderEntry.newBuilder().setName(h).addAllValues(obj.headers().getAll(h))); - } - ps.setBytes(idx++, hb.build().toByteArray()); - - serializeObjId(ps, idx++, obj.referenceIndex()); - - Stripes.Builder b = Stripes.newBuilder(); - obj.referenceIndexStripes().stream() - .map( - s -> - Stripe.newBuilder() - .setFirstKey(s.firstKey().rawString()) - .setLastKey(s.lastKey().rawString()) - .setSegment(s.segment().asBytes())) - .forEach(b::addStripes); - serializeBytes(ps, idx++, b.build().toByteString()); - - serializeObjIds(ps, idx++, obj.tail()); - serializeObjIds(ps, idx++, obj.secondaryParents()); - - ByteString index = obj.incrementalIndex(); - if (index.size() > incrementalIndexLimit) { - throw new ObjTooLargeException(index.size(), incrementalIndexLimit); - } - serializeBytes(ps, idx++, index); - - ps.setBoolean(idx++, obj.incompleteIndex()); - ps.setString(idx++, obj.commitType().name()); - return idx; - } - - @Override - CommitObj deserialize(ResultSet rs, ObjId id) throws SQLException { - CommitObj.Builder b = - CommitObj.commitBuilder() - .id(id) - .created(rs.getLong(COL_COMMIT_CREATED)) - .seq(rs.getLong(COL_COMMIT_SEQ)) - .message(rs.getString(COL_COMMIT_MESSAGE)) - .referenceIndex(deserializeObjId(rs, COL_COMMIT_REFERENCE_INDEX)) - .incrementalIndex(deserializeBytes(rs, COL_COMMIT_INCREMENTAL_INDEX)) - .incompleteIndex(rs.getBoolean(COL_COMMIT_INCOMPLETE_INDEX)) - .commitType(CommitType.valueOf(rs.getString(COL_COMMIT_TYPE))); - deserializeObjIds(rs, COL_COMMIT_TAIL, b::addTail); - deserializeObjIds(rs, COL_COMMIT_SECONDARY_PARENTS, b::addSecondaryParents); - - try { - CommitHeaders.Builder h = CommitHeaders.newCommitHeaders(); - Headers headers = Headers.parseFrom(rs.getBytes(COL_COMMIT_HEADERS)); - for (HeaderEntry e : headers.getHeadersList()) { - for (String v : e.getValuesList()) { - h.add(e.getName(), v); - } - } - b.headers(h.build()); - } catch (IOException e) { - throw new RuntimeException(e); - } - - try { - Stripes stripes = Stripes.parseFrom(rs.getBytes(COL_COMMIT_REFERENCE_INDEX_STRIPES)); - stripes.getStripesList().stream() - .map( - s -> - indexStripe( - keyFromString(s.getFirstKey()), - keyFromString(s.getLastKey()), - objIdFromByteBuffer(s.getSegment().asReadOnlyByteBuffer()))) - .forEach(b::addReferenceIndexStripes); - } catch (IOException e) { - throw new RuntimeException(e); - } - - return b.build(); - } - }); - STORE_OBJ_TYPE.put( - ObjType.REF, - new StoreObjDesc() { - @Override - int storeNone(PreparedStatement ps, int idx) throws SQLException { - ps.setNull(idx++, Types.VARCHAR); - ps.setNull(idx++, Types.VARCHAR); - ps.setNull(idx++, Types.BIGINT); - ps.setNull(idx++, Types.VARCHAR); - return idx; - } - - @Override - int store( - PreparedStatement ps, - int idx, - RefObj obj, - int incrementalIndexLimit, - int maxSerializedIndexSize) - throws SQLException { - ps.setString(idx++, obj.name()); - serializeObjId(ps, idx++, obj.initialPointer()); - ps.setLong(idx++, obj.createdAtMicros()); - serializeObjId(ps, idx++, obj.extendedInfoObj()); - return idx; - } - - @Override - RefObj deserialize(ResultSet rs, ObjId id) throws SQLException { - return ref( - id, - rs.getString(COL_REF_NAME), - deserializeObjId(rs, COL_REF_INITIAL_POINTER), - rs.getLong(COL_REF_CREATED_AT), - deserializeObjId(rs, COL_REF_EXTENDED_INFO)); - } - }); - STORE_OBJ_TYPE.put( - ObjType.VALUE, - new StoreObjDesc() { - @Override - int storeNone(PreparedStatement ps, int idx) throws SQLException { - ps.setNull(idx++, Types.VARCHAR); - ps.setNull(idx++, Types.TINYINT); - ps.setNull(idx++, Types.BINARY); - return idx; - } - - @Override - int store( - PreparedStatement ps, - int idx, - ContentValueObj obj, - int incrementalIndexLimit, - int maxSerializedIndexSize) - throws SQLException { - ps.setString(idx++, obj.contentId()); - ps.setInt(idx++, obj.payload()); - serializeBytes(ps, idx++, obj.data()); - return idx; - } - - @Override - ContentValueObj deserialize(ResultSet rs, ObjId id) throws SQLException { - return contentValue( - id, - rs.getString(COL_VALUE_CONTENT_ID), - rs.getInt(COL_VALUE_PAYLOAD), - requireNonNull(deserializeBytes(rs, COL_VALUE_DATA))); - } - }); - STORE_OBJ_TYPE.put( - ObjType.INDEX_SEGMENTS, - new StoreObjDesc() { - @Override - int storeNone(PreparedStatement ps, int idx) throws SQLException { - ps.setNull(idx++, Types.BINARY); - return idx; - } - - @Override - int store( - PreparedStatement ps, - int idx, - IndexSegmentsObj obj, - int incrementalIndexLimit, - int maxSerializedIndexSize) - throws SQLException { - Stripes.Builder b = Stripes.newBuilder(); - obj.stripes().stream() - .map( - s -> - Stripe.newBuilder() - .setFirstKey(s.firstKey().rawString()) - .setLastKey(s.lastKey().rawString()) - .setSegment(s.segment().asBytes())) - .forEach(b::addStripes); - serializeBytes(ps, idx++, b.build().toByteString()); - return idx; - } - - @Override - IndexSegmentsObj deserialize(ResultSet rs, ObjId id) throws SQLException { - try { - Stripes stripes = Stripes.parseFrom(rs.getBytes(COL_SEGMENTS_STRIPES)); - List stripeList = - stripes.getStripesList().stream() - .map( - s -> - indexStripe( - keyFromString(s.getFirstKey()), - keyFromString(s.getLastKey()), - objIdFromByteBuffer(s.getSegment().asReadOnlyByteBuffer()))) - .collect(Collectors.toList()); - return indexSegments(id, stripeList); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }); - STORE_OBJ_TYPE.put( - ObjType.INDEX, - new StoreObjDesc() { - @Override - int storeNone(PreparedStatement ps, int idx) throws SQLException { - ps.setNull(idx++, Types.BINARY); - return idx; - } - - @Override - int store( - PreparedStatement ps, - int idx, - IndexObj obj, - int incrementalIndexLimit, - int maxSerializedIndexSize) - throws SQLException, ObjTooLargeException { - ByteString index = obj.index(); - if (index.size() > maxSerializedIndexSize) { - throw new ObjTooLargeException(index.size(), maxSerializedIndexSize); - } - serializeBytes(ps, idx++, index); - return idx; - } - - @Override - IndexObj deserialize(ResultSet rs, ObjId id) throws SQLException { - ByteString index = deserializeBytes(rs, COL_INDEX_INDEX); - if (index != null) { - return index(id, index); - } - throw new IllegalStateException("Index column for object ID " + id + " is null"); - } - }); - STORE_OBJ_TYPE.put( - ObjType.TAG, - new StoreObjDesc() { - @Override - int storeNone(PreparedStatement ps, int idx) throws SQLException { - ps.setNull(idx++, Types.VARCHAR); - ps.setNull(idx++, Types.BINARY); - ps.setNull(idx++, Types.BINARY); - return idx; - } - - @Override - int store( - PreparedStatement ps, - int idx, - TagObj obj, - int incrementalIndexLimit, - int maxSerializedIndexSize) - throws SQLException { - ps.setString(idx++, obj.message()); - Headers.Builder hb = Headers.newBuilder(); - CommitHeaders headers = obj.headers(); - if (headers != null) { - for (String h : headers.keySet()) { - hb.addHeaders(HeaderEntry.newBuilder().setName(h).addAllValues(headers.getAll(h))); - } - } - ps.setBytes(idx++, hb.build().toByteArray()); - serializeBytes(ps, idx++, obj.signature()); - return idx; - } - - @Override - TagObj deserialize(ResultSet rs, ObjId id) throws SQLException { - CommitHeaders tagHeaders = null; - try { - Headers headers = Headers.parseFrom(deserializeBytes(rs, COL_TAG_HEADERS)); - if (headers.getHeadersCount() > 0) { - CommitHeaders.Builder h = CommitHeaders.newCommitHeaders(); - for (HeaderEntry e : headers.getHeadersList()) { - for (String v : e.getValuesList()) { - h.add(e.getName(), v); - } - } - tagHeaders = h.build(); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - - return tag( - id, - rs.getString(COL_TAG_MESSAGE), - tagHeaders, - deserializeBytes(rs, COL_TAG_SIGNATURE)); - } - }); - STORE_OBJ_TYPE.put( - ObjType.STRING, - new StoreObjDesc() { - @Override - int storeNone(PreparedStatement ps, int idx) throws SQLException { - ps.setNull(idx++, Types.VARCHAR); - ps.setNull(idx++, Types.VARCHAR); - ps.setNull(idx++, Types.VARCHAR); - ps.setNull(idx++, Types.VARCHAR); - ps.setNull(idx++, Types.BINARY); - return idx; - } - - @Override - int store( - PreparedStatement ps, - int idx, - StringObj obj, - int incrementalIndexLimit, - int maxSerializedIndexSize) - throws SQLException { - ps.setString(idx++, obj.contentType()); - ps.setString(idx++, obj.compression().name()); - ps.setString(idx++, obj.filename()); - serializeObjIds(ps, idx++, obj.predecessors()); - serializeBytes(ps, idx++, obj.text()); - return idx; - } - - @Override - StringObj deserialize(ResultSet rs, ObjId id) throws SQLException { - return stringData( - id, - rs.getString(COL_STRING_CONTENT_TYPE), - Compression.valueOf(rs.getString(COL_STRING_COMPRESSION)), - rs.getString(COL_STRING_FILENAME), - deserializeObjIds(rs, COL_STRING_PREDECESSORS), - deserializeBytes(rs, COL_STRING_TEXT)); - } - }); - } - - private static void serializeBytes(PreparedStatement ps, int idx, ByteString blob) - throws SQLException { - if (blob == null) { - ps.setNull(idx, Types.BLOB); - return; - } - ps.setBinaryStream(idx, blob.newInput()); - } - - private static ByteString deserializeBytes(ResultSet rs, int idx) throws SQLException { - byte[] bytes = rs.getBytes(idx); - return bytes != null ? unsafeWrap(bytes) : null; - } - - private static Reference deserializeReference(ResultSet rs) throws SQLException { - byte[] prevBytes = rs.getBytes(6); - List previousPointers = - prevBytes != null ? deserializePreviousPointers(prevBytes) : emptyList(); - return Reference.reference( - rs.getString(1), - deserializeObjId(rs, 2), - rs.getBoolean(3), - rs.getLong(4), - deserializeObjId(rs, 5), - previousPointers); - } - - private static ObjId deserializeObjId(ResultSet rs, int col) throws SQLException { - String s = rs.getString(col); - return s != null ? objIdFromString(s) : null; - } - - private static void serializeObjId(PreparedStatement ps, int col, ObjId value) - throws SQLException { - if (value != null) { - ps.setString(col, value.toString()); - } else { - ps.setNull(col, Types.VARCHAR); - } - } - - @SuppressWarnings("SameParameterValue") - private static List deserializeObjIds(ResultSet rs, int col) throws SQLException { - List r = new ArrayList<>(); - deserializeObjIds(rs, col, r::add); - return r; - } - - private static void deserializeObjIds(ResultSet rs, int col, Consumer consumer) - throws SQLException { - String s = rs.getString(col); - if (s == null || s.isEmpty()) { - return; - } - int i = 0; - while (true) { - int next = s.indexOf(',', i); - String idAsString; - if (next == -1) { - idAsString = s.substring(i); - } else { - idAsString = s.substring(i, next); - i = next + 1; - } - consumer.accept(objIdFromString(idAsString)); - if (next == -1) { - return; - } - } - } - - private static void serializeObjIds(PreparedStatement ps, int col, List values) - throws SQLException { - if (values != null && !values.isEmpty()) { - ps.setString(col, values.stream().map(ObjId::toString).collect(Collectors.joining(","))); - } else { - ps.setNull(col, Types.VARCHAR); - } - } - @VisibleForTesting static String sqlSelectMultiple(String sql, int count) { if (count == 1) { diff --git a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/JdbcBackend.java b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/JdbcBackend.java index 7a6cabf2d53..699ed133446 100644 --- a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/JdbcBackend.java +++ b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/JdbcBackend.java @@ -17,8 +17,11 @@ import static com.google.common.base.Preconditions.checkState; import static org.projectnessie.versioned.storage.jdbc.AbstractJdbcPersist.sqlSelectMultiple; +import static org.projectnessie.versioned.storage.jdbc.JdbcColumnType.BIGINT; +import static org.projectnessie.versioned.storage.jdbc.JdbcColumnType.BOOL; import static org.projectnessie.versioned.storage.jdbc.JdbcColumnType.NAME; import static org.projectnessie.versioned.storage.jdbc.JdbcColumnType.OBJ_ID; +import static org.projectnessie.versioned.storage.jdbc.JdbcColumnType.VARBINARY; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COLS_OBJS_ALL; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_OBJ_ID; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_REFS_CREATED_AT; @@ -28,8 +31,6 @@ import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_REFS_POINTER; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_REFS_PREVIOUS; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_REPO_ID; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.CREATE_TABLE_OBJS; -import static org.projectnessie.versioned.storage.jdbc.SqlConstants.CREATE_TABLE_REFS; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.ERASE_OBJS; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.ERASE_REFS; import static org.projectnessie.versioned.storage.jdbc.SqlConstants.TABLE_OBJS; @@ -41,11 +42,10 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.text.MessageFormat; -import java.util.Arrays; import java.util.LinkedHashMap; import java.util.Locale; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -60,6 +60,8 @@ final class JdbcBackend implements Backend { private final DataSource dataSource; private final boolean closeDataSource; private final JdbcBackendConfig config; + private final String createTableRefsSql; + private final String createTableObjsSql; JdbcBackend( @Nonnull @jakarta.annotation.Nonnull JdbcBackendConfig config, @@ -69,6 +71,70 @@ final class JdbcBackend implements Backend { this.dataSource = config.dataSource(); this.databaseSpecific = databaseSpecific; this.closeDataSource = closeDataSource; + createTableRefsSql = buildCreateTableRefsSql(databaseSpecific); + createTableObjsSql = buildCreateTableObjsSql(databaseSpecific); + } + + private String buildCreateTableRefsSql(DatabaseSpecific databaseSpecific) { + Map columnTypes = databaseSpecific.columnTypes(); + return "CREATE TABLE " + + TABLE_REFS + + "\n (\n " + + COL_REPO_ID + + " " + + columnTypes.get(NAME) + + ",\n " + + COL_REFS_NAME + + " " + + columnTypes.get(NAME) + + ",\n " + + COL_REFS_POINTER + + " " + + columnTypes.get(OBJ_ID) + + ",\n " + + COL_REFS_DELETED + + " " + + columnTypes.get(BOOL) + + ",\n " + + COL_REFS_CREATED_AT + + " " + + columnTypes.get(BIGINT) + + " DEFAULT 0,\n " + + COL_REFS_EXTENDED_INFO + + " " + + columnTypes.get(OBJ_ID) + + ",\n " + + COL_REFS_PREVIOUS + + " " + + columnTypes.get(VARBINARY) + + ",\n PRIMARY KEY (" + + COL_REPO_ID + + ", " + + COL_REFS_NAME + + ")\n )"; + } + + private String buildCreateTableObjsSql(DatabaseSpecific databaseSpecific) { + Map columnTypes = databaseSpecific.columnTypes(); + StringBuilder sb = + new StringBuilder() + .append("CREATE TABLE ") + .append(TABLE_OBJS) + .append(" (\n ") + .append(COL_REPO_ID) + .append(" ") + .append(columnTypes.get(NAME)); + for (Entry entry : COLS_OBJS_ALL.entrySet()) { + String colName = entry.getKey(); + String colType = columnTypes.get(entry.getValue()); + sb.append(",\n ").append(colName).append(" ").append(colType); + } + sb.append(",\n PRIMARY KEY (") + .append(COL_REPO_ID) + .append(", ") + .append(COL_OBJ_ID) + .append(")\n )"); + return sb.toString(); } static RuntimeException unhandledSQLException(SQLException e) { @@ -113,7 +179,7 @@ public void setupSchema() { createTableIfNotExists( conn, TABLE_REFS, - CREATE_TABLE_REFS, + createTableRefsSql, Stream.of( COL_REPO_ID, COL_REFS_NAME, @@ -127,9 +193,8 @@ public void setupSchema() { createTableIfNotExists( conn, TABLE_OBJS, - CREATE_TABLE_OBJS, - Stream.concat( - Stream.of(COL_REPO_ID), Arrays.stream(COLS_OBJS_ALL.split(",")).map(String::trim)) + createTableObjsSql, + Stream.concat(Stream.of(COL_REPO_ID), COLS_OBJS_ALL.keySet().stream()) .collect(Collectors.toSet()), ImmutableMap.of(COL_REPO_ID, nameTypeId, COL_OBJ_ID, objIdTypeId)); } catch (SQLException e) { @@ -144,16 +209,11 @@ private void createTableIfNotExists( Set expectedColumns, Map expectedPrimaryKey) throws SQLException { - Map columnTypesMap = databaseSpecific.columnTypes(); - Object[] types = - Arrays.stream(JdbcColumnType.values()).map(columnTypesMap::get).toArray(Object[]::new); // TODO implement catalog + schema stuff... String catalog = config.catalog(); String schema = config.schema(); - createTable = MessageFormat.format(createTable, types); - try (Statement st = conn.createStatement()) { if (conn.getMetaData().storesLowerCaseIdentifiers()) { tableName = tableName.toLowerCase(Locale.ROOT); diff --git a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/JdbcSerde.java b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/JdbcSerde.java new file mode 100644 index 00000000000..003f265615c --- /dev/null +++ b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/JdbcSerde.java @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.jdbc; + +import static java.util.Collections.emptyList; +import static org.projectnessie.nessie.relocated.protobuf.UnsafeByteOperations.unsafeWrap; +import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromString; +import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_REFS_CREATED_AT; +import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_REFS_DELETED; +import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_REFS_EXTENDED_INFO; +import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_REFS_NAME; +import static org.projectnessie.versioned.storage.jdbc.SqlConstants.COL_REFS_POINTER; +import static org.projectnessie.versioned.storage.serialize.ProtoSerialization.deserializePreviousPointers; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.common.persist.Reference; + +public class JdbcSerde { + + public static void serializeBytes( + PreparedStatement ps, int idx, ByteString blob, DatabaseSpecific databaseSpecific) + throws SQLException { + if (blob == null) { + ps.setNull(idx, databaseSpecific.columnTypeIds().get(JdbcColumnType.VARBINARY)); + return; + } + ps.setBinaryStream(idx, blob.newInput()); + } + + public static ByteString deserializeBytes(ResultSet rs, String col) throws SQLException { + byte[] bytes = rs.getBytes(col); + return bytes != null ? unsafeWrap(bytes) : null; + } + + public static ObjId deserializeObjId(ResultSet rs, String col) throws SQLException { + String s = rs.getString(col); + return s != null ? objIdFromString(s) : null; + } + + public static void serializeObjId( + PreparedStatement ps, int col, ObjId value, DatabaseSpecific databaseSpecific) + throws SQLException { + if (value != null) { + ps.setString(col, value.toString()); + } else { + ps.setNull(col, databaseSpecific.columnTypeIds().get(JdbcColumnType.VARCHAR)); + } + } + + @SuppressWarnings("SameParameterValue") + public static List deserializeObjIds(ResultSet rs, String col) throws SQLException { + List r = new ArrayList<>(); + deserializeObjIds(rs, col, r::add); + return r; + } + + public static void deserializeObjIds(ResultSet rs, String col, Consumer consumer) + throws SQLException { + String s = rs.getString(col); + if (s == null || s.isEmpty()) { + return; + } + int i = 0; + while (true) { + int next = s.indexOf(',', i); + String idAsString; + if (next == -1) { + idAsString = s.substring(i); + } else { + idAsString = s.substring(i, next); + i = next + 1; + } + consumer.accept(objIdFromString(idAsString)); + if (next == -1) { + return; + } + } + } + + public static void serializeObjIds( + PreparedStatement ps, int col, List values, DatabaseSpecific databaseSpecific) + throws SQLException { + if (values != null && !values.isEmpty()) { + ps.setString(col, values.stream().map(ObjId::toString).collect(Collectors.joining(","))); + } else { + ps.setNull(col, databaseSpecific.columnTypeIds().get(JdbcColumnType.VARCHAR)); + } + } + + public static Reference deserializeReference(ResultSet rs) throws SQLException { + byte[] prevBytes = rs.getBytes(6); + List previousPointers = + prevBytes != null ? deserializePreviousPointers(prevBytes) : emptyList(); + return Reference.reference( + rs.getString(COL_REFS_NAME), + deserializeObjId(rs, COL_REFS_POINTER), + rs.getBoolean(COL_REFS_DELETED), + rs.getLong(COL_REFS_CREATED_AT), + deserializeObjId(rs, COL_REFS_EXTENDED_INFO), + previousPointers); + } +} diff --git a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/SqlConstants.java b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/SqlConstants.java index 1364ba78c64..3cbb659f353 100644 --- a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/SqlConstants.java +++ b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/SqlConstants.java @@ -15,6 +15,13 @@ */ package org.projectnessie.versioned.storage.jdbc; +import static java.util.Map.entry; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import java.util.stream.Stream; +import org.projectnessie.versioned.storage.jdbc.serializers.ObjSerializers; + final class SqlConstants { static final int MAX_BATCH_SIZE = 50; @@ -31,72 +38,6 @@ final class SqlConstants { "DELETE FROM " + TABLE_OBJS + " WHERE " + COL_REPO_ID + "=? AND " + COL_OBJ_ID + "=?"; static final String COL_OBJ_TYPE = "obj_type"; - static final String COLS_COMMIT = - "c_created, c_seq, c_message, c_headers, c_reference_index, c_reference_index_stripes, c_tail, c_secondary_parents, c_incremental_index, c_incomplete_index, c_commit_type"; - static final String COLS_REF = "r_name, r_initial_pointer, r_created_at, r_extended_info"; - static final String COLS_VALUE = "v_content_id, v_payload, v_data"; - static final String COLS_SEGMENTS = "i_stripes"; - static final String COLS_INDEX = "i_index"; - static final String COLS_TAG = "t_message, t_headers, t_signature"; - static final String COLS_STRING = - "s_content_type, s_compression, s_filename, s_predecessors, s_text"; - - static final String STORE_OBJ = - "INSERT INTO " - + TABLE_OBJS - + " (" - + COL_REPO_ID - + ", " - + COL_OBJ_ID - + ", " - + COL_OBJ_TYPE - + ", " - // MUST keep enum order of ObjType here ! - + COLS_REF - + ", " - + COLS_COMMIT - + ", " - + COLS_TAG - + ", " - + COLS_VALUE - + ", " - + COLS_STRING - + ", " - + COLS_SEGMENTS - + ", " - + COLS_INDEX - + ") VALUES (?,?,? " - + ",?,?,?,? " // REF - + ",?,?,?,?,?,?,?,?,?,?,?" // COMMIT - + ",?,?,? " // TAG - + ",?,?,? " // VALUE - + ",?,?,?,?,? " // STRING - + ",? " // SEGMENTS - + ",? " // INDEX - + ")"; - - static final String CREATE_TABLE_OBJS = - "CREATE TABLE " - + TABLE_OBJS - + "\n (\n " - + COL_REPO_ID - + " {0}, " - + COL_OBJ_ID - + " {1}, " - + COL_OBJ_TYPE - + " {0}" - + ",\n c_created {5}, c_seq {5}, c_message {6}, c_headers {4}, c_reference_index {1}, c_reference_index_stripes {4}, c_tail {2}, c_secondary_parents {2}, c_incremental_index {4}, c_incomplete_index {3}, c_commit_type {0}" - + ",\n r_name {0}, r_initial_pointer {1}, r_created_at {5}, r_extended_info {1}" - + ",\n v_content_id {0}, v_payload {5}, v_data {4}" - + ",\n i_stripes {4}" - + ",\n i_index {4}" - + ",\n t_message {6}, t_headers {4}, t_signature {4}" - + ",\n s_content_type {0}, s_compression {0}, s_filename {0}, s_predecessors {2}, s_text {4}" - + ",\n PRIMARY KEY (" - + COL_REPO_ID - + ", " - + COL_OBJ_ID - + ")\n )"; static final String COL_REFS_NAME = "ref_name"; static final String COL_REFS_POINTER = "pointer"; static final String COL_REFS_DELETED = "deleted"; @@ -200,75 +141,16 @@ final class SqlConstants { + "=? AND " + COL_REFS_NAME + " IN (?)"; - static final String CREATE_TABLE_REFS = - "CREATE TABLE " - + TABLE_REFS - + "\n (\n " - + COL_REPO_ID - + " {0}, " - + COL_REFS_NAME - + " {0}, " - + COL_REFS_POINTER - + " {1}, " - + COL_REFS_DELETED - + " {3}, " - + COL_REFS_CREATED_AT - + " {5} DEFAULT 0, " - + COL_REFS_EXTENDED_INFO - + " {1}, " - + COL_REFS_PREVIOUS - + " {4}, " - + "\n PRIMARY KEY (" - + COL_REPO_ID - + ", " - + COL_REFS_NAME - + ")\n )"; - static final String COLS_OBJS_ALL = - COL_OBJ_ID - + ", " - + COL_OBJ_TYPE - + ", " - + COLS_COMMIT - + ", " - + COLS_REF - + ", " - + COLS_VALUE - + ", " - + COLS_SEGMENTS - + ", " - + COLS_INDEX - + ", " - + COLS_TAG - + ", " - + COLS_STRING; - static final int COL_COMMIT_CREATED = 3; // obj_id + obj_type before this column - static final int COL_COMMIT_SEQ = COL_COMMIT_CREATED + 1; - static final int COL_COMMIT_MESSAGE = COL_COMMIT_SEQ + 1; - static final int COL_COMMIT_HEADERS = COL_COMMIT_MESSAGE + 1; - static final int COL_COMMIT_REFERENCE_INDEX = COL_COMMIT_HEADERS + 1; - static final int COL_COMMIT_REFERENCE_INDEX_STRIPES = COL_COMMIT_REFERENCE_INDEX + 1; - static final int COL_COMMIT_TAIL = COL_COMMIT_REFERENCE_INDEX_STRIPES + 1; - static final int COL_COMMIT_SECONDARY_PARENTS = COL_COMMIT_TAIL + 1; - static final int COL_COMMIT_INCREMENTAL_INDEX = COL_COMMIT_SECONDARY_PARENTS + 1; - static final int COL_COMMIT_INCOMPLETE_INDEX = COL_COMMIT_INCREMENTAL_INDEX + 1; - static final int COL_COMMIT_TYPE = COL_COMMIT_INCOMPLETE_INDEX + 1; - static final int COL_REF_NAME = COL_COMMIT_TYPE + 1; - static final int COL_REF_INITIAL_POINTER = COL_REF_NAME + 1; - static final int COL_REF_CREATED_AT = COL_REF_INITIAL_POINTER + 1; - static final int COL_REF_EXTENDED_INFO = COL_REF_CREATED_AT + 1; - static final int COL_VALUE_CONTENT_ID = COL_REF_EXTENDED_INFO + 1; - static final int COL_VALUE_PAYLOAD = COL_VALUE_CONTENT_ID + 1; - static final int COL_VALUE_DATA = COL_VALUE_PAYLOAD + 1; - static final int COL_SEGMENTS_STRIPES = COL_VALUE_DATA + 1; - static final int COL_INDEX_INDEX = COL_SEGMENTS_STRIPES + 1; - static final int COL_TAG_MESSAGE = COL_INDEX_INDEX + 1; - static final int COL_TAG_HEADERS = COL_TAG_MESSAGE + 1; - static final int COL_TAG_SIGNATURE = COL_TAG_HEADERS + 1; - static final int COL_STRING_CONTENT_TYPE = COL_TAG_SIGNATURE + 1; - static final int COL_STRING_COMPRESSION = COL_STRING_CONTENT_TYPE + 1; - static final int COL_STRING_FILENAME = COL_STRING_COMPRESSION + 1; - static final int COL_STRING_PREDECESSORS = COL_STRING_FILENAME + 1; - static final int COL_STRING_TEXT = COL_STRING_PREDECESSORS + 1; + + static final Map COLS_OBJS_ALL = + Stream.concat( + Stream.of( + entry(COL_OBJ_ID, JdbcColumnType.OBJ_ID), + entry(COL_OBJ_TYPE, JdbcColumnType.NAME)), + ObjSerializers.ALL_SERIALIZERS.stream() + .flatMap(serializer -> serializer.columns().entrySet().stream()) + .sorted(Map.Entry.comparingByKey())) + .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); static final String FETCH_OBJ_TYPE = "SELECT " @@ -283,7 +165,7 @@ final class SqlConstants { static final String FIND_OBJS = "SELECT " - + COLS_OBJS_ALL + + String.join(", ", COLS_OBJS_ALL.keySet()) + " FROM " + TABLE_OBJS + " WHERE " @@ -296,7 +178,7 @@ final class SqlConstants { static final String SCAN_OBJS = "SELECT " - + COLS_OBJS_ALL + + String.join(", ", COLS_OBJS_ALL.keySet()) + " FROM " + TABLE_OBJS + " WHERE " diff --git a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/CommitObjSerializer.java b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/CommitObjSerializer.java new file mode 100644 index 00000000000..2be8a0fb0af --- /dev/null +++ b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/CommitObjSerializer.java @@ -0,0 +1,185 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.jdbc.serializers; + +import static org.projectnessie.versioned.storage.common.indexes.StoreKey.keyFromString; +import static org.projectnessie.versioned.storage.common.objtypes.IndexStripe.indexStripe; +import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromByteBuffer; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.deserializeBytes; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.deserializeObjId; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.deserializeObjIds; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.serializeBytes; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.serializeObjId; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.serializeObjIds; + +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import java.util.function.Function; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.objtypes.CommitHeaders; +import org.projectnessie.versioned.storage.common.objtypes.CommitObj; +import org.projectnessie.versioned.storage.common.objtypes.CommitType; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.HeaderEntry; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.Headers; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.Stripe; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.Stripes; +import org.projectnessie.versioned.storage.jdbc.DatabaseSpecific; +import org.projectnessie.versioned.storage.jdbc.JdbcColumnType; + +public class CommitObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new CommitObjSerializer(); + + private static final String COL_COMMIT_CREATED = "c_created"; + private static final String COL_COMMIT_SEQ = "c_seq"; + private static final String COL_COMMIT_MESSAGE = "c_message"; + private static final String COL_COMMIT_HEADERS = "c_headers"; + private static final String COL_COMMIT_REFERENCE_INDEX = "c_reference_index"; + private static final String COL_COMMIT_REFERENCE_INDEX_STRIPES = "c_reference_index_stripes"; + private static final String COL_COMMIT_TAIL = "c_tail"; + private static final String COL_COMMIT_SECONDARY_PARENTS = "c_secondary_parents"; + private static final String COL_COMMIT_INCREMENTAL_INDEX = "c_incremental_index"; + private static final String COL_COMMIT_INCOMPLETE_INDEX = "c_incomplete_index"; + private static final String COL_COMMIT_TYPE = "c_commit_type"; + + private static final Map COLS = + ImmutableMap.builder() + .put(COL_COMMIT_CREATED, JdbcColumnType.BIGINT) + .put(COL_COMMIT_SEQ, JdbcColumnType.BIGINT) + .put(COL_COMMIT_MESSAGE, JdbcColumnType.VARCHAR) + .put(COL_COMMIT_HEADERS, JdbcColumnType.VARBINARY) + .put(COL_COMMIT_REFERENCE_INDEX, JdbcColumnType.OBJ_ID) + .put(COL_COMMIT_REFERENCE_INDEX_STRIPES, JdbcColumnType.VARBINARY) + .put(COL_COMMIT_TAIL, JdbcColumnType.OBJ_ID_LIST) + .put(COL_COMMIT_SECONDARY_PARENTS, JdbcColumnType.OBJ_ID_LIST) + .put(COL_COMMIT_INCREMENTAL_INDEX, JdbcColumnType.VARBINARY) + .put(COL_COMMIT_INCOMPLETE_INDEX, JdbcColumnType.BOOL) + .put(COL_COMMIT_TYPE, JdbcColumnType.NAME) + .build(); + + private CommitObjSerializer() {} + + @Override + public Map columns() { + return COLS; + } + + @Override + public void serialize( + PreparedStatement ps, + CommitObj obj, + int incrementalIndexLimit, + int maxSerializedIndexSize, + Function nameToIdx, + DatabaseSpecific databaseSpecific) + throws SQLException, ObjTooLargeException { + ps.setLong(nameToIdx.apply(COL_COMMIT_CREATED), obj.created()); + ps.setLong(nameToIdx.apply(COL_COMMIT_SEQ), obj.seq()); + ps.setString(nameToIdx.apply(COL_COMMIT_MESSAGE), obj.message()); + + obj.headers(); + Headers.Builder hb = Headers.newBuilder(); + for (String h : obj.headers().keySet()) { + hb.addHeaders(HeaderEntry.newBuilder().setName(h).addAllValues(obj.headers().getAll(h))); + } + ps.setBytes(nameToIdx.apply(COL_COMMIT_HEADERS), hb.build().toByteArray()); + + serializeObjId( + ps, nameToIdx.apply(COL_COMMIT_REFERENCE_INDEX), obj.referenceIndex(), databaseSpecific); + + Stripes.Builder b = Stripes.newBuilder(); + obj.referenceIndexStripes().stream() + .map( + s -> + Stripe.newBuilder() + .setFirstKey(s.firstKey().rawString()) + .setLastKey(s.lastKey().rawString()) + .setSegment(s.segment().asBytes())) + .forEach(b::addStripes); + serializeBytes( + ps, + nameToIdx.apply(COL_COMMIT_REFERENCE_INDEX_STRIPES), + b.build().toByteString(), + databaseSpecific); + + serializeObjIds(ps, nameToIdx.apply(COL_COMMIT_TAIL), obj.tail(), databaseSpecific); + serializeObjIds( + ps, + nameToIdx.apply(COL_COMMIT_SECONDARY_PARENTS), + obj.secondaryParents(), + databaseSpecific); + + ByteString index = obj.incrementalIndex(); + if (index.size() > incrementalIndexLimit) { + throw new ObjTooLargeException(index.size(), incrementalIndexLimit); + } + serializeBytes(ps, nameToIdx.apply(COL_COMMIT_INCREMENTAL_INDEX), index, databaseSpecific); + + ps.setBoolean(nameToIdx.apply(COL_COMMIT_INCOMPLETE_INDEX), obj.incompleteIndex()); + ps.setString(nameToIdx.apply(COL_COMMIT_TYPE), obj.commitType().name()); + } + + @Override + public CommitObj deserialize(ResultSet rs, ObjId id) throws SQLException { + CommitObj.Builder b = + CommitObj.commitBuilder() + .id(id) + .created(rs.getLong(COL_COMMIT_CREATED)) + .seq(rs.getLong(COL_COMMIT_SEQ)) + .message(rs.getString(COL_COMMIT_MESSAGE)) + .referenceIndex(deserializeObjId(rs, COL_COMMIT_REFERENCE_INDEX)) + .incrementalIndex(deserializeBytes(rs, COL_COMMIT_INCREMENTAL_INDEX)) + .incompleteIndex(rs.getBoolean(COL_COMMIT_INCOMPLETE_INDEX)) + .commitType(CommitType.valueOf(rs.getString(COL_COMMIT_TYPE))); + deserializeObjIds(rs, COL_COMMIT_TAIL, b::addTail); + deserializeObjIds(rs, COL_COMMIT_SECONDARY_PARENTS, b::addSecondaryParents); + + try { + CommitHeaders.Builder h = CommitHeaders.newCommitHeaders(); + Headers headers = Headers.parseFrom(rs.getBytes(COL_COMMIT_HEADERS)); + for (HeaderEntry e : headers.getHeadersList()) { + for (String v : e.getValuesList()) { + h.add(e.getName(), v); + } + } + b.headers(h.build()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + try { + Stripes stripes = Stripes.parseFrom(rs.getBytes(COL_COMMIT_REFERENCE_INDEX_STRIPES)); + stripes.getStripesList().stream() + .map( + s -> + indexStripe( + keyFromString(s.getFirstKey()), + keyFromString(s.getLastKey()), + objIdFromByteBuffer(s.getSegment().asReadOnlyByteBuffer()))) + .forEach(b::addReferenceIndexStripes); + } catch (IOException e) { + throw new RuntimeException(e); + } + + return b.build(); + } +} diff --git a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/ContentValueObjSerializer.java b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/ContentValueObjSerializer.java new file mode 100644 index 00000000000..0d024de4017 --- /dev/null +++ b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/ContentValueObjSerializer.java @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.jdbc.serializers; + +import static java.util.Objects.requireNonNull; +import static org.projectnessie.versioned.storage.common.objtypes.ContentValueObj.contentValue; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.deserializeBytes; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.serializeBytes; + +import com.google.common.collect.ImmutableMap; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import java.util.function.Function; +import org.projectnessie.versioned.storage.common.objtypes.ContentValueObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.jdbc.DatabaseSpecific; +import org.projectnessie.versioned.storage.jdbc.JdbcColumnType; + +public class ContentValueObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new ContentValueObjSerializer(); + + private static final String COL_VALUE_CONTENT_ID = "v_content_id"; + private static final String COL_VALUE_PAYLOAD = "v_payload"; + private static final String COL_VALUE_DATA = "v_data"; + + private static final Map COLS = + ImmutableMap.of( + COL_VALUE_CONTENT_ID, JdbcColumnType.NAME, + COL_VALUE_PAYLOAD, JdbcColumnType.BIGINT, + COL_VALUE_DATA, JdbcColumnType.VARBINARY); + + private ContentValueObjSerializer() {} + + @Override + public Map columns() { + return COLS; + } + + @Override + public void serialize( + PreparedStatement ps, + ContentValueObj obj, + int incrementalIndexLimit, + int maxSerializedIndexSize, + Function nameToIdx, + DatabaseSpecific databaseSpecific) + throws SQLException { + ps.setString(nameToIdx.apply(COL_VALUE_CONTENT_ID), obj.contentId()); + ps.setInt(nameToIdx.apply(COL_VALUE_PAYLOAD), obj.payload()); + serializeBytes(ps, nameToIdx.apply(COL_VALUE_DATA), obj.data(), databaseSpecific); + } + + @Override + public ContentValueObj deserialize(ResultSet rs, ObjId id) throws SQLException { + return contentValue( + id, + rs.getString(COL_VALUE_CONTENT_ID), + rs.getInt(COL_VALUE_PAYLOAD), + requireNonNull(deserializeBytes(rs, COL_VALUE_DATA))); + } +} diff --git a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/CustomObjSerializer.java b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/CustomObjSerializer.java new file mode 100644 index 00000000000..78f6c35d9ec --- /dev/null +++ b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/CustomObjSerializer.java @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.jdbc.serializers; + +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.serializeBytes; + +import com.google.common.collect.ImmutableMap; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import java.util.function.Function; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.jdbc.DatabaseSpecific; +import org.projectnessie.versioned.storage.jdbc.JdbcColumnType; +import org.projectnessie.versioned.storage.serialize.SmileSerialization; + +public class CustomObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new CustomObjSerializer(); + + private static final String COL_CUSTOM_CLASS = "x_class"; + private static final String COL_CUSTOM_DATA = "x_data"; + + private static final Map COLS = + ImmutableMap.of( + COL_CUSTOM_CLASS, JdbcColumnType.NAME, + COL_CUSTOM_DATA, JdbcColumnType.VARBINARY); + + private CustomObjSerializer() {} + + @Override + public Map columns() { + return COLS; + } + + @Override + public void serialize( + PreparedStatement ps, + Obj obj, + int incrementalIndexLimit, + int maxSerializedIndexSize, + Function nameToIdx, + DatabaseSpecific databaseSpecific) + throws SQLException, ObjTooLargeException { + ps.setString(nameToIdx.apply(COL_CUSTOM_CLASS), obj.type().targetClass().getName()); + serializeBytes( + ps, + nameToIdx.apply(COL_CUSTOM_DATA), + ByteString.copyFrom(SmileSerialization.serializeObj(obj)), + databaseSpecific); + } + + @Override + public Obj deserialize(ResultSet rs, ObjId id) throws SQLException { + return SmileSerialization.deserializeObj( + id, rs.getBytes(COL_CUSTOM_DATA), rs.getString(COL_CUSTOM_CLASS)); + } +} diff --git a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/IndexObjSerializer.java b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/IndexObjSerializer.java new file mode 100644 index 00000000000..b321bf13b27 --- /dev/null +++ b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/IndexObjSerializer.java @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.jdbc.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.IndexObj.index; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.deserializeBytes; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.serializeBytes; + +import com.google.common.collect.ImmutableMap; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import java.util.function.Function; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.objtypes.IndexObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.jdbc.DatabaseSpecific; +import org.projectnessie.versioned.storage.jdbc.JdbcColumnType; + +public class IndexObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new IndexObjSerializer(); + + private static final String COL_INDEX_INDEX = "i_index"; + + private static final Map COLS = + ImmutableMap.of(COL_INDEX_INDEX, JdbcColumnType.VARBINARY); + + private IndexObjSerializer() {} + + @Override + public Map columns() { + return COLS; + } + + @Override + public void serialize( + PreparedStatement ps, + IndexObj obj, + int incrementalIndexLimit, + int maxSerializedIndexSize, + Function nameToIdx, + DatabaseSpecific databaseSpecific) + throws SQLException, ObjTooLargeException { + ByteString index = obj.index(); + if (index.size() > maxSerializedIndexSize) { + throw new ObjTooLargeException(index.size(), maxSerializedIndexSize); + } + serializeBytes(ps, nameToIdx.apply(COL_INDEX_INDEX), index, databaseSpecific); + } + + @Override + public IndexObj deserialize(ResultSet rs, ObjId id) throws SQLException { + ByteString index = deserializeBytes(rs, COL_INDEX_INDEX); + if (index != null) { + return index(id, index); + } + throw new IllegalStateException("Index column for object ID " + id + " is null"); + } +} diff --git a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/IndexSegmentsObjSerializer.java b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/IndexSegmentsObjSerializer.java new file mode 100644 index 00000000000..9f97b445b22 --- /dev/null +++ b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/IndexSegmentsObjSerializer.java @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.jdbc.serializers; + +import static org.projectnessie.versioned.storage.common.indexes.StoreKey.keyFromString; +import static org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj.indexSegments; +import static org.projectnessie.versioned.storage.common.objtypes.IndexStripe.indexStripe; +import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromByteBuffer; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.serializeBytes; + +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj; +import org.projectnessie.versioned.storage.common.objtypes.IndexStripe; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.Stripe; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.Stripes; +import org.projectnessie.versioned.storage.jdbc.DatabaseSpecific; +import org.projectnessie.versioned.storage.jdbc.JdbcColumnType; + +public class IndexSegmentsObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new IndexSegmentsObjSerializer(); + + private static final String COL_SEGMENTS_STRIPES = "i_stripes"; + + private static final Map COLS = + ImmutableMap.of(COL_SEGMENTS_STRIPES, JdbcColumnType.VARBINARY); + + private IndexSegmentsObjSerializer() {} + + @Override + public Map columns() { + return COLS; + } + + @Override + public void serialize( + PreparedStatement ps, + IndexSegmentsObj obj, + int incrementalIndexLimit, + int maxSerializedIndexSize, + Function nameToIdx, + DatabaseSpecific databaseSpecific) + throws SQLException { + Stripes.Builder b = Stripes.newBuilder(); + obj.stripes().stream() + .map( + s -> + Stripe.newBuilder() + .setFirstKey(s.firstKey().rawString()) + .setLastKey(s.lastKey().rawString()) + .setSegment(s.segment().asBytes())) + .forEach(b::addStripes); + serializeBytes( + ps, nameToIdx.apply(COL_SEGMENTS_STRIPES), b.build().toByteString(), databaseSpecific); + } + + @Override + public IndexSegmentsObj deserialize(ResultSet rs, ObjId id) throws SQLException { + try { + Stripes stripes = Stripes.parseFrom(rs.getBytes(COL_SEGMENTS_STRIPES)); + List stripeList = + stripes.getStripesList().stream() + .map( + s -> + indexStripe( + keyFromString(s.getFirstKey()), + keyFromString(s.getLastKey()), + objIdFromByteBuffer(s.getSegment().asReadOnlyByteBuffer()))) + .collect(Collectors.toList()); + return indexSegments(id, stripeList); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/ObjSerializer.java b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/ObjSerializer.java new file mode 100644 index 00000000000..7911a171e7c --- /dev/null +++ b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/ObjSerializer.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.jdbc.serializers; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import java.util.Map.Entry; +import java.util.function.Function; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.jdbc.DatabaseSpecific; +import org.projectnessie.versioned.storage.jdbc.JdbcColumnType; + +public interface ObjSerializer { + + Map columns(); + + void serialize( + PreparedStatement ps, + O obj, + int incrementalIndexLimit, + int maxSerializedIndexSize, + Function nameToIdx, + DatabaseSpecific databaseSpecific) + throws SQLException, ObjTooLargeException; + + O deserialize(ResultSet rs, ObjId id) throws SQLException; + + default void setNull( + PreparedStatement ps, Function nameToIdx, DatabaseSpecific databaseSpecific) + throws SQLException { + Map typeMap = databaseSpecific.columnTypeIds(); + for (Entry entry : columns().entrySet()) { + String col = entry.getKey(); + JdbcColumnType type = entry.getValue(); + ps.setNull(nameToIdx.apply(col), typeMap.get(type)); + } + } +} diff --git a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/ObjSerializers.java b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/ObjSerializers.java new file mode 100644 index 00000000000..16398c7ea14 --- /dev/null +++ b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/ObjSerializers.java @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.jdbc.serializers; + +import java.util.HashSet; +import java.util.Set; +import javax.annotation.Nonnull; +import org.projectnessie.versioned.storage.common.objtypes.StandardObjType; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjType; + +public final class ObjSerializers { + + public static final Set> ALL_SERIALIZERS = + Set.of( + CommitObjSerializer.INSTANCE, + ContentValueObjSerializer.INSTANCE, + IndexSegmentsObjSerializer.INSTANCE, + IndexObjSerializer.INSTANCE, + RefObjSerializer.INSTANCE, + StringObjSerializer.INSTANCE, + TagObjSerializer.INSTANCE, + CustomObjSerializer.INSTANCE); + + static { + Set columnNames = new HashSet<>(); + ALL_SERIALIZERS.forEach( + serializer -> { + for (String col : serializer.columns().keySet()) { + if (!columnNames.add(col)) { + throw new IllegalStateException("Duplicate column name: " + col); + } + } + }); + } + + @Nonnull + @jakarta.annotation.Nonnull + public static ObjSerializer forType(@Nonnull @jakarta.annotation.Nonnull ObjType type) { + ObjSerializer serializer = CustomObjSerializer.INSTANCE; + if (type instanceof StandardObjType) { + switch ((StandardObjType) type) { + case COMMIT: + serializer = CommitObjSerializer.INSTANCE; + break; + case INDEX_SEGMENTS: + serializer = IndexSegmentsObjSerializer.INSTANCE; + break; + case INDEX: + serializer = IndexObjSerializer.INSTANCE; + break; + case REF: + serializer = RefObjSerializer.INSTANCE; + break; + case STRING: + serializer = StringObjSerializer.INSTANCE; + break; + case TAG: + serializer = TagObjSerializer.INSTANCE; + break; + case VALUE: + serializer = ContentValueObjSerializer.INSTANCE; + break; + default: + throw new IllegalArgumentException("Unknown standard object type: " + type); + } + } + @SuppressWarnings("unchecked") + ObjSerializer cast = (ObjSerializer) serializer; + return cast; + } +} diff --git a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/RefObjSerializer.java b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/RefObjSerializer.java new file mode 100644 index 00000000000..9d111fdc281 --- /dev/null +++ b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/RefObjSerializer.java @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.jdbc.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.RefObj.ref; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.deserializeObjId; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.serializeObjId; + +import com.google.common.collect.ImmutableMap; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import java.util.function.Function; +import org.projectnessie.versioned.storage.common.objtypes.RefObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.jdbc.DatabaseSpecific; +import org.projectnessie.versioned.storage.jdbc.JdbcColumnType; + +public class RefObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new RefObjSerializer(); + + private static final String COL_REF_NAME = "r_name"; + private static final String COL_REF_INITIAL_POINTER = "r_initial_pointer"; + private static final String COL_REF_CREATED_AT = "r_created_at"; + private static final String COL_REF_EXTENDED_INFO = "r_extended_info"; + + private static final Map COLS = + ImmutableMap.of( + COL_REF_NAME, JdbcColumnType.NAME, + COL_REF_INITIAL_POINTER, JdbcColumnType.OBJ_ID, + COL_REF_CREATED_AT, JdbcColumnType.BIGINT, + COL_REF_EXTENDED_INFO, JdbcColumnType.OBJ_ID); + + private RefObjSerializer() {} + + @Override + public Map columns() { + return COLS; + } + + @Override + public void serialize( + PreparedStatement ps, + RefObj obj, + int incrementalIndexLimit, + int maxSerializedIndexSize, + Function nameToIdx, + DatabaseSpecific databaseSpecific) + throws SQLException { + ps.setString(nameToIdx.apply(COL_REF_NAME), obj.name()); + serializeObjId( + ps, nameToIdx.apply(COL_REF_INITIAL_POINTER), obj.initialPointer(), databaseSpecific); + ps.setLong(nameToIdx.apply(COL_REF_CREATED_AT), obj.createdAtMicros()); + serializeObjId( + ps, nameToIdx.apply(COL_REF_EXTENDED_INFO), obj.extendedInfoObj(), databaseSpecific); + } + + @Override + public RefObj deserialize(ResultSet rs, ObjId id) throws SQLException { + return ref( + id, + rs.getString(COL_REF_NAME), + deserializeObjId(rs, COL_REF_INITIAL_POINTER), + rs.getLong(COL_REF_CREATED_AT), + deserializeObjId(rs, COL_REF_EXTENDED_INFO)); + } +} diff --git a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/StringObjSerializer.java b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/StringObjSerializer.java new file mode 100644 index 00000000000..6b67c201adb --- /dev/null +++ b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/StringObjSerializer.java @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.jdbc.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.StringObj.stringData; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.deserializeBytes; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.deserializeObjIds; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.serializeBytes; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.serializeObjIds; + +import com.google.common.collect.ImmutableMap; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import java.util.function.Function; +import org.projectnessie.versioned.storage.common.objtypes.Compression; +import org.projectnessie.versioned.storage.common.objtypes.StringObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.jdbc.DatabaseSpecific; +import org.projectnessie.versioned.storage.jdbc.JdbcColumnType; + +public class StringObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new StringObjSerializer(); + + private static final String COL_STRING_CONTENT_TYPE = "s_content_type"; + private static final String COL_STRING_COMPRESSION = "s_compression"; + private static final String COL_STRING_FILENAME = "s_filename"; + private static final String COL_STRING_PREDECESSORS = "s_predecessors"; + private static final String COL_STRING_TEXT = "s_text"; + + private static final Map COLS = + ImmutableMap.of( + COL_STRING_CONTENT_TYPE, JdbcColumnType.NAME, + COL_STRING_COMPRESSION, JdbcColumnType.NAME, + COL_STRING_FILENAME, JdbcColumnType.NAME, + COL_STRING_PREDECESSORS, JdbcColumnType.OBJ_ID_LIST, + COL_STRING_TEXT, JdbcColumnType.VARBINARY); + + private StringObjSerializer() {} + + @Override + public Map columns() { + return COLS; + } + + @Override + public void serialize( + PreparedStatement ps, + StringObj obj, + int incrementalIndexLimit, + int maxSerializedIndexSize, + Function nameToIdx, + DatabaseSpecific databaseSpecific) + throws SQLException { + ps.setString(nameToIdx.apply(COL_STRING_CONTENT_TYPE), obj.contentType()); + ps.setString(nameToIdx.apply(COL_STRING_COMPRESSION), obj.compression().name()); + ps.setString(nameToIdx.apply(COL_STRING_FILENAME), obj.filename()); + serializeObjIds( + ps, nameToIdx.apply(COL_STRING_PREDECESSORS), obj.predecessors(), databaseSpecific); + serializeBytes(ps, nameToIdx.apply(COL_STRING_TEXT), obj.text(), databaseSpecific); + } + + @Override + public StringObj deserialize(ResultSet rs, ObjId id) throws SQLException { + return stringData( + id, + rs.getString(COL_STRING_CONTENT_TYPE), + Compression.valueOf(rs.getString(COL_STRING_COMPRESSION)), + rs.getString(COL_STRING_FILENAME), + deserializeObjIds(rs, COL_STRING_PREDECESSORS), + deserializeBytes(rs, COL_STRING_TEXT)); + } +} diff --git a/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/TagObjSerializer.java b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/TagObjSerializer.java new file mode 100644 index 00000000000..96d2c47f6e8 --- /dev/null +++ b/versioned/storage/jdbc/src/main/java/org/projectnessie/versioned/storage/jdbc/serializers/TagObjSerializer.java @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.jdbc.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.TagObj.tag; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.deserializeBytes; +import static org.projectnessie.versioned.storage.jdbc.JdbcSerde.serializeBytes; + +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Map; +import java.util.function.Function; +import org.projectnessie.versioned.storage.common.objtypes.CommitHeaders; +import org.projectnessie.versioned.storage.common.objtypes.TagObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.HeaderEntry; +import org.projectnessie.versioned.storage.common.proto.StorageTypes.Headers; +import org.projectnessie.versioned.storage.jdbc.DatabaseSpecific; +import org.projectnessie.versioned.storage.jdbc.JdbcColumnType; + +public class TagObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new TagObjSerializer(); + + private static final String COL_TAG_MESSAGE = "t_message"; + private static final String COL_TAG_HEADERS = "t_headers"; + private static final String COL_TAG_SIGNATURE = "t_signature"; + + private static final Map COLS = + ImmutableMap.of( + COL_TAG_MESSAGE, JdbcColumnType.VARCHAR, + COL_TAG_HEADERS, JdbcColumnType.VARBINARY, + COL_TAG_SIGNATURE, JdbcColumnType.VARBINARY); + + private TagObjSerializer() {} + + @Override + public Map columns() { + return COLS; + } + + @Override + public void serialize( + PreparedStatement ps, + TagObj obj, + int incrementalIndexLimit, + int maxSerializedIndexSize, + Function nameToIdx, + DatabaseSpecific databaseSpecific) + throws SQLException { + ps.setString(nameToIdx.apply(COL_TAG_MESSAGE), obj.message()); + Headers.Builder hb = Headers.newBuilder(); + CommitHeaders headers = obj.headers(); + if (headers != null) { + for (String h : headers.keySet()) { + hb.addHeaders(HeaderEntry.newBuilder().setName(h).addAllValues(headers.getAll(h))); + } + } + ps.setBytes(nameToIdx.apply(COL_TAG_HEADERS), hb.build().toByteArray()); + serializeBytes(ps, nameToIdx.apply(COL_TAG_SIGNATURE), obj.signature(), databaseSpecific); + } + + @Override + public TagObj deserialize(ResultSet rs, ObjId id) throws SQLException { + CommitHeaders tagHeaders = null; + try { + Headers headers = Headers.parseFrom(deserializeBytes(rs, COL_TAG_HEADERS)); + if (headers.getHeadersCount() > 0) { + CommitHeaders.Builder h = CommitHeaders.newCommitHeaders(); + for (HeaderEntry e : headers.getHeadersList()) { + for (String v : e.getValuesList()) { + h.add(e.getName(), v); + } + } + tagHeaders = h.build(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + + return tag( + id, rs.getString(COL_TAG_MESSAGE), tagHeaders, deserializeBytes(rs, COL_TAG_SIGNATURE)); + } +} diff --git a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/MongoDBConstants.java b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/MongoDBConstants.java index 54bedf5500d..012abaec536 100644 --- a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/MongoDBConstants.java +++ b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/MongoDBConstants.java @@ -33,53 +33,6 @@ final class MongoDBConstants { static final String COL_REPO = "r"; static final String COL_OBJ_TYPE = "y"; - static final String COL_COMMIT = "c"; - static final String COL_REF = "e"; - static final String COL_VALUE = "v"; - static final String COL_SEGMENTS = "I"; - static final String COL_INDEX = "i"; - static final String COL_TAG = "t"; - static final String COL_STRING = "s"; - - static final String COL_COMMIT_CREATED = "c"; - static final String COL_COMMIT_SEQ = "q"; - static final String COL_COMMIT_MESSAGE = "m"; - static final String COL_COMMIT_HEADERS = "h"; - static final String COL_COMMIT_REFERENCE_INDEX = "x"; - static final String COL_COMMIT_REFERENCE_INDEX_STRIPES = "r"; - static final String COL_COMMIT_TAIL = "t"; - static final String COL_COMMIT_SECONDARY_PARENTS = "s"; - static final String COL_COMMIT_INCREMENTAL_INDEX = "i"; - static final String COL_COMMIT_INCOMPLETE_INDEX = "n"; - static final String COL_COMMIT_TYPE = "y"; - - static final String COL_REF_NAME = "n"; - static final String COL_REF_INITIAL_POINTER = "p"; - static final String COL_REF_CREATED_AT = "c"; - static final String COL_REF_EXTENDED_INFO = "e"; - - static final String COL_VALUE_CONTENT_ID = "i"; - static final String COL_VALUE_PAYLOAD = "p"; - static final String COL_VALUE_DATA = "d"; - - static final String COL_SEGMENTS_STRIPES = "s"; - static final String COL_STRIPES_FIRST_KEY = "f"; - static final String COL_STRIPES_LAST_KEY = "l"; - static final String COL_STRIPES_SEGMENT = "s"; - - static final String COL_INDEX_INDEX = "i"; - - static final String COL_TAG_COMMIT_ID = "i"; - static final String COL_TAG_MESSAGE = "m"; - static final String COL_TAG_HEADERS = "h"; - static final String COL_TAG_SIGNATURE = "s"; - - static final String COL_STRING_CONTENT_TYPE = "y"; - static final String COL_STRING_COMPRESSION = "c"; - static final String COL_STRING_FILENAME = "f"; - static final String COL_STRING_PREDECESSORS = "p"; - static final String COL_STRING_TEXT = "t"; - static final String ID_REPO_PATH = ID_PROPERTY_NAME + "." + COL_REPO; static final String ID_OBJ_ID_PATH = ID_PROPERTY_NAME + "." + COL_OBJ_ID; static final String ID_REFERENCES_NAME_PATH = ID_PROPERTY_NAME + "." + COL_REFERENCES_NAME; diff --git a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/MongoDBPersist.java b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/MongoDBPersist.java index f80ccc98367..78a6228ef16 100644 --- a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/MongoDBPersist.java +++ b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/MongoDBPersist.java @@ -16,7 +16,6 @@ package org.projectnessie.versioned.storage.mongodb; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkState; import static com.mongodb.ErrorCategory.DUPLICATE_KEY; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -27,67 +26,20 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singleton; import static java.util.stream.Collectors.toList; -import static org.projectnessie.nessie.relocated.protobuf.UnsafeByteOperations.unsafeWrap; -import static org.projectnessie.versioned.storage.common.indexes.StoreKey.keyFromString; -import static org.projectnessie.versioned.storage.common.objtypes.CommitHeaders.newCommitHeaders; -import static org.projectnessie.versioned.storage.common.objtypes.CommitObj.commitBuilder; -import static org.projectnessie.versioned.storage.common.objtypes.ContentValueObj.contentValue; -import static org.projectnessie.versioned.storage.common.objtypes.IndexObj.index; -import static org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj.indexSegments; -import static org.projectnessie.versioned.storage.common.objtypes.IndexStripe.indexStripe; -import static org.projectnessie.versioned.storage.common.objtypes.RefObj.ref; -import static org.projectnessie.versioned.storage.common.objtypes.StringObj.stringData; -import static org.projectnessie.versioned.storage.common.objtypes.TagObj.tag; import static org.projectnessie.versioned.storage.common.persist.Reference.reference; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_COMMIT; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_COMMIT_CREATED; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_COMMIT_HEADERS; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_COMMIT_INCOMPLETE_INDEX; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_COMMIT_INCREMENTAL_INDEX; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_COMMIT_MESSAGE; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_COMMIT_REFERENCE_INDEX; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_COMMIT_REFERENCE_INDEX_STRIPES; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_COMMIT_SECONDARY_PARENTS; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_COMMIT_SEQ; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_COMMIT_TAIL; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_COMMIT_TYPE; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_INDEX; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_INDEX_INDEX; import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_OBJ_ID; import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_OBJ_TYPE; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_REF; import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_REFERENCES_CREATED_AT; import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_REFERENCES_DELETED; import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_REFERENCES_EXTENDED_INFO; import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_REFERENCES_NAME; import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_REFERENCES_POINTER; import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_REFERENCES_PREVIOUS; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_REF_CREATED_AT; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_REF_EXTENDED_INFO; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_REF_INITIAL_POINTER; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_REF_NAME; import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_REPO; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_SEGMENTS; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_SEGMENTS_STRIPES; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_STRING; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_STRING_COMPRESSION; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_STRING_CONTENT_TYPE; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_STRING_FILENAME; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_STRING_PREDECESSORS; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_STRING_TEXT; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_STRIPES_FIRST_KEY; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_STRIPES_LAST_KEY; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_STRIPES_SEGMENT; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_TAG; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_TAG_HEADERS; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_TAG_MESSAGE; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_TAG_SIGNATURE; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_VALUE; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_VALUE_CONTENT_ID; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_VALUE_DATA; -import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.COL_VALUE_PAYLOAD; import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.ID_PROPERTY_NAME; import static org.projectnessie.versioned.storage.mongodb.MongoDBConstants.ID_REPO_PATH; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.binaryToObjId; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.objIdToBinary; import static org.projectnessie.versioned.storage.serialize.ProtoSerialization.deserializePreviousPointers; import static org.projectnessie.versioned.storage.serialize.ProtoSerialization.serializePreviousPointers; @@ -107,12 +59,9 @@ import com.mongodb.client.result.UpdateResult; import java.util.ArrayList; import java.util.Arrays; -import java.util.EnumMap; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Consumer; import java.util.stream.Stream; import javax.annotation.Nonnull; import org.agrona.collections.Hashing; @@ -120,34 +69,23 @@ import org.bson.Document; import org.bson.conversions.Bson; import org.bson.types.Binary; -import org.projectnessie.nessie.relocated.protobuf.ByteString; import org.projectnessie.versioned.storage.common.config.StoreConfig; import org.projectnessie.versioned.storage.common.exceptions.ObjNotFoundException; import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; import org.projectnessie.versioned.storage.common.exceptions.RefAlreadyExistsException; import org.projectnessie.versioned.storage.common.exceptions.RefConditionFailedException; import org.projectnessie.versioned.storage.common.exceptions.RefNotFoundException; -import org.projectnessie.versioned.storage.common.objtypes.CommitHeaders; -import org.projectnessie.versioned.storage.common.objtypes.CommitObj; -import org.projectnessie.versioned.storage.common.objtypes.CommitType; -import org.projectnessie.versioned.storage.common.objtypes.Compression; -import org.projectnessie.versioned.storage.common.objtypes.ContentValueObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj; -import org.projectnessie.versioned.storage.common.objtypes.IndexStripe; -import org.projectnessie.versioned.storage.common.objtypes.RefObj; -import org.projectnessie.versioned.storage.common.objtypes.StringObj; -import org.projectnessie.versioned.storage.common.objtypes.TagObj; import org.projectnessie.versioned.storage.common.persist.CloseableIterator; import org.projectnessie.versioned.storage.common.persist.Obj; import org.projectnessie.versioned.storage.common.persist.ObjId; import org.projectnessie.versioned.storage.common.persist.ObjType; +import org.projectnessie.versioned.storage.common.persist.ObjTypes; import org.projectnessie.versioned.storage.common.persist.Persist; import org.projectnessie.versioned.storage.common.persist.Reference; +import org.projectnessie.versioned.storage.mongodb.serializers.ObjSerializer; +import org.projectnessie.versioned.storage.mongodb.serializers.ObjSerializers; public class MongoDBPersist implements Persist { - private static final Map> STORE_OBJ_TYPE = new EnumMap<>(ObjType.class); - private static final ObjType[] ALL_OBJ_TYPES = ObjType.values(); private final StoreConfig config; private final MongoDBBackend backend; @@ -189,49 +127,6 @@ private Document idObjDoc(ObjId id) { return idDoc; } - private static Binary bytesToBinary(ByteString bytes) { - return new Binary(bytes.toByteArray()); - } - - private static ByteString binaryToBytes(Binary binary) { - return binary != null ? unsafeWrap(binary.getData()) : null; - } - - private static Binary objIdToBinary(ObjId id) { - return new Binary(id.asByteArray()); - } - - private static ObjId binaryToObjId(Binary id) { - return id != null ? ObjId.objIdFromByteArray(id.getData()) : null; - } - - private static void objIdsToDoc(Document doc, String n, List ids) { - if (ids == null || ids.isEmpty()) { - return; - } - doc.put(n, objIdsToBinary(ids)); - } - - private static List objIdsToBinary(List ids) { - if (ids == null) { - return emptyList(); - } - return ids.stream().map(MongoDBPersist::objIdToBinary).collect(toList()); - } - - private static List binaryToObjIds(List ids) { - if (ids == null) { - return emptyList(); - } - return ids.stream().map(MongoDBPersist::binaryToObjId).collect(toList()); - } - - private static void binaryToObjIds(List ids, Consumer receiver) { - if (ids != null) { - ids.stream().map(MongoDBPersist::binaryToObjId).forEach(receiver); - } - } - @Nonnull @jakarta.annotation.Nonnull @Override @@ -460,7 +355,7 @@ public ObjType fetchObjType(@Nonnull @jakarta.annotation.Nonnull ObjId id) throw new ObjNotFoundException(id); } - return objTypeFromItem(doc); + return ObjTypes.forShortName(doc.getString(COL_OBJ_TYPE)); } @Nonnull @@ -579,26 +474,6 @@ private static ObjId objIdFromDoc(Document doc) { return binaryToObjId(doc.get(ID_PROPERTY_NAME, Document.class).get(COL_OBJ_ID, Binary.class)); } - private ObjType objTypeFromItem(Document doc) { - return objTypeFromItem(doc.getString(COL_OBJ_TYPE)); - } - - private ObjType objTypeFromItem(String shortType) { - for (ObjType type : ALL_OBJ_TYPES) { - if (type.shortName().equals(shortType)) { - return type; - } - } - throw new IllegalStateException("Cannot deserialize object short type " + shortType); - } - - private StoreObjDesc objTypeFromDoc(Document doc) { - ObjType type = objTypeFromItem(doc); - StoreObjDesc storeObj = STORE_OBJ_TYPE.get(type); - checkState(storeObj != null, "Cannot deserialize object type %s", type); - return storeObj; - } - private static int objIdIndex(Obj[] objs, ObjId id) { for (int i = 0; i < objs.length; i++) { if (id.equals(objs[i].id())) { @@ -693,12 +568,12 @@ private Obj docToObj(Document doc) { } private Obj docToObj(@Nonnull @jakarta.annotation.Nonnull ObjId id, Document doc) { - StoreObjDesc storeObj = objTypeFromDoc(doc); - Document inner = doc.get(storeObj.typeName, Document.class); - return storeObj.docToObj(id, inner); + ObjType type = ObjTypes.forShortName(doc.getString(COL_OBJ_TYPE)); + ObjSerializer serializer = ObjSerializers.forType(type); + Document inner = doc.get(serializer.fieldName(), Document.class); + return serializer.docToObj(id, inner); } - @SuppressWarnings("unchecked") private Document objToDoc( @Nonnull @jakarta.annotation.Nonnull Obj obj, boolean ignoreSoftSizeRestrictions) throws ObjTooLargeException { @@ -706,9 +581,7 @@ private Document objToDoc( checkArgument(id != null, "Obj to store must have a non-null ID"); ObjType type = obj.type(); - @SuppressWarnings("rawtypes") - StoreObjDesc storeObj = STORE_OBJ_TYPE.get(type); - checkArgument(storeObj != null, "Cannot serialize object type %s ", type); + ObjSerializer serializer = ObjSerializers.forType(type); Document doc = new Document(); Document inner = new Document(); @@ -718,303 +591,11 @@ private Document objToDoc( ignoreSoftSizeRestrictions ? Integer.MAX_VALUE : effectiveIncrementalIndexSizeLimit(); int indexSizeLimit = ignoreSoftSizeRestrictions ? Integer.MAX_VALUE : effectiveIndexSegmentSizeLimit(); - storeObj.objToDoc(obj, inner, incrementalIndexSizeLimit, indexSizeLimit); - doc.put(storeObj.typeName, inner); + serializer.objToDoc(obj, inner, incrementalIndexSizeLimit, indexSizeLimit); + doc.put(serializer.fieldName(), inner); return doc; } - abstract static class StoreObjDesc { - final String typeName; - - StoreObjDesc(String typeName) { - this.typeName = typeName; - } - - abstract void objToDoc( - O obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) - throws ObjTooLargeException; - - abstract O docToObj(ObjId id, Document doc); - } - - static { - STORE_OBJ_TYPE.put( - ObjType.COMMIT, - new StoreObjDesc(COL_COMMIT) { - @Override - void objToDoc( - CommitObj obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) - throws ObjTooLargeException { - doc.put(COL_COMMIT_SEQ, obj.seq()); - doc.put(COL_COMMIT_CREATED, obj.created()); - ObjId referenceIndex = obj.referenceIndex(); - if (referenceIndex != null) { - doc.put(COL_COMMIT_REFERENCE_INDEX, objIdToBinary(referenceIndex)); - } - doc.put(COL_COMMIT_MESSAGE, obj.message()); - objIdsToDoc(doc, COL_COMMIT_TAIL, obj.tail()); - objIdsToDoc(doc, COL_COMMIT_SECONDARY_PARENTS, obj.secondaryParents()); - - ByteString index = obj.incrementalIndex(); - if (index.size() > incrementalIndexLimit) { - throw new ObjTooLargeException(index.size(), incrementalIndexLimit); - } - doc.put(COL_COMMIT_INCREMENTAL_INDEX, bytesToBinary(index)); - - List indexStripes = obj.referenceIndexStripes(); - if (!indexStripes.isEmpty()) { - doc.put(COL_COMMIT_REFERENCE_INDEX_STRIPES, stripesToDocs(indexStripes)); - } - - Document headerDoc = new Document(); - CommitHeaders headers = obj.headers(); - for (String s : headers.keySet()) { - headerDoc.put(s, headers.getAll(s)); - } - if (!headerDoc.isEmpty()) { - doc.put(COL_COMMIT_HEADERS, headerDoc); - } - - doc.put(COL_COMMIT_INCOMPLETE_INDEX, obj.incompleteIndex()); - doc.put(COL_COMMIT_TYPE, obj.commitType().shortName()); - } - - @Override - CommitObj docToObj(ObjId id, Document doc) { - CommitObj.Builder b = - commitBuilder() - .id(id) - .seq(doc.getLong(COL_COMMIT_SEQ)) - .created(doc.getLong(COL_COMMIT_CREATED)) - .message(doc.getString(COL_COMMIT_MESSAGE)) - .incrementalIndex( - binaryToBytes(doc.get(COL_COMMIT_INCREMENTAL_INDEX, Binary.class))) - .incompleteIndex(doc.getBoolean(COL_COMMIT_INCOMPLETE_INDEX)) - .commitType(CommitType.fromShortName(doc.getString(COL_COMMIT_TYPE))); - Binary v = doc.get(COL_COMMIT_REFERENCE_INDEX, Binary.class); - if (v != null) { - b.referenceIndex(binaryToObjId(v)); - } - - fromStripesDocList( - doc, COL_COMMIT_REFERENCE_INDEX_STRIPES, b::addReferenceIndexStripes); - - binaryToObjIds(doc.getList(COL_COMMIT_TAIL, Binary.class), b::addTail); - binaryToObjIds( - doc.getList(COL_COMMIT_SECONDARY_PARENTS, Binary.class), b::addSecondaryParents); - - CommitHeaders.Builder headers = newCommitHeaders(); - Document headerDoc = doc.get(COL_COMMIT_HEADERS, Document.class); - if (headerDoc != null) { - headerDoc.forEach( - (k, o) -> { - @SuppressWarnings({"unchecked", "rawtypes"}) - List l = (List) o; - l.forEach(hv -> headers.add(k, hv)); - }); - } - b.headers(headers.build()); - - return b.build(); - } - }); - STORE_OBJ_TYPE.put( - ObjType.REF, - new StoreObjDesc(COL_REF) { - - @Override - void objToDoc( - RefObj obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) { - doc.put(COL_REF_NAME, obj.name()); - doc.put(COL_REF_CREATED_AT, obj.createdAtMicros()); - doc.put(COL_REF_INITIAL_POINTER, objIdToBinary(obj.initialPointer())); - ObjId extendedInfoObj = obj.extendedInfoObj(); - if (extendedInfoObj != null) { - doc.put(COL_REF_EXTENDED_INFO, objIdToBinary(extendedInfoObj)); - } - } - - @Override - RefObj docToObj(ObjId id, Document doc) { - return ref( - id, - doc.getString(COL_REF_NAME), - binaryToObjId(doc.get(COL_REF_INITIAL_POINTER, Binary.class)), - doc.getLong(COL_REF_CREATED_AT), - binaryToObjId(doc.get(COL_REF_EXTENDED_INFO, Binary.class))); - } - }); - STORE_OBJ_TYPE.put( - ObjType.VALUE, - new StoreObjDesc(COL_VALUE) { - @Override - void objToDoc( - ContentValueObj obj, - Document doc, - int incrementalIndexLimit, - int maxSerializedIndexSize) { - doc.put(COL_VALUE_CONTENT_ID, obj.contentId()); - doc.put(COL_VALUE_PAYLOAD, obj.payload()); - doc.put(COL_VALUE_DATA, bytesToBinary(obj.data())); - } - - @Override - ContentValueObj docToObj(ObjId id, Document doc) { - return contentValue( - id, - doc.getString(COL_VALUE_CONTENT_ID), - doc.getInteger(COL_VALUE_PAYLOAD), - binaryToBytes(doc.get(COL_VALUE_DATA, Binary.class))); - } - }); - STORE_OBJ_TYPE.put( - ObjType.INDEX_SEGMENTS, - new StoreObjDesc(COL_SEGMENTS) { - @Override - void objToDoc( - IndexSegmentsObj obj, - Document doc, - int incrementalIndexLimit, - int maxSerializedIndexSize) { - doc.put(COL_SEGMENTS_STRIPES, stripesToDocs(obj.stripes())); - } - - @Override - IndexSegmentsObj docToObj(ObjId id, Document doc) { - List stripes = new ArrayList<>(); - fromStripesDocList(doc, COL_SEGMENTS_STRIPES, stripes::add); - return indexSegments(id, stripes); - } - }); - STORE_OBJ_TYPE.put( - ObjType.INDEX, - new StoreObjDesc(COL_INDEX) { - @Override - void objToDoc( - IndexObj obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) - throws ObjTooLargeException { - ByteString index = obj.index(); - if (index.size() > maxSerializedIndexSize) { - throw new ObjTooLargeException(index.size(), maxSerializedIndexSize); - } - doc.put(COL_INDEX_INDEX, bytesToBinary(index)); - } - - @Override - IndexObj docToObj(ObjId id, Document doc) { - return index(id, binaryToBytes(doc.get(COL_INDEX_INDEX, Binary.class))); - } - }); - STORE_OBJ_TYPE.put( - ObjType.TAG, - new StoreObjDesc(COL_TAG) { - @Override - void objToDoc( - TagObj obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) { - String message = obj.message(); - if (message != null) { - doc.put(COL_TAG_MESSAGE, message); - } - - Document headerDoc = new Document(); - CommitHeaders headers = obj.headers(); - if (headers != null) { - for (String s : headers.keySet()) { - headerDoc.put(s, headers.getAll(s)); - } - if (!headerDoc.isEmpty()) { - doc.put(COL_TAG_HEADERS, headerDoc); - } - } - - ByteString signature = obj.signature(); - if (signature != null) { - doc.put(COL_TAG_SIGNATURE, bytesToBinary(signature)); - } - } - - @Override - TagObj docToObj(ObjId id, Document doc) { - CommitHeaders tagHeaders = null; - Document headerDoc = doc.get(COL_COMMIT_HEADERS, Document.class); - if (headerDoc != null) { - CommitHeaders.Builder headers = newCommitHeaders(); - headerDoc.forEach( - (k, o) -> { - @SuppressWarnings({"unchecked", "rawtypes"}) - List l = (List) o; - l.forEach(hv -> headers.add(k, hv)); - }); - tagHeaders = headers.build(); - } - - return tag( - id, - doc.getString(COL_TAG_MESSAGE), - tagHeaders, - binaryToBytes(doc.get(COL_TAG_SIGNATURE, Binary.class))); - } - }); - STORE_OBJ_TYPE.put( - ObjType.STRING, - new StoreObjDesc(COL_STRING) { - @Override - void objToDoc( - StringObj obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) { - String s = obj.contentType(); - if (s != null && !s.isEmpty()) { - doc.put(COL_STRING_CONTENT_TYPE, s); - } - doc.put(COL_STRING_COMPRESSION, obj.compression().name()); - s = obj.filename(); - if (s != null && !s.isEmpty()) { - doc.put(COL_STRING_FILENAME, s); - } - objIdsToDoc(doc, COL_STRING_PREDECESSORS, obj.predecessors()); - doc.put(COL_STRING_TEXT, bytesToBinary(obj.text())); - } - - @Override - StringObj docToObj(ObjId id, Document doc) { - return stringData( - id, - doc.getString(COL_STRING_CONTENT_TYPE), - Compression.valueOf(doc.getString(COL_STRING_COMPRESSION)), - doc.getString(COL_STRING_FILENAME), - binaryToObjIds(doc.getList(COL_STRING_PREDECESSORS, Binary.class)), - binaryToBytes(doc.get(COL_STRING_TEXT, Binary.class))); - } - }); - } - - private static void fromStripesDocList( - Document doc, String attrName, Consumer consumer) { - List refIndexStripes = doc.getList(attrName, Document.class); - if (refIndexStripes != null) { - for (Document seg : refIndexStripes) { - consumer.accept( - indexStripe( - keyFromString(seg.getString(COL_STRIPES_FIRST_KEY)), - keyFromString(seg.getString(COL_STRIPES_LAST_KEY)), - binaryToObjId(seg.get(COL_STRIPES_SEGMENT, Binary.class)))); - } - } - } - - @Nonnull - @jakarta.annotation.Nonnull - private static List stripesToDocs(List stripes) { - List stripesDocs = new ArrayList<>(); - for (IndexStripe stripe : stripes) { - Document sv = new Document(); - sv.put(COL_STRIPES_FIRST_KEY, stripe.firstKey().rawString()); - sv.put(COL_STRIPES_LAST_KEY, stripe.lastKey().rawString()); - sv.put(COL_STRIPES_SEGMENT, objIdToBinary(stripe.segment())); - stripesDocs.add(sv); - } - return stripesDocs; - } - private class ScanAllObjectsIterator extends AbstractIterator implements CloseableIterator { diff --git a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/MongoDBSerde.java b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/MongoDBSerde.java new file mode 100644 index 00000000000..6f81c7dd298 --- /dev/null +++ b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/MongoDBSerde.java @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.mongodb; + +import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; +import static org.projectnessie.nessie.relocated.protobuf.UnsafeByteOperations.unsafeWrap; +import static org.projectnessie.versioned.storage.common.indexes.StoreKey.keyFromString; +import static org.projectnessie.versioned.storage.common.objtypes.IndexStripe.indexStripe; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import javax.annotation.Nonnull; +import org.bson.Document; +import org.bson.types.Binary; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.common.objtypes.IndexStripe; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public final class MongoDBSerde { + + private static final String COL_STRIPES_FIRST_KEY = "f"; + private static final String COL_STRIPES_LAST_KEY = "l"; + private static final String COL_STRIPES_SEGMENT = "s"; + + private MongoDBSerde() {} + + public static Binary bytesToBinary(ByteString bytes) { + return new Binary(bytes.toByteArray()); + } + + public static ByteString binaryToBytes(Binary binary) { + return binary != null ? unsafeWrap(binary.getData()) : null; + } + + public static Binary objIdToBinary(ObjId id) { + return new Binary(id.asByteArray()); + } + + public static ObjId binaryToObjId(Binary id) { + return id != null ? ObjId.objIdFromByteArray(id.getData()) : null; + } + + public static void objIdsToDoc(Document doc, String n, List ids) { + if (ids == null || ids.isEmpty()) { + return; + } + doc.put(n, objIdsToBinary(ids)); + } + + public static List objIdsToBinary(List ids) { + if (ids == null) { + return emptyList(); + } + return ids.stream().map(MongoDBSerde::objIdToBinary).collect(toList()); + } + + public static List binaryToObjIds(List ids) { + if (ids == null) { + return emptyList(); + } + return ids.stream().map(MongoDBSerde::binaryToObjId).collect(toList()); + } + + public static void binaryToObjIds(List ids, Consumer receiver) { + if (ids != null) { + ids.stream().map(MongoDBSerde::binaryToObjId).forEach(receiver); + } + } + + public static void fromStripesDocList( + Document doc, String attrName, Consumer consumer) { + List refIndexStripes = doc.getList(attrName, Document.class); + if (refIndexStripes != null) { + for (Document seg : refIndexStripes) { + consumer.accept( + indexStripe( + keyFromString(seg.getString(COL_STRIPES_FIRST_KEY)), + keyFromString(seg.getString(COL_STRIPES_LAST_KEY)), + binaryToObjId(seg.get(COL_STRIPES_SEGMENT, Binary.class)))); + } + } + } + + @Nonnull + @jakarta.annotation.Nonnull + public static List stripesToDocs(List stripes) { + List stripesDocs = new ArrayList<>(); + for (IndexStripe stripe : stripes) { + Document sv = new Document(); + sv.put(COL_STRIPES_FIRST_KEY, stripe.firstKey().rawString()); + sv.put(COL_STRIPES_LAST_KEY, stripe.lastKey().rawString()); + sv.put(COL_STRIPES_SEGMENT, objIdToBinary(stripe.segment())); + stripesDocs.add(sv); + } + return stripesDocs; + } +} diff --git a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/CommitObjSerializer.java b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/CommitObjSerializer.java new file mode 100644 index 00000000000..7f6274ff3f5 --- /dev/null +++ b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/CommitObjSerializer.java @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.mongodb.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.CommitHeaders.newCommitHeaders; +import static org.projectnessie.versioned.storage.common.objtypes.CommitObj.commitBuilder; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.binaryToBytes; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.binaryToObjId; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.binaryToObjIds; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.bytesToBinary; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.fromStripesDocList; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.objIdToBinary; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.objIdsToDoc; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.stripesToDocs; + +import java.util.List; +import org.bson.Document; +import org.bson.types.Binary; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.objtypes.CommitHeaders; +import org.projectnessie.versioned.storage.common.objtypes.CommitObj; +import org.projectnessie.versioned.storage.common.objtypes.CommitType; +import org.projectnessie.versioned.storage.common.objtypes.IndexStripe; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public class CommitObjSerializer implements ObjSerializer { + + public static final CommitObjSerializer INSTANCE = new CommitObjSerializer(); + + private static final String COL_COMMIT = "c"; + + private static final String COL_COMMIT_CREATED = "c"; + private static final String COL_COMMIT_SEQ = "q"; + private static final String COL_COMMIT_MESSAGE = "m"; + private static final String COL_COMMIT_HEADERS = "h"; + private static final String COL_COMMIT_REFERENCE_INDEX = "x"; + private static final String COL_COMMIT_REFERENCE_INDEX_STRIPES = "r"; + private static final String COL_COMMIT_TAIL = "t"; + private static final String COL_COMMIT_SECONDARY_PARENTS = "s"; + private static final String COL_COMMIT_INCREMENTAL_INDEX = "i"; + private static final String COL_COMMIT_INCOMPLETE_INDEX = "n"; + private static final String COL_COMMIT_TYPE = "y"; + + private CommitObjSerializer() {} + + @Override + public String fieldName() { + return COL_COMMIT; + } + + @Override + public void objToDoc( + CommitObj obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) + throws ObjTooLargeException { + doc.put(COL_COMMIT_SEQ, obj.seq()); + doc.put(COL_COMMIT_CREATED, obj.created()); + ObjId referenceIndex = obj.referenceIndex(); + if (referenceIndex != null) { + doc.put(COL_COMMIT_REFERENCE_INDEX, objIdToBinary(referenceIndex)); + } + doc.put(COL_COMMIT_MESSAGE, obj.message()); + objIdsToDoc(doc, COL_COMMIT_TAIL, obj.tail()); + objIdsToDoc(doc, COL_COMMIT_SECONDARY_PARENTS, obj.secondaryParents()); + + ByteString index = obj.incrementalIndex(); + if (index.size() > incrementalIndexLimit) { + throw new ObjTooLargeException(index.size(), incrementalIndexLimit); + } + doc.put(COL_COMMIT_INCREMENTAL_INDEX, bytesToBinary(index)); + + List indexStripes = obj.referenceIndexStripes(); + if (!indexStripes.isEmpty()) { + doc.put(COL_COMMIT_REFERENCE_INDEX_STRIPES, stripesToDocs(indexStripes)); + } + + Document headerDoc = new Document(); + CommitHeaders headers = obj.headers(); + for (String s : headers.keySet()) { + headerDoc.put(s, headers.getAll(s)); + } + if (!headerDoc.isEmpty()) { + doc.put(COL_COMMIT_HEADERS, headerDoc); + } + + doc.put(COL_COMMIT_INCOMPLETE_INDEX, obj.incompleteIndex()); + doc.put(COL_COMMIT_TYPE, obj.commitType().shortName()); + } + + @Override + public CommitObj docToObj(ObjId id, Document doc) { + CommitObj.Builder b = + commitBuilder() + .id(id) + .seq(doc.getLong(COL_COMMIT_SEQ)) + .created(doc.getLong(COL_COMMIT_CREATED)) + .message(doc.getString(COL_COMMIT_MESSAGE)) + .incrementalIndex(binaryToBytes(doc.get(COL_COMMIT_INCREMENTAL_INDEX, Binary.class))) + .incompleteIndex(doc.getBoolean(COL_COMMIT_INCOMPLETE_INDEX)) + .commitType(CommitType.fromShortName(doc.getString(COL_COMMIT_TYPE))); + Binary v = doc.get(COL_COMMIT_REFERENCE_INDEX, Binary.class); + if (v != null) { + b.referenceIndex(binaryToObjId(v)); + } + + fromStripesDocList(doc, COL_COMMIT_REFERENCE_INDEX_STRIPES, b::addReferenceIndexStripes); + + binaryToObjIds(doc.getList(COL_COMMIT_TAIL, Binary.class), b::addTail); + binaryToObjIds(doc.getList(COL_COMMIT_SECONDARY_PARENTS, Binary.class), b::addSecondaryParents); + + CommitHeaders.Builder headers = newCommitHeaders(); + Document headerDoc = doc.get(COL_COMMIT_HEADERS, Document.class); + if (headerDoc != null) { + headerDoc.forEach( + (k, o) -> { + @SuppressWarnings({"unchecked", "rawtypes"}) + List l = (List) o; + l.forEach(hv -> headers.add(k, hv)); + }); + } + b.headers(headers.build()); + + return b.build(); + } +} diff --git a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/ContentValueObjSerializer.java b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/ContentValueObjSerializer.java new file mode 100644 index 00000000000..5e375590dc7 --- /dev/null +++ b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/ContentValueObjSerializer.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.mongodb.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.ContentValueObj.contentValue; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.binaryToBytes; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.bytesToBinary; + +import org.bson.Document; +import org.bson.types.Binary; +import org.projectnessie.versioned.storage.common.objtypes.ContentValueObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public class ContentValueObjSerializer implements ObjSerializer { + + public static final ContentValueObjSerializer INSTANCE = new ContentValueObjSerializer(); + + private static final String COL_VALUE = "v"; + + private static final String COL_VALUE_CONTENT_ID = "i"; + private static final String COL_VALUE_PAYLOAD = "p"; + private static final String COL_VALUE_DATA = "d"; + + private ContentValueObjSerializer() {} + + @Override + public String fieldName() { + return COL_VALUE; + } + + @Override + public void objToDoc( + ContentValueObj obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) { + doc.put(COL_VALUE_CONTENT_ID, obj.contentId()); + doc.put(COL_VALUE_PAYLOAD, obj.payload()); + doc.put(COL_VALUE_DATA, bytesToBinary(obj.data())); + } + + @Override + public ContentValueObj docToObj(ObjId id, Document doc) { + return contentValue( + id, + doc.getString(COL_VALUE_CONTENT_ID), + doc.getInteger(COL_VALUE_PAYLOAD), + binaryToBytes(doc.get(COL_VALUE_DATA, Binary.class))); + } +} diff --git a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/CustomObjSerializer.java b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/CustomObjSerializer.java new file mode 100644 index 00000000000..4d70585a12b --- /dev/null +++ b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/CustomObjSerializer.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.mongodb.serializers; + +import org.bson.Document; +import org.bson.types.Binary; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.serialize.SmileSerialization; + +public class CustomObjSerializer implements ObjSerializer { + + public static final ObjSerializer INSTANCE = new CustomObjSerializer(); + + private static final String COL_CUSTOM = "x"; + + private static final String COL_CUSTOM_CLASS = "x_class"; + private static final String COL_CUSTOM_DATA = "x_data"; + + private CustomObjSerializer() {} + + @Override + public String fieldName() { + return COL_CUSTOM; + } + + @Override + public void objToDoc(Obj obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) + throws ObjTooLargeException { + doc.put(COL_CUSTOM_CLASS, obj.type().targetClass().getName()); + doc.put(COL_CUSTOM_DATA, new Binary(SmileSerialization.serializeObj(obj))); + } + + @Override + public Obj docToObj(ObjId id, Document doc) { + byte[] data = doc.get(COL_CUSTOM_DATA, Binary.class).getData(); + return SmileSerialization.deserializeObj(id, data, doc.getString(COL_CUSTOM_CLASS)); + } +} diff --git a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/IndexObjSerializer.java b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/IndexObjSerializer.java new file mode 100644 index 00000000000..23b527b1159 --- /dev/null +++ b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/IndexObjSerializer.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.mongodb.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.IndexObj.index; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.binaryToBytes; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.bytesToBinary; + +import org.bson.Document; +import org.bson.types.Binary; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.objtypes.IndexObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public class IndexObjSerializer implements ObjSerializer { + + public static final IndexObjSerializer INSTANCE = new IndexObjSerializer(); + + private static final String COL_INDEX = "i"; + private static final String COL_INDEX_INDEX = "i"; + + private IndexObjSerializer() {} + + @Override + public String fieldName() { + return COL_INDEX; + } + + @Override + public void objToDoc( + IndexObj obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) + throws ObjTooLargeException { + ByteString index = obj.index(); + if (index.size() > maxSerializedIndexSize) { + throw new ObjTooLargeException(index.size(), maxSerializedIndexSize); + } + doc.put(COL_INDEX_INDEX, bytesToBinary(index)); + } + + @Override + public IndexObj docToObj(ObjId id, Document doc) { + return index(id, binaryToBytes(doc.get(COL_INDEX_INDEX, Binary.class))); + } +} diff --git a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/IndexSegmentsObjSerializer.java b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/IndexSegmentsObjSerializer.java new file mode 100644 index 00000000000..bbcbb79ed3d --- /dev/null +++ b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/IndexSegmentsObjSerializer.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.mongodb.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj.indexSegments; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.fromStripesDocList; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.stripesToDocs; + +import java.util.ArrayList; +import java.util.List; +import org.bson.Document; +import org.projectnessie.versioned.storage.common.objtypes.IndexSegmentsObj; +import org.projectnessie.versioned.storage.common.objtypes.IndexStripe; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public class IndexSegmentsObjSerializer implements ObjSerializer { + + public static final IndexSegmentsObjSerializer INSTANCE = new IndexSegmentsObjSerializer(); + + private static final String COL_SEGMENTS = "I"; + private static final String COL_SEGMENTS_STRIPES = "s"; + + private IndexSegmentsObjSerializer() {} + + @Override + public String fieldName() { + return COL_SEGMENTS; + } + + @Override + public void objToDoc( + IndexSegmentsObj obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) { + doc.put(COL_SEGMENTS_STRIPES, stripesToDocs(obj.stripes())); + } + + @Override + public IndexSegmentsObj docToObj(ObjId id, Document doc) { + List stripes = new ArrayList<>(); + fromStripesDocList(doc, COL_SEGMENTS_STRIPES, stripes::add); + return indexSegments(id, stripes); + } +} diff --git a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/ObjSerializer.java b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/ObjSerializer.java new file mode 100644 index 00000000000..f6c49cc4f52 --- /dev/null +++ b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/ObjSerializer.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.mongodb.serializers; + +import org.bson.Document; +import org.projectnessie.versioned.storage.common.exceptions.ObjTooLargeException; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjId; +import org.projectnessie.versioned.storage.common.persist.ObjType; + +public interface ObjSerializer { + + /** + * The name of the document's top-level field that will contain the serialized object. Note that + * this is generally equal to {@link ObjType#shortName()}, but not always, for historical reasons. + */ + String fieldName(); + + void objToDoc(O obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) + throws ObjTooLargeException; + + O docToObj(ObjId id, Document doc); +} diff --git a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/ObjSerializers.java b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/ObjSerializers.java new file mode 100644 index 00000000000..8d979cdc522 --- /dev/null +++ b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/ObjSerializers.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.mongodb.serializers; + +import static java.util.Objects.requireNonNull; + +import java.util.HashSet; +import java.util.Set; +import javax.annotation.Nonnull; +import org.projectnessie.versioned.storage.common.objtypes.StandardObjType; +import org.projectnessie.versioned.storage.common.persist.Obj; +import org.projectnessie.versioned.storage.common.persist.ObjType; + +public final class ObjSerializers { + + public static final Set> ALL_SERIALIZERS = + Set.of( + CommitObjSerializer.INSTANCE, + ContentValueObjSerializer.INSTANCE, + IndexSegmentsObjSerializer.INSTANCE, + IndexObjSerializer.INSTANCE, + RefObjSerializer.INSTANCE, + StringObjSerializer.INSTANCE, + TagObjSerializer.INSTANCE, + CustomObjSerializer.INSTANCE); + + static { + Set payloadFieldNames = new HashSet<>(); + ALL_SERIALIZERS.forEach( + serializer -> { + String fieldName = requireNonNull(serializer.fieldName()); + if (!payloadFieldNames.add(fieldName)) { + throw new IllegalStateException("Duplicate field name: " + fieldName); + } + }); + } + + @Nonnull + @jakarta.annotation.Nonnull + public static ObjSerializer forType(@Nonnull @jakarta.annotation.Nonnull ObjType type) { + ObjSerializer serializer = CustomObjSerializer.INSTANCE; + if (type instanceof StandardObjType) { + switch ((StandardObjType) type) { + case COMMIT: + serializer = CommitObjSerializer.INSTANCE; + break; + case INDEX_SEGMENTS: + serializer = IndexSegmentsObjSerializer.INSTANCE; + break; + case INDEX: + serializer = IndexObjSerializer.INSTANCE; + break; + case REF: + serializer = RefObjSerializer.INSTANCE; + break; + case STRING: + serializer = StringObjSerializer.INSTANCE; + break; + case TAG: + serializer = TagObjSerializer.INSTANCE; + break; + case VALUE: + serializer = ContentValueObjSerializer.INSTANCE; + break; + default: + throw new IllegalArgumentException("Unknown standard object type: " + type); + } + } + @SuppressWarnings("unchecked") + ObjSerializer cast = (ObjSerializer) serializer; + return cast; + } +} diff --git a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/RefObjSerializer.java b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/RefObjSerializer.java new file mode 100644 index 00000000000..52c70f46a79 --- /dev/null +++ b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/RefObjSerializer.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.mongodb.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.RefObj.ref; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.binaryToObjId; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.objIdToBinary; + +import org.bson.Document; +import org.bson.types.Binary; +import org.projectnessie.versioned.storage.common.objtypes.RefObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public class RefObjSerializer implements ObjSerializer { + + public static final RefObjSerializer INSTANCE = new RefObjSerializer(); + + private static final String COL_REF = "e"; + + private static final String COL_REF_NAME = "n"; + private static final String COL_REF_INITIAL_POINTER = "p"; + private static final String COL_REF_CREATED_AT = "c"; + private static final String COL_REF_EXTENDED_INFO = "e"; + + private RefObjSerializer() {} + + @Override + public String fieldName() { + return COL_REF; + } + + @Override + public void objToDoc( + RefObj obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) { + doc.put(COL_REF_NAME, obj.name()); + doc.put(COL_REF_CREATED_AT, obj.createdAtMicros()); + doc.put(COL_REF_INITIAL_POINTER, objIdToBinary(obj.initialPointer())); + ObjId extendedInfoObj = obj.extendedInfoObj(); + if (extendedInfoObj != null) { + doc.put(COL_REF_EXTENDED_INFO, objIdToBinary(extendedInfoObj)); + } + } + + @Override + public RefObj docToObj(ObjId id, Document doc) { + return ref( + id, + doc.getString(COL_REF_NAME), + binaryToObjId(doc.get(COL_REF_INITIAL_POINTER, Binary.class)), + doc.getLong(COL_REF_CREATED_AT), + binaryToObjId(doc.get(COL_REF_EXTENDED_INFO, Binary.class))); + } +} diff --git a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/StringObjSerializer.java b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/StringObjSerializer.java new file mode 100644 index 00000000000..6654334b87e --- /dev/null +++ b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/StringObjSerializer.java @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.mongodb.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.StringObj.stringData; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.binaryToBytes; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.binaryToObjIds; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.bytesToBinary; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.objIdsToDoc; + +import org.bson.Document; +import org.bson.types.Binary; +import org.projectnessie.versioned.storage.common.objtypes.Compression; +import org.projectnessie.versioned.storage.common.objtypes.StringObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public class StringObjSerializer implements ObjSerializer { + + public static final StringObjSerializer INSTANCE = new StringObjSerializer(); + + private static final String COL_STRING = "s"; + + private static final String COL_STRING_CONTENT_TYPE = "y"; + private static final String COL_STRING_COMPRESSION = "c"; + private static final String COL_STRING_FILENAME = "f"; + private static final String COL_STRING_PREDECESSORS = "p"; + private static final String COL_STRING_TEXT = "t"; + + private StringObjSerializer() {} + + @Override + public String fieldName() { + return COL_STRING; + } + + @Override + public void objToDoc( + StringObj obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) { + String s = obj.contentType(); + if (s != null && !s.isEmpty()) { + doc.put(COL_STRING_CONTENT_TYPE, s); + } + doc.put(COL_STRING_COMPRESSION, obj.compression().name()); + s = obj.filename(); + if (s != null && !s.isEmpty()) { + doc.put(COL_STRING_FILENAME, s); + } + objIdsToDoc(doc, COL_STRING_PREDECESSORS, obj.predecessors()); + doc.put(COL_STRING_TEXT, bytesToBinary(obj.text())); + } + + @Override + public StringObj docToObj(ObjId id, Document doc) { + return stringData( + id, + doc.getString(COL_STRING_CONTENT_TYPE), + Compression.valueOf(doc.getString(COL_STRING_COMPRESSION)), + doc.getString(COL_STRING_FILENAME), + binaryToObjIds(doc.getList(COL_STRING_PREDECESSORS, Binary.class)), + binaryToBytes(doc.get(COL_STRING_TEXT, Binary.class))); + } +} diff --git a/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/TagObjSerializer.java b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/TagObjSerializer.java new file mode 100644 index 00000000000..62f8c5cfbf7 --- /dev/null +++ b/versioned/storage/mongodb/src/main/java/org/projectnessie/versioned/storage/mongodb/serializers/TagObjSerializer.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2023 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.versioned.storage.mongodb.serializers; + +import static org.projectnessie.versioned.storage.common.objtypes.CommitHeaders.newCommitHeaders; +import static org.projectnessie.versioned.storage.common.objtypes.TagObj.tag; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.binaryToBytes; +import static org.projectnessie.versioned.storage.mongodb.MongoDBSerde.bytesToBinary; + +import java.util.List; +import org.bson.Document; +import org.bson.types.Binary; +import org.projectnessie.nessie.relocated.protobuf.ByteString; +import org.projectnessie.versioned.storage.common.objtypes.CommitHeaders; +import org.projectnessie.versioned.storage.common.objtypes.TagObj; +import org.projectnessie.versioned.storage.common.persist.ObjId; + +public class TagObjSerializer implements ObjSerializer { + + public static final TagObjSerializer INSTANCE = new TagObjSerializer(); + + private static final String COL_TAG = "t"; + + private static final String COL_TAG_MESSAGE = "m"; + private static final String COL_TAG_HEADERS = "h"; + private static final String COL_TAG_SIGNATURE = "s"; + + private TagObjSerializer() {} + + @Override + public String fieldName() { + return COL_TAG; + } + + @Override + public void objToDoc( + TagObj obj, Document doc, int incrementalIndexLimit, int maxSerializedIndexSize) { + String message = obj.message(); + if (message != null) { + doc.put(COL_TAG_MESSAGE, message); + } + + Document headerDoc = new Document(); + CommitHeaders headers = obj.headers(); + if (headers != null) { + for (String s : headers.keySet()) { + headerDoc.put(s, headers.getAll(s)); + } + if (!headerDoc.isEmpty()) { + doc.put(COL_TAG_HEADERS, headerDoc); + } + } + + ByteString signature = obj.signature(); + if (signature != null) { + doc.put(COL_TAG_SIGNATURE, bytesToBinary(signature)); + } + } + + @Override + public TagObj docToObj(ObjId id, Document doc) { + CommitHeaders tagHeaders = null; + Document headerDoc = doc.get(COL_TAG_HEADERS, Document.class); + if (headerDoc != null) { + CommitHeaders.Builder headers = newCommitHeaders(); + headerDoc.forEach( + (k, o) -> { + @SuppressWarnings({"unchecked", "rawtypes"}) + List l = (List) o; + l.forEach(hv -> headers.add(k, hv)); + }); + tagHeaders = headers.build(); + } + + return tag( + id, + doc.getString(COL_TAG_MESSAGE), + tagHeaders, + binaryToBytes(doc.get(COL_TAG_SIGNATURE, Binary.class))); + } +} diff --git a/versioned/storage/rocksdb/src/main/java/org/projectnessie/versioned/storage/rocksdb/RocksDBPersist.java b/versioned/storage/rocksdb/src/main/java/org/projectnessie/versioned/storage/rocksdb/RocksDBPersist.java index ed984a341be..05f246dbc9f 100644 --- a/versioned/storage/rocksdb/src/main/java/org/projectnessie/versioned/storage/rocksdb/RocksDBPersist.java +++ b/versioned/storage/rocksdb/src/main/java/org/projectnessie/versioned/storage/rocksdb/RocksDBPersist.java @@ -290,7 +290,7 @@ public T fetchTypedObj( @Nonnull @jakarta.annotation.Nonnull ObjId id, ObjType type, Class typeClass) throws ObjNotFoundException { Obj obj = fetchObj(id); - if (obj.type() != type) { + if (!obj.type().equals(type)) { throw new ObjNotFoundException(id); } @SuppressWarnings("unchecked") diff --git a/versioned/storage/store/src/main/java/org/projectnessie/versioned/storage/versionstore/ContentMapping.java b/versioned/storage/store/src/main/java/org/projectnessie/versioned/storage/versionstore/ContentMapping.java index 1a314ac2b8a..9d62d73bdb9 100644 --- a/versioned/storage/store/src/main/java/org/projectnessie/versioned/storage/versionstore/ContentMapping.java +++ b/versioned/storage/store/src/main/java/org/projectnessie/versioned/storage/versionstore/ContentMapping.java @@ -20,7 +20,7 @@ import static java.util.Objects.requireNonNull; import static org.projectnessie.versioned.storage.common.logic.Logics.indexesLogic; import static org.projectnessie.versioned.storage.common.objtypes.ContentValueObj.contentValue; -import static org.projectnessie.versioned.storage.common.persist.ObjType.VALUE; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.VALUE; import static org.projectnessie.versioned.storage.versionstore.TypeMapping.keyToStoreKey; import static org.projectnessie.versioned.storage.versionstore.TypeMapping.objIdToHash; import static org.projectnessie.versioned.storage.versionstore.TypeMapping.storeKeyToKey; diff --git a/versioned/storage/store/src/main/java/org/projectnessie/versioned/storage/versionstore/VersionStoreImpl.java b/versioned/storage/store/src/main/java/org/projectnessie/versioned/storage/versionstore/VersionStoreImpl.java index 88d8a701194..68ddcedaaba 100644 --- a/versioned/storage/store/src/main/java/org/projectnessie/versioned/storage/versionstore/VersionStoreImpl.java +++ b/versioned/storage/store/src/main/java/org/projectnessie/versioned/storage/versionstore/VersionStoreImpl.java @@ -34,8 +34,8 @@ import static org.projectnessie.versioned.storage.common.logic.PagingToken.fromString; import static org.projectnessie.versioned.storage.common.logic.PagingToken.pagingToken; import static org.projectnessie.versioned.storage.common.logic.ReferencesQuery.referencesQuery; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.COMMIT; import static org.projectnessie.versioned.storage.common.persist.ObjId.EMPTY_OBJ_ID; -import static org.projectnessie.versioned.storage.common.persist.ObjType.COMMIT; import static org.projectnessie.versioned.storage.common.persist.Reference.reference; import static org.projectnessie.versioned.storage.versionstore.BaseCommitHelper.committingOperation; import static org.projectnessie.versioned.storage.versionstore.BaseCommitHelper.dryRunCommitterSupplier; diff --git a/versioned/storage/store/src/test/java/org/projectnessie/versioned/storage/versionstore/TestRefMapping.java b/versioned/storage/store/src/test/java/org/projectnessie/versioned/storage/versionstore/TestRefMapping.java index 4460f2513c9..91020d84e22 100644 --- a/versioned/storage/store/src/test/java/org/projectnessie/versioned/storage/versionstore/TestRefMapping.java +++ b/versioned/storage/store/src/test/java/org/projectnessie/versioned/storage/versionstore/TestRefMapping.java @@ -39,10 +39,10 @@ import static org.projectnessie.versioned.storage.common.objtypes.CommitHeaders.EMPTY_COMMIT_HEADERS; import static org.projectnessie.versioned.storage.common.objtypes.CommitHeaders.newCommitHeaders; import static org.projectnessie.versioned.storage.common.objtypes.CommitObj.commitBuilder; +import static org.projectnessie.versioned.storage.common.objtypes.StandardObjType.COMMIT; import static org.projectnessie.versioned.storage.common.persist.ObjId.EMPTY_OBJ_ID; import static org.projectnessie.versioned.storage.common.persist.ObjId.objIdFromString; import static org.projectnessie.versioned.storage.common.persist.ObjId.randomObjId; -import static org.projectnessie.versioned.storage.common.persist.ObjType.COMMIT; import static org.projectnessie.versioned.storage.common.persist.Reference.reference; import static org.projectnessie.versioned.storage.versionstore.RefMapping.NO_ANCESTOR; import static org.projectnessie.versioned.storage.versionstore.RefMapping.REFS_HEADS; diff --git a/versioned/transfer/src/test/java/org/projectnessie/versioned/transfer/TestExportImportV2.java b/versioned/transfer/src/test/java/org/projectnessie/versioned/transfer/TestExportImportV2.java index a1336995780..69ba9fdce3d 100644 --- a/versioned/transfer/src/test/java/org/projectnessie/versioned/transfer/TestExportImportV2.java +++ b/versioned/transfer/src/test/java/org/projectnessie/versioned/transfer/TestExportImportV2.java @@ -20,7 +20,7 @@ import static org.projectnessie.versioned.storage.common.logic.Logics.repositoryLogic; import java.io.IOException; -import java.util.EnumSet; +import java.util.Set; import java.util.Spliterators; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -30,9 +30,9 @@ import org.projectnessie.versioned.storage.common.logic.RepositoryDescription; import org.projectnessie.versioned.storage.common.objtypes.CommitObj; import org.projectnessie.versioned.storage.common.objtypes.CommitType; +import org.projectnessie.versioned.storage.common.objtypes.StandardObjType; import org.projectnessie.versioned.storage.common.persist.CloseableIterator; import org.projectnessie.versioned.storage.common.persist.Obj; -import org.projectnessie.versioned.storage.common.persist.ObjType; import org.projectnessie.versioned.storage.common.persist.Persist; import org.projectnessie.versioned.storage.inmemory.InmemoryBackendFactory; import org.projectnessie.versioned.storage.testextension.NessieBackendName; @@ -101,7 +101,7 @@ ExportMeta exportRepo(boolean fullScan) throws IOException { @Override Stream scanAllTargetCommits() { - CloseableIterator iter = persistImport.scanAllObjects(EnumSet.of(ObjType.COMMIT)); + CloseableIterator iter = persistImport.scanAllObjects(Set.of(StandardObjType.COMMIT)); return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iter, 0), false) .onClose(iter::close) .map(CommitObj.class::cast) diff --git a/versioned/transfer/src/test/java/org/projectnessie/versioned/transfer/TestMigrationFromDatabaseAdapterToPersist.java b/versioned/transfer/src/test/java/org/projectnessie/versioned/transfer/TestMigrationFromDatabaseAdapterToPersist.java index d9eb97469e0..993d6908707 100644 --- a/versioned/transfer/src/test/java/org/projectnessie/versioned/transfer/TestMigrationFromDatabaseAdapterToPersist.java +++ b/versioned/transfer/src/test/java/org/projectnessie/versioned/transfer/TestMigrationFromDatabaseAdapterToPersist.java @@ -20,7 +20,7 @@ import com.google.errorprone.annotations.MustBeClosed; import java.io.IOException; -import java.util.EnumSet; +import java.util.Set; import java.util.Spliterators; import java.util.stream.Stream; import java.util.stream.StreamSupport; @@ -37,9 +37,9 @@ import org.projectnessie.versioned.storage.common.logic.RepositoryDescription; import org.projectnessie.versioned.storage.common.objtypes.CommitObj; import org.projectnessie.versioned.storage.common.objtypes.CommitType; +import org.projectnessie.versioned.storage.common.objtypes.StandardObjType; import org.projectnessie.versioned.storage.common.persist.CloseableIterator; import org.projectnessie.versioned.storage.common.persist.Obj; -import org.projectnessie.versioned.storage.common.persist.ObjType; import org.projectnessie.versioned.storage.common.persist.Persist; import org.projectnessie.versioned.storage.inmemory.InmemoryBackendFactory; import org.projectnessie.versioned.storage.testextension.NessieBackendName; @@ -106,7 +106,7 @@ ExportMeta exportRepo(boolean fullScan) throws IOException { @Override @MustBeClosed Stream scanAllTargetCommits() { - CloseableIterator iter = persist.scanAllObjects(EnumSet.of(ObjType.COMMIT)); + CloseableIterator iter = persist.scanAllObjects(Set.of(StandardObjType.COMMIT)); return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iter, 0), false) .onClose(iter::close) .map(CommitObj.class::cast)