From 4fbeb9dba890e24bc1fdd5b02bece0428d140b0c Mon Sep 17 00:00:00 2001 From: Matus Faro Date: Mon, 28 Oct 2024 02:04:04 -0400 Subject: [PATCH] Improved request creation --- README.md | 171 +++----- .../singletable/DynamoMapperImpl.java | 403 +++--------------- .../io/dataspray/singletable/DynamoUtil.java | 4 +- .../singletable/ExpressionBuilder.java | 335 +++++++++++++-- .../java/io/dataspray/singletable/Schema.java | 19 +- .../io/dataspray/singletable/TableSchema.java | 12 +- .../ConditionAndFilterExpressionBuilder.java | 16 + .../builder/ConditionExpressionBuilder.java | 9 + .../singletable/builder/DeleteBuilder.java | 33 ++ .../singletable/{ => builder}/Expression.java | 10 +- .../builder/FilterExpressionBuilder.java | 8 + .../singletable/builder/GetBuilder.java | 48 +++ .../builder/MappingExpression.java | 6 + .../singletable/builder/Mappings.java | 18 + .../singletable/builder/PutBuilder.java | 50 +++ .../singletable/builder/QueryBuilder.java | 50 +++ .../singletable/builder/UpdateBuilder.java | 62 +++ .../builder/UpdateExpressionBuilder.java | 32 ++ .../singletable/DynamoExpressionTest.java | 28 +- .../io/dataspray/singletable/ReadmeTest.java | 221 ++++++++++ 20 files changed, 1007 insertions(+), 528 deletions(-) create mode 100644 single-table/src/main/java/io/dataspray/singletable/builder/ConditionAndFilterExpressionBuilder.java create mode 100644 single-table/src/main/java/io/dataspray/singletable/builder/ConditionExpressionBuilder.java create mode 100644 single-table/src/main/java/io/dataspray/singletable/builder/DeleteBuilder.java rename single-table/src/main/java/io/dataspray/singletable/{ => builder}/Expression.java (60%) create mode 100644 single-table/src/main/java/io/dataspray/singletable/builder/FilterExpressionBuilder.java create mode 100644 single-table/src/main/java/io/dataspray/singletable/builder/GetBuilder.java create mode 100644 single-table/src/main/java/io/dataspray/singletable/builder/MappingExpression.java create mode 100644 single-table/src/main/java/io/dataspray/singletable/builder/Mappings.java create mode 100644 single-table/src/main/java/io/dataspray/singletable/builder/PutBuilder.java create mode 100644 single-table/src/main/java/io/dataspray/singletable/builder/QueryBuilder.java create mode 100644 single-table/src/main/java/io/dataspray/singletable/builder/UpdateBuilder.java create mode 100644 single-table/src/main/java/io/dataspray/singletable/builder/UpdateExpressionBuilder.java create mode 100644 single-table/src/test/java/io/dataspray/singletable/ReadmeTest.java diff --git a/README.md b/README.md index cc8d05d..674048a 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ It's as simple as: @Value @DynamoTable(type = Primary, partitionKeys = "accountId", rangePrefix = "account") @DynamoTable(type = Gsi, indexNumber = 1, partitionKeys = {"apiKey"}, rangePrefix = "accountByApiKey") -@DynamoTable(type = Gsi, indexNumber = 2, partitionKeys = {"oauthGuid"}, rangePrefix = "accountByOauthGuid") +@DynamoTable(type = Gsi, indexNumber = 2, partitionKeys = {"email"}, rangePrefix = "accountByEmail") class Account { @NonNull String accountId; @@ -26,17 +26,17 @@ class Account { // Initialize schema SingleTable singleTable = SingleTable.builder() - .dynamoDoc(dynamoDoc) .tablePrefix("project").build(); TableSchema schema = singleTable.parseTableSchema(Account.class); // Insert new account Account account = new Account("8426", "matus@example.com", null); -schema.table().putItem(new PutItemSpec().withItem(schema.toItem(account))); +schema.put().item(account).execute(client); // Fetch other account -Optional otherAccountOpt = Optional.ofNullable(schema.fromItem(schema.table().getItem( - schema.primaryKey(Map.of("accountId","9473"))))); +Optional otherAccountOpt = Optional.ofNullable(schema.fromAttrMap( + dynamo.getItem(schema.get().key(schema.primaryKey(Map + .of("accountId", "9473"))).build()).item())); ``` Okay, it could be simpler... @@ -99,7 +99,7 @@ Note you need to indicate how many LSIs and GSIs you would like to create. This your schemas. But don't worry you can always add more later. ```java -singleTable.createTableIfNotExists(2, 2); +singleTable.createTableIfNotExists(dynamo, 2, 2); ``` #### Via CDK @@ -113,69 +113,39 @@ singleTable.createCdkTable(this, "my-stack-name", 2, 2); ### Insert an item ```java -client.putItem(PutItemRequest.builder() - .tableName(schema.tableName()) - .item(schema.toAttrMap(myAccount)) - .build()) +schema.put() + .item(account) + .execute(client); ``` -### Update and Condition expressions builder +### Put with condition builder ```java -ExpressionBuilder expressionBuilder = schema.expressionBuilder(); - -// Apply conditions -expressionBuilder - // Item exists - .conditionExists() - // Particular field exists - .conditionFieldExists("cancelDate") - // Particular field equals a value - .conditionFieldEquals("isCancelled", false); - -// Modify data -expressionBuilder - // Overwrite field - .set("apiKey", apiKey) - // Increment field value - .setIncrement("votersCount", 1); - // Add to a set - .add("transactionIds", ImmutableSet.of("4234", "5312")) - // Remove entry from a json field - .remove(ImmutableList.of("entryJson", entryId, "isMoved")); - -Expression expression = expressionBuilder.build(); - -// For PUTs -client.updateItem(expression.toUpdateItemRequestBuilder() - .key(schema.primaryKey(expectedData)) - .build()); - -// For other requests -expression.updateExpression().ifPresent(builder::updateExpression); -expression.conditionExpression().ifPresent(builder::conditionExpression); -expression.expressionAttributeNames().ifPresent(builder::expressionAttributeNames); -expression.expressionAttributeValues().ifPresent(builder::expressionAttributeValues); +Optional updatedAccountOpt = schema.update() + .key(Map.of("accountId", "12345")) + + // Apply conditions + .conditionExists() + .conditionFieldExists("cancelDate") + .conditionFieldEquals("isCancelled", false) + + // Modify data + .set("apiKey", apiKey) + .setIncrement("votersCount", 1) + // Add to a set + .add("transactionIds", ImmutableSet.of("4234", "5312")) + // Remove entry from a json field + .remove(ImmutableList.of("entryJson", entryId, "isMoved")) + + .execute(client); ``` -### Select an item +### Get an item ```java -Account account = schema.fromAttrMap(client.getItem(b -> b - .tableName(schema.tableName()) - .key(schema.primaryKey(Map.of( - "accountId","account-id-123")))) - .item()); -``` - -You may want to wrap it in an optional if you prefer not to work with nulls: - -```java -Optional accountOpt = Optional.ofNullable(schema.fromAttrMap(client.getItem(b -> b - .tableName(schema.tableName()) - .key(schema.primaryKey(Map.of( - "accountId","account-id-123")))) - .item())); +Optional accountOpt = schema.get() + .key(Map.of("accountId", "12345")) + .execute(client); ``` ### Query ranges with paging @@ -189,12 +159,11 @@ using `withExclusiveStartKey` to continue quering where we left off. Optional cursor = Optional.empty(); do { // Prepare request - QueryRequest.Builder builder = QueryRequest.builder() - .tableName(schema.tableName()) - // Query by partition key - .keyConditions(schema.attrMapToConditions(schema.partitionKey(Map.of( - "partitionKey", partitionKey)))) - .limit(2); + QueryRequest.Builder builder = schema.query() + // Query by partition key + .keyConditions(Map.of("partitionKey", "partitionKey")) + .builder() + .limit(2); cursor.ifPresent(exclusiveStartKey -> builder.exclusiveStartKey(schema.toExclusiveStartKey(exclusiveStartKey))); // Perform request @@ -205,8 +174,8 @@ do { // Process results response.items().stream() - .map(schema::fromAttrMap) - .forEachOrdered(processor::process); + .map(schema::fromAttrMap) + .forEachOrdered(processor::process); } while (cursor.isPresent()); ``` @@ -234,41 +203,28 @@ And our usage would be: ```java String catId = "A18D5B00"; - Cat myCat = new Cat(catId); // Insertion is same as before, sharding is done under the hood -schema.table().putItem(PutItemRequest.builder() - .tableName(primary.tableName()) - .item(schema.toAttrMap(myCat)) - .build()); +schema.put() + .item(myCat) + .execute(client); // Retrieving cat is also same as before -Cat otherCat = schema.fromAttrMap(client.getItem(GetItemRequest.builder() - .tableName(schema.tableName()) - .key(schema.primaryKey(Map.of( - "catId", catId))) - .build()) - .item()); - -// Finally let's query some cats without an entire table scan -ShardPageResult result = singleTable.fetchShardNextPage( - client, - schema, - /* Pagination token */ Optional.empty(), - /* page size */ 100); -processCats(result.getItems()); +Optional catOpt = schema.get() + .key(Map.of("catId", catId)) + .execute(dynamo); // Finally let's dump all our cats using pagination Optional cursorOpt = Optional.empty(); do { - ShardPageResult result = singleTable.fetchShardNextPage( + ShardPageResult result = singleTable.fetchShardNextPage( client, schema, - cursorOpt, + /* Pagination token */ cursorOpt, /* page size */ 100); - cursorOpt = result.getCursorOpt(); - processCats(result.getItems()); + cursorOpt = result.getCursorOpt(); + processCats(result.getItems()); } while (cursorOpt.isPresent()); ``` @@ -279,27 +235,20 @@ overwriting the entire record whether it exists or not and for particular fields based on previous value. ```java -int catCountDiff = 4; // We want to increment by this amount - -HashMap userCounterNameMap = Maps.newHashMap(); -HashMap userCounterValueMap = Maps.newHashMap(); - -userCounterNameMap.put("#catCount", "catCount"); -userCounterValueMap.put(":diff", catCountDiff); -userCounterValueMap.put(":zero", 0L); - -String upsertExpression = schema.upsertExpression( - new CatCounter(bagId, catCountDiff), - userCounterNameMap, - userCounterValueMap, - // Indicate we are computing catCount ourselves - ImmutableSet.of("catCount"), - // Compute catCount by adding existing value (or zero) to our catCountDiff - ", #catCount = if_not_exists(#catCount, :zero) + :diff"); +Account account = new Account("12345", "asda@example.com", "api-key"); + +// Upsert -- create it +schema.update() + .upsert(account) + .execute(client); + +// Upsert -- update it +schema.update() + .upsert(account.toBuilder().apiKey("new-key").build()) + .execute(client); ``` -In this case, we have overwritten the `CatCounter` entirely except the `catCount` field. The field we are manually -calculating by adding the previous value if exists to our `catCountDiff` +In this case, we have first created an account and then updated the api key. ### Filter records diff --git a/single-table/src/main/java/io/dataspray/singletable/DynamoMapperImpl.java b/single-table/src/main/java/io/dataspray/singletable/DynamoMapperImpl.java index 65c7321..a25a6df 100644 --- a/single-table/src/main/java/io/dataspray/singletable/DynamoMapperImpl.java +++ b/single-table/src/main/java/io/dataspray/singletable/DynamoMapperImpl.java @@ -3,7 +3,6 @@ package io.dataspray.singletable; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.*; @@ -11,7 +10,9 @@ import com.google.gson.annotations.SerializedName; import com.google.gson.reflect.TypeToken; import io.dataspray.singletable.DynamoConvertersProxy.*; +import io.dataspray.singletable.builder.*; import lombok.NonNull; +import lombok.RequiredArgsConstructor; import lombok.Value; import lombok.extern.slf4j.Slf4j; import software.amazon.awscdk.services.dynamodb.AttributeType; @@ -27,7 +28,6 @@ import java.util.function.BiConsumer; import java.util.function.BiFunction; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.LongStream; import java.util.stream.Stream; @@ -499,285 +499,6 @@ private SchemaImpl parseSchema(TableType type, long indexNumber, Class ImmutableMap fieldAttrMarshallers = fieldAttrMarshallersBuilder.build(); ImmutableMap fieldAttrUnMarshallers = fieldAttrUnMarshallersBuilder.build(); - // Expression Builder Supplier - Supplier expressionBuilderSupplier = () -> new ExpressionBuilder() { - private boolean built = false; - private final Map nameMap = Maps.newHashMap(); - private final Map valMap = Maps.newHashMap(); - private final Map setUpdates = Maps.newHashMap(); - private final Map removeUpdates = Maps.newHashMap(); - private final Map addUpdates = Maps.newHashMap(); - private final Map deleteUpdates = Maps.newHashMap(); - private final List conditionExpressions = Lists.newArrayList(); - - @Override - public ExpressionBuilder set(String fieldName, Object object) { - checkState(!built); - checkState(!setUpdates.containsKey(fieldName)); - setUpdates.put(fieldName, - fieldMapping(fieldName) + " = " + valueMapping(fieldName, object)); - return this; - } - - @Override - public ExpressionBuilder set(ImmutableList namePath, AttributeValue value) { - checkState(!built); - checkArgument(!namePath.isEmpty()); - String fieldMapping = fieldMapping(namePath); - checkState(!addUpdates.containsKey(fieldMapping)); - setUpdates.put(fieldMapping, - fieldMapping + " = " + constantMapping(namePath, value)); - return this; - } - - @Override - public ExpressionBuilder setIncrement(String fieldName, Number increment) { - checkState(!built); - checkState(!setUpdates.containsKey(fieldName)); - setUpdates.put(fieldName, String.format("%s = if_not_exists(%s, %s) + %s", - fieldMapping(fieldName), - fieldMapping(fieldName), - constantMapping("zero", AttributeValue.fromN("0")), - valueMapping(fieldName, increment))); - return this; - } - - @Override - public ExpressionBuilder setExpression(String fieldName, String valueExpression) { - checkState(!built); - checkState(!setUpdates.containsKey(fieldName)); - setUpdates.put(fieldName, - fieldMapping(fieldName) + " = " + valueExpression); - return this; - } - - @Override - public ExpressionBuilder setExpression(String expression) { - checkState(!built); - setUpdates.put(expression, expression); - return this; - } - - @Override - public ExpressionBuilder add(String fieldName, Object object) { - checkState(!built); - checkState(!addUpdates.containsKey(fieldName)); - addUpdates.put(fieldName, - fieldMapping(fieldName) + " " + valueMapping(fieldName, object)); - return this; - } - - @Override - public ExpressionBuilder add(ImmutableList fieldPath, AttributeValue value) { - checkState(!built); - checkArgument(!fieldPath.isEmpty()); - String fieldMapping = fieldMapping(fieldPath); - checkState(!addUpdates.containsKey(fieldMapping)); - addUpdates.put(fieldMapping, - fieldMapping + " " + constantMapping(fieldPath, value)); - return this; - } - - @Override - public ExpressionBuilder remove(String fieldName) { - checkState(!built); - checkState(!removeUpdates.containsKey(fieldName)); - removeUpdates.put(fieldName, fieldMapping(fieldName)); - return this; - } - - @Override - public ExpressionBuilder remove(ImmutableList fieldPath) { - checkState(!built); - checkArgument(!fieldPath.isEmpty()); - String fieldMapping = fieldMapping(fieldPath); - checkState(!addUpdates.containsKey(fieldMapping)); - removeUpdates.put(fieldMapping, fieldMapping); - return this; - } - - @Override - public ExpressionBuilder delete(String fieldName, Object object) { - checkState(!built); - checkState(!deleteUpdates.containsKey(fieldName)); - deleteUpdates.put(fieldName, - fieldMapping(fieldName) + " " + valueMapping(fieldName, object)); - return this; - } - - @Override - public AttributeValue toAttrVal(String fieldName, Object object) { - if (object instanceof String) { - // For partition range keys and strings in general, there is no marshaller - return AttributeValue.fromS((String) object); - } else { - return checkNotNull(fieldAttrMarshallers.get(fieldName), - "Unknown field name %s", fieldName) - .marshall(object); - } - } - - @Override - public String fieldMapping(String fieldName) { - checkState(!built); - String mappedName = "#" + sanitizeFieldMapping(fieldName); - nameMap.put(mappedName, fieldName); - return mappedName; - } - - @Override - public String fieldMapping(ImmutableList fieldPath) { - return fieldPath.stream() - .map(this::fieldMapping) - .collect(Collectors.joining(".")); - } - - @Override - public String fieldMapping(String fieldName, String fieldValue) { - checkState(!built); - String mappedName = "#" + sanitizeFieldMapping(fieldName); - nameMap.put(mappedName, fieldValue); - return mappedName; - } - - @Override - public String valueMapping(String fieldName, Object object) { - checkState(!built); - return constantMapping(fieldName, toAttrVal(fieldName, object)); - } - - @Override - public String constantMapping(String name, AttributeValue value) { - checkState(!built); - String mappedName = ":" + sanitizeFieldMapping(name); - valMap.put(mappedName, value); - return mappedName; - } - - @Override - public String constantMapping(ImmutableList namePath, AttributeValue value) { - return constantMapping(namePath.stream() - .map(String::toLowerCase) - .collect(Collectors.joining("X")), value); - } - - @Override - public ExpressionBuilder condition(String expression) { - checkState(!built); - conditionExpressions.add(expression); - return this; - } - - @Override - public ExpressionBuilder conditionExists() { - checkState(!built); - conditionExpressions.add("attribute_exists(" + fieldMapping(partitionKeyName) + ")"); - return this; - } - - @Override - public ExpressionBuilder conditionNotExists() { - checkState(!built); - conditionExpressions.add("attribute_not_exists(" + fieldMapping(partitionKeyName) + ")"); - return this; - } - - @Override - public ExpressionBuilder conditionFieldEquals(String fieldName, Object objectOther) { - checkState(!built); - conditionExpressions.add(fieldMapping(fieldName) + " = " + valueMapping(fieldName, objectOther)); - return this; - } - - @Override - public ExpressionBuilder conditionFieldExists(String fieldName) { - checkState(!built); - conditionExpressions.add("attribute_exists(" + fieldMapping(fieldName) + ")"); - return this; - } - - @Override - public ExpressionBuilder conditionFieldNotExists(String fieldName) { - checkState(!built); - conditionExpressions.add("attribute_not_exists(" + fieldMapping(fieldName) + ")"); - return this; - } - - @Override - public Expression build() { - built = true; - ArrayList updates = Lists.newArrayList(); - if (!setUpdates.isEmpty()) { - updates.add("SET " + String.join(", ", setUpdates.values())); - } - if (!addUpdates.isEmpty()) { - updates.add("ADD " + String.join(", ", addUpdates.values())); - } - if (!removeUpdates.isEmpty()) { - updates.add("REMOVE " + String.join(", ", removeUpdates.values())); - } - if (!deleteUpdates.isEmpty()) { - updates.add("DELETE " + String.join(", ", deleteUpdates.values())); - } - final Optional updateOpt = Optional.ofNullable(Strings.emptyToNull(String.join(" ", updates))); - final Optional conditionOpt = Optional.ofNullable(Strings.emptyToNull(String.join(" AND ", conditionExpressions))); - final Optional> nameImmutableMapOpt = nameMap.isEmpty() ? Optional.empty() : Optional.of(ImmutableMap.copyOf(nameMap)); - final Optional> valImmutableMapOpt = valMap.isEmpty() ? Optional.empty() : Optional.of(ImmutableMap.copyOf(valMap)); - log.trace("Built dynamo expression: update {} condition {} nameMap {} valKeys {}", - updateOpt, conditionOpt, nameImmutableMapOpt, valImmutableMapOpt.map(ImmutableMap::keySet)); - return new Expression() { - @Override - public UpdateItemRequest.Builder toUpdateItemRequestBuilder() { - return toUpdateItemRequestBuilder(UpdateItemRequest.builder()); - } - - @Override - public UpdateItemRequest.Builder toUpdateItemRequestBuilder(UpdateItemRequest.Builder builder) { - builder.tableName(tableName); - updateExpression().ifPresent(builder::updateExpression); - conditionExpression().ifPresent(builder::conditionExpression); - expressionAttributeNames().ifPresent(builder::expressionAttributeNames); - expressionAttributeValues().ifPresent(builder::expressionAttributeValues); - return builder; - } - - @Override - public Optional updateExpression() { - return updateOpt; - } - - @Override - public Optional conditionExpression() { - return conditionOpt; - } - - @Override - public Optional> expressionAttributeNames() { - return nameImmutableMapOpt; - } - - @Override - public Optional> expressionAttributeValues() { - return valImmutableMapOpt; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("updateExpression", this.updateExpression()) - .add("conditionExpression", this.conditionExpression()) - .add("nameMap", this.expressionAttributeNames()) - .add("valMap", this.expressionAttributeValues()) - .toString(); - } - }; - } - - private String sanitizeFieldMapping(String fieldName) { - return fieldName.replaceAll("(^[^a-z])|[^a-zA-Z0-9]", "x"); - } - }; - return new SchemaImpl( type, partitionKeys, @@ -796,7 +517,6 @@ private String sanitizeFieldMapping(String fieldName) { fromAttrMapToCtorArgs, objCtor, toAttrMapMapper, - expressionBuilderSupplier, partitionKeyValueObjGetter, partitionKeyValueMapGetter, partitionKeyValueMapShardGetter); @@ -869,6 +589,7 @@ private Optional findInClassSet(Class clazz, ImmutableSet, return Optional.empty(); } + @RequiredArgsConstructor public class SchemaImpl implements TableSchema, IndexSchema { private final TableType type; private final String[] partitionKeys; @@ -887,62 +608,40 @@ public class SchemaImpl implements TableSchema, IndexSchema { private final Function, Object[]> fromAttrMapToCtorArgs; private final Constructor objCtor; private final Function> toAttrMapMapper; - private final Supplier expressionBuilderSupplier; private final Function partitionKeyValueObjGetter; private final Function, String> partitionKeyValueMapGetter; private final BiFunction, Integer, String> partitionKeyValueMapShardGetter; - public SchemaImpl( - TableType type, - String[] partitionKeys, - String[] shardKeys, - int shardCount, - String[] rangeKeys, - Field[] partitionKeyFields, - Field[] rangeKeyFields, - String rangePrefix, - String tableName, - Optional indexNameOpt, - String partitionKeyName, - String rangeKeyName, - ImmutableMap fieldAttrMarshallers, - ImmutableMap fieldAttrUnMarshallers, - Function, Object[]> fromAttrMapToCtorArgs, - Constructor objCtor, - Function> toAttrMapMapper, - Supplier expressionBuilderSupplier, - Function partitionKeyValueObjGetter, - Function, - String> partitionKeyValueMapGetter, - BiFunction, Integer, String> partitionKeyValueMapShardGetter) { - this.type = type; - this.partitionKeys = partitionKeys; - this.shardKeys = shardKeys; - this.shardCount = shardCount; - this.rangeKeys = rangeKeys; - this.partitionKeyFields = partitionKeyFields; - this.rangeKeyFields = rangeKeyFields; - this.rangePrefix = rangePrefix; - this.tableName = tableName; - this.indexNameOpt = indexNameOpt; - this.partitionKeyName = partitionKeyName; - this.rangeKeyName = rangeKeyName; - this.fieldAttrMarshallers = fieldAttrMarshallers; - this.fieldAttrUnMarshallers = fieldAttrUnMarshallers; - this.fromAttrMapToCtorArgs = fromAttrMapToCtorArgs; - this.objCtor = objCtor; - this.toAttrMapMapper = toAttrMapMapper; - this.expressionBuilderSupplier = expressionBuilderSupplier; - this.partitionKeyValueObjGetter = partitionKeyValueObjGetter; - this.partitionKeyValueMapGetter = partitionKeyValueMapGetter; - this.partitionKeyValueMapShardGetter = partitionKeyValueMapShardGetter; - } - @Override public String tableName() { return tableName; } + @Override + public QueryBuilder query() { + return new QueryBuilder<>(this); + } + + @Override + public GetBuilder get() { + return new GetBuilder<>(this); + } + + @Override + public PutBuilder put() { + return new PutBuilder<>(this); + } + + @Override + public DeleteBuilder delete() { + return new DeleteBuilder<>(this); + } + + @Override + public UpdateBuilder update() { + return new UpdateBuilder<>(this); + } + @Override public Optional indexNameOpt() { return indexNameOpt; @@ -953,11 +652,6 @@ public String indexName() { return indexNameOpt.orElseThrow(); } - @Override - public ExpressionBuilder expressionBuilder() { - return expressionBuilderSupplier.get(); - } - @Override public Map primaryKey(T obj) { return Map.ofEntries( @@ -1081,14 +775,34 @@ public String rangeValuePartial(Map values) { .toArray(String[]::new)); } + @Override + public AttributeValue toAttrValue(Object object) { + if (object instanceof AttributeValue) { + return (AttributeValue) object; + } + MarshallerAttrVal marshaller = findMarshallerAttrVal(Optional.empty(), object.getClass()); + if (marshaller == null) { + throw new RuntimeException("Cannot find marshaller for " + object.getClass()); + } + return marshaller.marshall(object); + } + @Override public AttributeValue toAttrValue(String fieldName, Object object) { - return fieldAttrMarshallers.get(fieldName).marshall(object); + MarshallerAttrVal marshaller = fieldAttrMarshallers.get(fieldName); + if (marshaller == null) { + throw new RuntimeException("Cannot find marshaller for field " + fieldName); + } + return marshaller.marshall(object); } @Override public Object fromAttrValue(String fieldName, AttributeValue attrVal) { - return fieldAttrUnMarshallers.get(fieldName).unmarshall(attrVal); + UnMarshallerAttrVal unMarshaller = fieldAttrUnMarshallers.get(fieldName); + if (unMarshaller == null) { + throw new RuntimeException("Cannot find unmarshaller for field " + fieldName); + } + return unMarshaller.unmarshall(attrVal); } @Override @@ -1117,23 +831,6 @@ public int shardCount() { return shardCount; } - @Override - public String upsertExpression(T object, Map nameMap, Map valMap, ImmutableSet skipFieldNames, String additionalExpression) { - List setUpdates = Lists.newArrayList(); - toAttrMapMapper.apply(object).forEach((key, value) -> { - if (partitionKeyName.equals(key) || rangeKeyName.equals(key)) { - return; - } - if (skipFieldNames.contains(key)) { - return; - } - nameMap.put("#" + key, key); - valMap.put(":" + key, value); - setUpdates.add("#" + key + " = " + ":" + key); - }); - return "SET " + String.join(", ", setUpdates) + additionalExpression; - } - @Override public Optional serializeLastEvaluatedKey(Map lastEvaluatedKey) { if (lastEvaluatedKey.isEmpty()) { diff --git a/single-table/src/main/java/io/dataspray/singletable/DynamoUtil.java b/single-table/src/main/java/io/dataspray/singletable/DynamoUtil.java index ea18a1f..b07e6c3 100644 --- a/single-table/src/main/java/io/dataspray/singletable/DynamoUtil.java +++ b/single-table/src/main/java/io/dataspray/singletable/DynamoUtil.java @@ -21,7 +21,7 @@ public ShardPageResult fetchShardNextPage(DynamoDbClient client, Schema ShardPageResult fetchShardNextPage(DynamoDbClient client, Schema schema, Optional cursorOpt, int maxPageSize, Map values) { + public ShardPageResult fetchShardNextPage(DynamoDbClient client, Schema schema, Optional cursorOpt, int maxPageSize, Map keyConditions) { checkArgument(maxPageSize > 0, "Max page size must be greater than zero"); Optional shardAndExclusiveStartKeyOpt = cursorOpt.map(schema::toShardedExclusiveStartKey); ImmutableList.Builder itemsBuilder = ImmutableList.builder(); @@ -31,7 +31,7 @@ public ShardPageResult fetchShardNextPage(DynamoDbClient client, Schema fieldPath, AttributeValue value); +import static com.google.common.base.Preconditions.*; - ExpressionBuilder setIncrement(String fieldName, Number increment); +@Slf4j +@RequiredArgsConstructor +public abstract class ExpressionBuilder implements Mappings, UpdateExpressionBuilder, + ConditionExpressionBuilder

, FilterExpressionBuilder

{ - ExpressionBuilder setExpression(String fieldName, String valueExpression); + protected final Schema schema; - ExpressionBuilder setExpression(String expression); + protected boolean built = false; + private final Map nameMap = Maps.newHashMap(); + private final Map valMap = Maps.newHashMap(); + private final Map setUpdates = Maps.newHashMap(); + private final Map removeUpdates = Maps.newHashMap(); + private final Map addUpdates = Maps.newHashMap(); + private final Map deleteUpdates = Maps.newHashMap(); + private final List conditionExpressions = Lists.newArrayList(); - ExpressionBuilder add(String fieldName, Object object); + abstract protected P getParent(); - ExpressionBuilder add(ImmutableList fieldPath, AttributeValue value); + @Override + public P conditionExpression(String expression) { + checkState(!built); + conditionExpressions.add(expression); + return getParent(); + } - ExpressionBuilder remove(String fieldName); + @Override + public P conditionExpression(MappingExpression mappingExpression) { + conditionExpression(mappingExpression.getExpression(this)); + return getParent(); + } - ExpressionBuilder remove(ImmutableList fieldPath); + @Override + public P filterExpression(String expression) { + conditionExpression(expression); + return getParent(); + } - ExpressionBuilder delete(String fieldName, Object object); + @Override + public P filterExpression(MappingExpression mappingExpression) { + conditionExpression(mappingExpression); + return getParent(); + } + @Override + public P conditionExists() { + checkState(!built); + conditionExpressions.add("attribute_exists(" + fieldMapping(schema.partitionKeyName()) + ")"); + return getParent(); + } - AttributeValue toAttrVal(String fieldName, Object object); + @Override + public P conditionNotExists() { + checkState(!built); + conditionExpressions.add("attribute_not_exists(" + fieldMapping(schema.partitionKeyName()) + ")"); + return getParent(); + } - String fieldMapping(String fieldName); + @Override + public P conditionFieldEquals(String fieldName, Object objectOther) { + checkState(!built); + conditionExpressions.add(fieldMapping(fieldName) + " = " + valueMapping(fieldName, objectOther)); + return getParent(); + } - String fieldMapping(ImmutableList fieldPath); + @Override + public P conditionFieldNotEquals(String fieldName, Object objectOther) { + checkState(!built); + conditionExpressions.add(fieldMapping(fieldName) + " <> " + valueMapping(fieldName, objectOther)); + return getParent(); + } - String fieldMapping(String fieldName, String fieldValue); + @Override + public P conditionFieldExists(String fieldName) { + checkState(!built); + conditionExpressions.add("attribute_exists(" + fieldMapping(fieldName) + ")"); + return getParent(); + } - String valueMapping(String fieldName, Object object); + @Override + public P conditionFieldNotExists(String fieldName) { + checkState(!built); + conditionExpressions.add("attribute_not_exists(" + fieldMapping(fieldName) + ")"); + return getParent(); + } - /** - * Maps a constant such as 0 (zero) to a field. - * - * @return the name to be referenced in the expression - */ - String constantMapping(String name, AttributeValue object); + @Override + public P updateExpression(String expression) { + checkState(!built); + setUpdates.put(expression, expression); + return getParent(); + } - String constantMapping(ImmutableList namePath, AttributeValue value); + @Override + public P updateExpression(MappingExpression mappingExpression) { + updateExpression(mappingExpression.getExpression(this)); + return getParent(); + } + @Override + public P upsert(T item) { + upsert(item, ImmutableSet.of()); + return getParent(); + } - ExpressionBuilder condition(String expression); + @Override + public P upsert(T item, ImmutableSet skipFieldNames) { + schema.toAttrMap(item).forEach((key, value) -> { + if (schema.partitionKeyName().equals(key) || schema.rangeKeyName().equals(key)) { + return; + } + if (skipFieldNames.contains(key)) { + return; + } + set(key, value); + }); + return getParent(); + } - ExpressionBuilder conditionExists(); + @Override + public P set(String fieldName, Object object) { + checkState(!built); + checkState(!setUpdates.containsKey(fieldName)); + setUpdates.put(fieldName, + fieldMapping(fieldName) + " = " + valueMapping(fieldName, object)); + return getParent(); + } - ExpressionBuilder conditionNotExists(); + @Override + public P set(ImmutableList fieldPath, AttributeValue value) { + checkState(!built); + checkArgument(!fieldPath.isEmpty()); + String fieldMapping = fieldMapping(fieldPath); + checkState(!addUpdates.containsKey(fieldMapping)); + setUpdates.put(fieldMapping, + fieldMapping + " = " + constantMapping(fieldPath, value)); + return getParent(); + } - ExpressionBuilder conditionFieldEquals(String fieldName, Object objectOther); + @Override + public P setIncrement(String fieldName, Number increment) { + checkState(!built); + checkState(!setUpdates.containsKey(fieldName)); + setUpdates.put(fieldName, String.format("%s = if_not_exists(%s, %s) + %s", + fieldMapping(fieldName), + fieldMapping(fieldName), + constantMapping("zero", AttributeValue.fromN("0")), + valueMapping(fieldName, increment))); + return getParent(); + } - ExpressionBuilder conditionFieldExists(String fieldName); + @Override + public P add(String fieldName, Object object) { + checkState(!built); + checkState(!addUpdates.containsKey(fieldName)); + addUpdates.put(fieldName, + fieldMapping(fieldName) + " " + valueMapping(fieldName, object)); + return getParent(); + } - ExpressionBuilder conditionFieldNotExists(String fieldName); + @Override + public P add(ImmutableList fieldPath, AttributeValue value) { + checkState(!built); + checkArgument(!fieldPath.isEmpty()); + String fieldMapping = fieldMapping(fieldPath); + checkState(!addUpdates.containsKey(fieldMapping)); + addUpdates.put(fieldMapping, + fieldMapping + " " + constantMapping(fieldPath, value)); + return getParent(); + } + @Override + public P remove(String fieldName) { + checkState(!built); + checkState(!removeUpdates.containsKey(fieldName)); + removeUpdates.put(fieldName, fieldMapping(fieldName)); + return getParent(); + } - Expression build(); + @Override + public P remove(ImmutableList fieldPath) { + checkState(!built); + checkArgument(!fieldPath.isEmpty()); + String fieldMapping = fieldMapping(fieldPath); + checkState(!addUpdates.containsKey(fieldMapping)); + removeUpdates.put(fieldMapping, fieldMapping); + return getParent(); + } + + @Override + public P delete(String fieldName, Object object) { + checkState(!built); + checkState(!deleteUpdates.containsKey(fieldName)); + deleteUpdates.put(fieldName, + fieldMapping(fieldName) + " " + valueMapping(fieldName, object)); + return getParent(); + } + + @Override + public String fieldMapping(String fieldName) { + checkState(!built); + String mappedName = "#" + sanitizeFieldMapping(fieldName); + nameMap.put(mappedName, fieldName); + return mappedName; + } + + @Override + public String fieldMapping(ImmutableList fieldPath) { + return fieldPath.stream() + .map(this::fieldMapping) + .collect(Collectors.joining(".")); + } + + @Override + public String fieldMapping(String fieldName, String fieldValue) { + checkState(!built); + String mappedName = "#" + sanitizeFieldMapping(fieldName); + nameMap.put(mappedName, fieldValue); + return mappedName; + } + + @Override + public String valueMapping(String fieldName, Object object) { + checkState(!built); + return constantMapping(fieldName, object); + } + + @Override + public String constantMapping(String name, Object object) { + checkState(!built); + String mappedName = ":" + sanitizeFieldMapping(name); + AttributeValue value = schema.toAttrValue(object); + valMap.put(mappedName, value); + return mappedName; + } + + @Override + public String constantMapping(ImmutableList namePath, Object value) { + return constantMapping(namePath.stream() + .map(String::toLowerCase) + .collect(Collectors.joining("X")), value); + } + + protected Expression buildExpression() { + built = true; + ArrayList updates = Lists.newArrayList(); + if (!setUpdates.isEmpty()) { + updates.add("SET " + String.join(", ", setUpdates.values())); + } + if (!addUpdates.isEmpty()) { + updates.add("ADD " + String.join(", ", addUpdates.values())); + } + if (!removeUpdates.isEmpty()) { + updates.add("REMOVE " + String.join(", ", removeUpdates.values())); + } + if (!deleteUpdates.isEmpty()) { + updates.add("DELETE " + String.join(", ", deleteUpdates.values())); + } + final Optional updateOpt = Optional.ofNullable(Strings.emptyToNull(String.join(" ", updates))); + final Optional conditionOpt = Optional.ofNullable(Strings.emptyToNull(String.join(" AND ", conditionExpressions))); + final Optional> nameImmutableMapOpt = nameMap.isEmpty() ? Optional.empty() : Optional.of(ImmutableMap.copyOf(nameMap)); + final Optional> valImmutableMapOpt = valMap.isEmpty() ? Optional.empty() : Optional.of(ImmutableMap.copyOf(valMap)); + log.trace("Built dynamo expression: update {} condition {} nameMap {} valKeys {}", + updateOpt, conditionOpt, nameImmutableMapOpt, valImmutableMapOpt.map(ImmutableMap::keySet)); + return new Expression() { + + @Override + public Optional updateExpression() { + return updateOpt; + } + + @Override + public Optional conditionExpression() { + return conditionOpt; + } + + @Override + public Optional filterExpression() { + return conditionOpt; + } + + @Override + public Optional> expressionAttributeNames() { + return nameImmutableMapOpt; + } + + @Override + public Optional> expressionAttributeValues() { + return valImmutableMapOpt; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("updateExpression", this.updateExpression()) + .add("conditionExpression", this.conditionExpression()) + .add("nameMap", this.expressionAttributeNames()) + .add("valMap", this.expressionAttributeValues()) + .toString(); + } + }; + } + + private String sanitizeFieldMapping(String fieldName) { + return fieldName.replaceAll("(^[^a-z])|[^a-zA-Z0-9]", "x"); + } + + @Override + public String toString() { + return buildExpression().toString(); + } } diff --git a/single-table/src/main/java/io/dataspray/singletable/Schema.java b/single-table/src/main/java/io/dataspray/singletable/Schema.java index 42da3b7..3ad0b6a 100644 --- a/single-table/src/main/java/io/dataspray/singletable/Schema.java +++ b/single-table/src/main/java/io/dataspray/singletable/Schema.java @@ -4,6 +4,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import io.dataspray.singletable.builder.QueryBuilder; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.Condition; @@ -14,10 +15,9 @@ public interface Schema { String tableName(); - /** - * If this is a an IndexSchema, returns the name of the index. - */ - Optional indexNameOpt(); + + QueryBuilder query(); + Map primaryKey(T obj); @@ -66,6 +66,8 @@ public interface Schema { String rangeValuePartial(Map values); + AttributeValue toAttrValue(Object object); + AttributeValue toAttrValue(String fieldName, Object object); Object fromAttrValue(String fieldName, AttributeValue attrVal); @@ -81,8 +83,6 @@ public interface Schema { int shardCount(); - String upsertExpression(T object, Map nameMap, Map valMap, ImmutableSet skipFieldNames, String additionalExpression); - Optional serializeLastEvaluatedKey(Map lastEvaluatedKey); Map toExclusiveStartKey(String serializedlastEvaluatedKey); @@ -94,4 +94,11 @@ public interface Schema { String serializeShardedLastEvaluatedKey(ShardAndExclusiveStartKey shardAndExclusiveStartKey); ShardAndExclusiveStartKey toShardedExclusiveStartKey(String serializedShardedLastEvaluatedKey); + + + /** + * If this is an IndexSchema, returns the name of the index. + */ + @Deprecated + Optional indexNameOpt(); } diff --git a/single-table/src/main/java/io/dataspray/singletable/TableSchema.java b/single-table/src/main/java/io/dataspray/singletable/TableSchema.java index 1cde8df..91c4e05 100644 --- a/single-table/src/main/java/io/dataspray/singletable/TableSchema.java +++ b/single-table/src/main/java/io/dataspray/singletable/TableSchema.java @@ -2,6 +2,16 @@ // SPDX-License-Identifier: Apache-2.0 package io.dataspray.singletable; +import io.dataspray.singletable.builder.*; +import software.amazon.awssdk.services.dynamodb.model.*; + public interface TableSchema extends Schema { - ExpressionBuilder expressionBuilder(); + + GetBuilder get(); + + PutBuilder put(); + + DeleteBuilder delete(); + + UpdateBuilder update(); } diff --git a/single-table/src/main/java/io/dataspray/singletable/builder/ConditionAndFilterExpressionBuilder.java b/single-table/src/main/java/io/dataspray/singletable/builder/ConditionAndFilterExpressionBuilder.java new file mode 100644 index 0000000..7a4bf37 --- /dev/null +++ b/single-table/src/main/java/io/dataspray/singletable/builder/ConditionAndFilterExpressionBuilder.java @@ -0,0 +1,16 @@ +package io.dataspray.singletable.builder; + +public interface ConditionAndFilterExpressionBuilder

{ + + P conditionExists(); + + P conditionNotExists(); + + P conditionFieldEquals(String fieldName, Object objectOther); + + P conditionFieldNotEquals(String fieldName, Object objectOther); + + P conditionFieldExists(String fieldName); + + P conditionFieldNotExists(String fieldName); +} diff --git a/single-table/src/main/java/io/dataspray/singletable/builder/ConditionExpressionBuilder.java b/single-table/src/main/java/io/dataspray/singletable/builder/ConditionExpressionBuilder.java new file mode 100644 index 0000000..70d30a7 --- /dev/null +++ b/single-table/src/main/java/io/dataspray/singletable/builder/ConditionExpressionBuilder.java @@ -0,0 +1,9 @@ +package io.dataspray.singletable.builder; + +public interface ConditionExpressionBuilder

extends ConditionAndFilterExpressionBuilder

{ + + P conditionExpression(String expression); + + P conditionExpression(MappingExpression mappingExpression); + +} diff --git a/single-table/src/main/java/io/dataspray/singletable/builder/DeleteBuilder.java b/single-table/src/main/java/io/dataspray/singletable/builder/DeleteBuilder.java new file mode 100644 index 0000000..812e9c8 --- /dev/null +++ b/single-table/src/main/java/io/dataspray/singletable/builder/DeleteBuilder.java @@ -0,0 +1,33 @@ +package io.dataspray.singletable.builder; + +import io.dataspray.singletable.ExpressionBuilder; +import io.dataspray.singletable.Schema; +import software.amazon.awssdk.services.dynamodb.model.DeleteItemRequest; + +import static com.google.common.base.Preconditions.checkState; + +public class DeleteBuilder extends ExpressionBuilder> implements ConditionExpressionBuilder> { + + public DeleteBuilder(Schema schema) { + super(schema); + } + + protected DeleteBuilder getParent() { + return this; + } + + public DeleteItemRequest.Builder builder() { + Expression expression = buildExpression(); + DeleteItemRequest.Builder builder = DeleteItemRequest.builder(); + builder.tableName(schema.tableName()); + checkState(expression.updateExpression().isEmpty(), "Delete does not support update expression"); + expression.conditionExpression().ifPresent(builder::conditionExpression); + expression.expressionAttributeNames().ifPresent(builder::expressionAttributeNames); + expression.expressionAttributeValues().ifPresent(builder::expressionAttributeValues); + return builder; + } + + public DeleteItemRequest build() { + return builder().build(); + } +} diff --git a/single-table/src/main/java/io/dataspray/singletable/Expression.java b/single-table/src/main/java/io/dataspray/singletable/builder/Expression.java similarity index 60% rename from single-table/src/main/java/io/dataspray/singletable/Expression.java rename to single-table/src/main/java/io/dataspray/singletable/builder/Expression.java index 8f757ac..37dc84e 100644 --- a/single-table/src/main/java/io/dataspray/singletable/Expression.java +++ b/single-table/src/main/java/io/dataspray/singletable/builder/Expression.java @@ -1,22 +1,20 @@ // SPDX-FileCopyrightText: 2019-2022 Matus Faro // SPDX-License-Identifier: Apache-2.0 -package io.dataspray.singletable; +package io.dataspray.singletable.builder; import com.google.common.collect.ImmutableMap; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest; +import software.amazon.awssdk.services.dynamodb.model.*; import java.util.Optional; public interface Expression { - UpdateItemRequest.Builder toUpdateItemRequestBuilder(); - - UpdateItemRequest.Builder toUpdateItemRequestBuilder(UpdateItemRequest.Builder builder); Optional updateExpression(); Optional conditionExpression(); + Optional filterExpression(); + Optional> expressionAttributeNames(); Optional> expressionAttributeValues(); diff --git a/single-table/src/main/java/io/dataspray/singletable/builder/FilterExpressionBuilder.java b/single-table/src/main/java/io/dataspray/singletable/builder/FilterExpressionBuilder.java new file mode 100644 index 0000000..a371b0b --- /dev/null +++ b/single-table/src/main/java/io/dataspray/singletable/builder/FilterExpressionBuilder.java @@ -0,0 +1,8 @@ +package io.dataspray.singletable.builder; + +public interface FilterExpressionBuilder

extends ConditionAndFilterExpressionBuilder

{ + + P filterExpression(String expression); + + P filterExpression(MappingExpression mappingExpression); +} diff --git a/single-table/src/main/java/io/dataspray/singletable/builder/GetBuilder.java b/single-table/src/main/java/io/dataspray/singletable/builder/GetBuilder.java new file mode 100644 index 0000000..5a78c26 --- /dev/null +++ b/single-table/src/main/java/io/dataspray/singletable/builder/GetBuilder.java @@ -0,0 +1,48 @@ +package io.dataspray.singletable.builder; + +import io.dataspray.singletable.ExpressionBuilder; +import io.dataspray.singletable.Schema; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.GetItemRequest; + +import java.util.Map; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkState; + +public class GetBuilder extends ExpressionBuilder> { + + public GetBuilder(Schema schema) { + super(schema); + } + + protected GetBuilder getParent() { + return this; + } + + private Optional> keyOpt = Optional.empty(); + + public GetBuilder key(Map primaryKey) { + checkState(!built); + this.keyOpt = Optional.of(schema.primaryKey(primaryKey)); + return this; + } + + public GetItemRequest.Builder builder() { + GetItemRequest.Builder builder = GetItemRequest.builder(); + builder.tableName(schema.tableName()); + keyOpt.ifPresent(builder::key); + return builder; + } + + public GetItemRequest build() { + return builder().build(); + } + + public Optional execute(DynamoDbClient dynamo) { + return Optional.ofNullable(schema.fromAttrMap(dynamo + .getItem(build()) + .item())); + } +} diff --git a/single-table/src/main/java/io/dataspray/singletable/builder/MappingExpression.java b/single-table/src/main/java/io/dataspray/singletable/builder/MappingExpression.java new file mode 100644 index 0000000..653a6a0 --- /dev/null +++ b/single-table/src/main/java/io/dataspray/singletable/builder/MappingExpression.java @@ -0,0 +1,6 @@ +package io.dataspray.singletable.builder; + +public interface MappingExpression { + + String getExpression(Mappings mappings); +} diff --git a/single-table/src/main/java/io/dataspray/singletable/builder/Mappings.java b/single-table/src/main/java/io/dataspray/singletable/builder/Mappings.java new file mode 100644 index 0000000..634fec5 --- /dev/null +++ b/single-table/src/main/java/io/dataspray/singletable/builder/Mappings.java @@ -0,0 +1,18 @@ +package io.dataspray.singletable.builder; + +import com.google.common.collect.ImmutableList; + +public interface Mappings { + + String fieldMapping(String fieldName); + + String fieldMapping(ImmutableList fieldPath); + + String fieldMapping(String fieldName, String fieldValue); + + String valueMapping(String fieldName, Object object); + + String constantMapping(String name, Object object); + + String constantMapping(ImmutableList namePath, Object value); +} diff --git a/single-table/src/main/java/io/dataspray/singletable/builder/PutBuilder.java b/single-table/src/main/java/io/dataspray/singletable/builder/PutBuilder.java new file mode 100644 index 0000000..0612cb5 --- /dev/null +++ b/single-table/src/main/java/io/dataspray/singletable/builder/PutBuilder.java @@ -0,0 +1,50 @@ +package io.dataspray.singletable.builder; + +import io.dataspray.singletable.ExpressionBuilder; +import io.dataspray.singletable.Schema; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +import software.amazon.awssdk.services.dynamodb.model.PutItemResponse; + +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkState; + +public class PutBuilder extends ExpressionBuilder> implements ConditionExpressionBuilder> { + + public PutBuilder(Schema schema) { + super(schema); + } + + protected PutBuilder getParent() { + return this; + } + + private Optional itemOpt = Optional.empty(); + + public PutBuilder item(T item) { + checkState(!built); + this.itemOpt = Optional.of(item); + return this; + } + + public PutItemRequest.Builder builder() { + Expression expression = buildExpression(); + PutItemRequest.Builder builder = PutItemRequest.builder(); + builder.tableName(schema.tableName()); + checkState(expression.updateExpression().isEmpty(), "Put does not support update expression"); + expression.conditionExpression().ifPresent(builder::conditionExpression); + expression.expressionAttributeNames().ifPresent(builder::expressionAttributeNames); + expression.expressionAttributeValues().ifPresent(builder::expressionAttributeValues); + itemOpt.ifPresent(item -> builder.item(schema.toAttrMap(item))); + return builder; + } + + public PutItemRequest build() { + return builder().build(); + } + + public PutItemResponse execute(DynamoDbClient dynamo) { + return dynamo.putItem(build()); + } +} diff --git a/single-table/src/main/java/io/dataspray/singletable/builder/QueryBuilder.java b/single-table/src/main/java/io/dataspray/singletable/builder/QueryBuilder.java new file mode 100644 index 0000000..d426be0 --- /dev/null +++ b/single-table/src/main/java/io/dataspray/singletable/builder/QueryBuilder.java @@ -0,0 +1,50 @@ +package io.dataspray.singletable.builder; + +import io.dataspray.singletable.ExpressionBuilder; +import io.dataspray.singletable.Schema; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; + +import java.util.Map; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkState; + +public class QueryBuilder extends ExpressionBuilder> implements FilterExpressionBuilder>, ConditionExpressionBuilder> { + + public QueryBuilder(Schema schema) { + super(schema); + } + + protected QueryBuilder getParent() { + return this; + } + + private Optional> keyConditionsOpt = Optional.empty(); + + public QueryBuilder keyConditions(Map primaryKey) { + checkState(!built); + this.keyConditionsOpt = Optional.of(schema.attrMapToConditions(schema.primaryKey(primaryKey))); + return this; + } + + public QueryRequest.Builder builder() { + Expression expression = buildExpression(); + QueryRequest.Builder builder = QueryRequest.builder(); + builder.tableName(schema.tableName()); + checkState(expression.updateExpression().isEmpty(), "Query does not support update expression"); + expression.conditionExpression().ifPresent(builder::filterExpression); + expression.expressionAttributeNames().ifPresent(builder::expressionAttributeNames); + expression.expressionAttributeValues().ifPresent(builder::expressionAttributeValues); + keyConditionsOpt.ifPresent(builder::keyConditions); + return builder; + } + + public QueryRequest build() { + return builder().build(); + } + + public QueryResponse execute(DynamoDbClient dynamo) { + return dynamo.query(build()); + } +} diff --git a/single-table/src/main/java/io/dataspray/singletable/builder/UpdateBuilder.java b/single-table/src/main/java/io/dataspray/singletable/builder/UpdateBuilder.java new file mode 100644 index 0000000..e1f01fe --- /dev/null +++ b/single-table/src/main/java/io/dataspray/singletable/builder/UpdateBuilder.java @@ -0,0 +1,62 @@ +package io.dataspray.singletable.builder; + +import com.google.common.collect.ImmutableSet; +import io.dataspray.singletable.ExpressionBuilder; +import io.dataspray.singletable.Schema; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; + +import java.util.Map; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkState; + +public class UpdateBuilder extends ExpressionBuilder> implements ConditionExpressionBuilder>, UpdateExpressionBuilder> { + + public UpdateBuilder(Schema schema) { + super(schema); + } + + protected UpdateBuilder getParent() { + return this; + } + + private Optional> keyOpt = Optional.empty(); + + public UpdateBuilder key(Map primaryKey) { + checkState(!built); + this.keyOpt = Optional.of(schema.primaryKey(primaryKey)); + return this; + } + + + @Override + public UpdateBuilder upsert(T item, ImmutableSet skipFieldNames) { + checkState(!built); + this.keyOpt = Optional.of(schema.primaryKey(item)); + return super.upsert(item, skipFieldNames); + } + + + public UpdateItemRequest.Builder builder() { + Expression expression = buildExpression(); + UpdateItemRequest.Builder builder = UpdateItemRequest.builder(); + builder.tableName(schema.tableName()); + expression.updateExpression().ifPresent(builder::updateExpression); + expression.conditionExpression().ifPresent(builder::conditionExpression); + expression.expressionAttributeNames().ifPresent(builder::expressionAttributeNames); + expression.expressionAttributeValues().ifPresent(builder::expressionAttributeValues); + keyOpt.ifPresent(builder::key); + return builder; + } + + public UpdateItemRequest build() { + return builder().build(); + } + + public Optional execute(DynamoDbClient dynamo) { + return Optional.ofNullable(schema.fromAttrMap(dynamo.updateItem(builder() + .returnValues(ReturnValue.ALL_NEW) + .build()).attributes())); + } +} diff --git a/single-table/src/main/java/io/dataspray/singletable/builder/UpdateExpressionBuilder.java b/single-table/src/main/java/io/dataspray/singletable/builder/UpdateExpressionBuilder.java new file mode 100644 index 0000000..fe41f91 --- /dev/null +++ b/single-table/src/main/java/io/dataspray/singletable/builder/UpdateExpressionBuilder.java @@ -0,0 +1,32 @@ +package io.dataspray.singletable.builder; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; + +public interface UpdateExpressionBuilder { + + P updateExpression(String expression); + + P updateExpression(MappingExpression mappingExpression); + + P upsert(T item); + + P upsert(T item, ImmutableSet skipFieldNames); + + P set(String fieldName, Object object); + + P set(ImmutableList fieldPath, AttributeValue value); + + P setIncrement(String fieldName, Number increment); + + P add(String fieldName, Object object); + + P add(ImmutableList fieldPath, AttributeValue value); + + P remove(String fieldName); + + P remove(ImmutableList fieldPath); + + P delete(String fieldName, Object object); +} diff --git a/single-table/src/test/java/io/dataspray/singletable/DynamoExpressionTest.java b/single-table/src/test/java/io/dataspray/singletable/DynamoExpressionTest.java index 4bd3c27..3c5f476 100644 --- a/single-table/src/test/java/io/dataspray/singletable/DynamoExpressionTest.java +++ b/single-table/src/test/java/io/dataspray/singletable/DynamoExpressionTest.java @@ -2,12 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 package io.dataspray.singletable; +import io.dataspray.singletable.builder.Expression; +import io.dataspray.singletable.builder.UpdateBuilder; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Value; import lombok.extern.slf4j.Slf4j; import org.junit.Test; import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException; +import software.amazon.awssdk.services.dynamodb.model.UpdateItemRequest; import java.util.UUID; @@ -35,15 +38,16 @@ public static class Data { public void test() throws Exception { TableSchema primary = mapper.parseTableSchema(Data.class); - Expression expression = primary.expressionBuilder() + UpdateBuilder updateBuilder = primary.update() .set("f1", "CHANGED") .conditionExists() .conditionFieldEquals("f2", 4L) .conditionFieldExists("f3") - .conditionFieldNotExists("f1") - .build(); + .conditionFieldNotExists("f1"); + log.info("updateBuilder: {}", updateBuilder); + UpdateItemRequest.Builder updateItemRequestBuilder = updateBuilder.builder(); - assertExpression(primary, expression, putData(primary, Data.builder() + assertExpression(primary, updateItemRequestBuilder, putData(primary, Data.builder() .id(UUID.randomUUID().toString()) .rk(UUID.randomUUID().toString()) .f2(4L) @@ -52,7 +56,7 @@ public void test() throws Exception { .build()) .toBuilder().f1("CHANGED").build()); - assertExpressionConditionFails(primary, expression, putData(primary, Data.builder() + assertExpressionConditionFails(primary, updateItemRequestBuilder, putData(primary, Data.builder() .id(UUID.randomUUID().toString()) .rk(UUID.randomUUID().toString()) .f2(5L) // Incorrect @@ -60,7 +64,7 @@ public void test() throws Exception { .f4(3) .build())); - assertExpressionConditionFails(primary, expression, putData(primary, Data.builder() + assertExpressionConditionFails(primary, updateItemRequestBuilder, putData(primary, Data.builder() .id(UUID.randomUUID().toString()) .rk(UUID.randomUUID().toString()) .f2(4L) @@ -68,7 +72,7 @@ public void test() throws Exception { .f4(7) .build())); - assertExpressionConditionFails(primary, expression, putData(primary, Data.builder() + assertExpressionConditionFails(primary, updateItemRequestBuilder, putData(primary, Data.builder() .id(UUID.randomUUID().toString()) .rk(UUID.randomUUID().toString()) .f1("htg") // Should be missing @@ -77,19 +81,17 @@ public void test() throws Exception { .build())); } - void assertExpression(Schema schema, Expression expression, T expectedData) { - client.updateItem(expression.toUpdateItemRequestBuilder() - .key(schema.primaryKey(expectedData)).build()); + void assertExpression(Schema schema, UpdateItemRequest.Builder builder, T expectedData) { + client.updateItem(builder.key(schema.primaryKey(expectedData)).build()); T actualData = schema.fromAttrMap(client.getItem(b -> b .tableName(schema.tableName()) .key(schema.primaryKey(expectedData))).item()); assertEquals(expectedData, actualData); } - void assertExpressionConditionFails(Schema schema, Expression expression, T expectedData) { + void assertExpressionConditionFails(Schema schema, UpdateItemRequest.Builder builder, T expectedData) { try { - client.updateItem(expression.toUpdateItemRequestBuilder() - .key(schema.primaryKey(expectedData)).build()); + client.updateItem(builder.key(schema.primaryKey(expectedData)).build()); fail("Expected ConditionalCheckFailedException"); } catch (ConditionalCheckFailedException ex) { // Expected diff --git a/single-table/src/test/java/io/dataspray/singletable/ReadmeTest.java b/single-table/src/test/java/io/dataspray/singletable/ReadmeTest.java new file mode 100644 index 0000000..3c98f08 --- /dev/null +++ b/single-table/src/test/java/io/dataspray/singletable/ReadmeTest.java @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: 2019-2022 Matus Faro +// SPDX-License-Identifier: Apache-2.0 +package io.dataspray.singletable; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import lombok.*; +import lombok.extern.slf4j.Slf4j; +import org.junit.Before; +import org.junit.Test; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.QueryRequest; +import software.amazon.awssdk.services.dynamodb.model.QueryResponse; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static io.dataspray.singletable.TableType.Gsi; +import static io.dataspray.singletable.TableType.Primary; + +@Slf4j +public class ReadmeTest extends AbstractDynamoTest { + + @Builder(toBuilder = true) + @Value + @AllArgsConstructor + @DynamoTable(type = Primary, partitionKeys = "accountId", rangePrefix = "account") + @DynamoTable(type = Gsi, indexNumber = 1, partitionKeys = {"apiKey"}, rangePrefix = "accountByApiKey") + @DynamoTable(type = Gsi, indexNumber = 2, partitionKeys = {"email"}, rangePrefix = "accountByEmail") + static class Account { + @NonNull + String accountId; + + @NonNull + String email; + + @ToString.Exclude + String apiKey; + } + + @Test(timeout = 20_000L) + public void testExample() throws Exception { + + // Initialize schema + SingleTable singleTable = SingleTable.builder() + .tablePrefix(tablePrefix).build(); + TableSchema schema = singleTable.parseTableSchema(Account.class); + + // Insert new account + Account account = new Account("8426", "matus@example.com", null); + schema.put().item(account).execute(client); + + // Fetch other account + Optional otherAccountOpt = schema.get() + .key(Map.of("accountId", "abc-9473")) + .execute(client); + } + + @Test(timeout = 20_000L) + public void testInitialize() throws Exception { + DynamoDbClient dynamo = client; + + SingleTable singleTable = SingleTable.builder() + .tablePrefix(tablePrefix) + .build(); + TableSchema accountSchema = singleTable.parseTableSchema(Account.class); + IndexSchema accountByApiKeySchema = singleTable.parseGlobalSecondaryIndexSchema(1, Account.class); + } + + @Test(timeout = 20_000L) + public void testInsert() throws Exception { + Account account = new Account("12345", "email@example.com", "api-key"); + TableSchema schema = singleTable.parseTableSchema(Account.class); + + // --------- + + schema.put() + .item(account) + .execute(client); + } + + @Test(timeout = 20_000L) + public void testPut() throws Exception { + String apiKey = "api-key"; + String entryId = "entry-id"; + TableSchema schema = singleTable.parseTableSchema(Account.class); + + // --------- + + Optional updatedAccountOpt = schema.update() + .key(Map.of("accountId", "1234fsd432e5")) + + // Apply conditions + .conditionNotExists() + .conditionFieldNotExists("email") +// .conditionFieldNotEquals("apiKey", "123") + + // Modify data + .set("accountId", "asfasdf") + .set("email", "grfgerfg") + .setIncrement("votersCount", 1) + // Remove entry from a json field +// .remove(ImmutableList.of("entryJson", entryId, "isMoved")) + + .execute(client); + } + + @Test(timeout = 20_000L) + public void testGet() throws Exception { + TableSchema schema = singleTable.parseTableSchema(Account.class); + + // --------- + + Optional accountOpt = schema.get() + .key(Map.of("accountId", "12345")) + .execute(client); + } + + @FunctionalInterface + interface Processor { + void process(Account account); + } + + @Test(timeout = 20_000L) + public void testQuery() throws Exception { + TableSchema schema = singleTable.parseTableSchema(Account.class); + Processor processor = account -> log.info("Account: {}", account); + + // --------- + + Optional cursor = Optional.empty(); + do { + // Prepare request + QueryRequest.Builder builder = schema.query() + // Query by partition key + .keyConditions(Map.of("accountId", "12345")) + .builder() + .limit(2); + cursor.ifPresent(exclusiveStartKey -> builder.exclusiveStartKey(schema.toExclusiveStartKey(exclusiveStartKey))); + + // Perform request + QueryResponse response = client.query(builder.build()); + + // Retrieve next cursor + cursor = schema.serializeLastEvaluatedKey(response.lastEvaluatedKey()); + + // Process results + response.items().stream() + .map(schema::fromAttrMap) + .forEachOrdered(this::processAccount); + } while (cursor.isPresent()); + } + + private void processAccount(Account account) { + } + + @Value + @AllArgsConstructor + @DynamoTable(type = Primary, shardKeys = {"catId"}, shardPrefix = "cat", shardCount = 100, rangePrefix = "cat", rangeKeys = "catId") + public static class Cat { + @NonNull + String catId; + } + + @Test(timeout = 20_000L) + public void testScanType() throws Exception { + + TableSchema schema = singleTable.parseTableSchema(Cat.class); + + // --------- + + String catId = "A18D5B00"; + Cat myCat = new Cat(catId); + + // Insertion is same as before, sharding is done under the hood + schema.put() + .item(myCat) + .execute(client); + + // Retrieving cat is also same as before + Optional catOpt = schema.get() + .key(Map.of("catId", catId)) + .execute(client); + + // Finally let's dump all our cats using pagination + Optional cursorOpt = Optional.empty(); + do { + ShardPageResult result = singleTable.fetchShardNextPage( + client, + schema, + /* Pagination token */ cursorOpt, + /* page size */ 100); + cursorOpt = result.getCursorOpt(); + processCats(result.getItems()); + } while (cursorOpt.isPresent()); + } + + private void processCats(ImmutableList cats) { + } + + + @Test(timeout = 20_000L) + public void testUpsert() throws Exception { + TableSchema schema = singleTable.parseTableSchema(Account.class); + + // --------- + + Account account = new Account("12345", "asda@example.com", "api-key"); + + // Upsert -- create it + schema.update() + .upsert(account) + .execute(client); + + // Upsert -- update it + schema.update() + .upsert(account.toBuilder().apiKey("new-key").build()) + .execute(client); + } +} \ No newline at end of file