From f1ae7b0508608e8c95b16ef017654d47faa76722 Mon Sep 17 00:00:00 2001 From: Artyom Gabeev Date: Thu, 25 May 2023 15:56:56 +0300 Subject: [PATCH 1/6] Upgrade AWS SDK to v2 --- CONTRIBUTORS.md | 1 + java/app-encryption/README.md | 44 ++- java/app-encryption/pom.xml | 27 +- .../java/com/godaddy/asherah/TestSetup.java | 24 +- .../regression/DynamoDbGlobalTableIT.java | 61 ++-- .../kms/AwsKeyManagementServiceImpl.java | 85 +++--- .../kms/AwsKmsClientFactory.java | 26 +- .../persistence/DynamoDbMetastoreImpl.java | 262 +++++++++--------- .../kms/AwsKeyManagementServiceImplTest.java | 68 +++-- .../DynamoDbMetastoreImplTest.java | 249 +++++++++-------- samples/java/reference-app/pom.xml | 2 +- .../com/godaddy/asherah/referenceapp/App.java | 11 +- server/java/pom.xml | 2 +- .../asherah/grpc/AppEncryptionConfig.java | 18 +- .../asherah/grpc/AppEncryptionConfigTest.java | 6 +- 15 files changed, 489 insertions(+), 397 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index d157eb19d..6e7087837 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -17,3 +17,4 @@ These people contributed to Asherah's design, implementation, testing, and open * Nikhil Lohia ([@nikoo28](https://github.com/nikoo28)) * Ryanne Fox * Sushant Mimani ([@sushantmimani](https://github.com/sushantmimani)) + * Artyom Gabeev diff --git a/java/app-encryption/README.md b/java/app-encryption/README.md index 1d6ee53bd..95469c139 100644 --- a/java/app-encryption/README.md +++ b/java/app-encryption/README.md @@ -4,6 +4,7 @@ Application level envelope encryption SDK for Java with support for cloud-agnost [![Version](https://img.shields.io/maven-central/v/com.godaddy.asherah/appencryption)](https://mvnrepository.com/artifact/com.godaddy.asherah/appencryption) * [Installation](#installation) + * [Upgrade](#upgrade) * [Quick Start](#quick-start) * [How to Use Asherah](#how-to-use-asherah) * [Define the Metastore](#define-the-metastore) @@ -30,11 +31,31 @@ You can specify the current release of Asherah as a project dependency using the com.godaddy.asherah appencryption - 0.1.1 + 0.3.0 ``` +## Upgrade + +### 0.3.x + +Starting with version 0.3.0 Asherah uses AWS SDK v2. AWS SDK both major versions can be used in the same project, +and v2 dependencies will be provided as transitive. There is single breaking change around DynamoDB metastore configuration: +Previous DynamoDB client was initialized withing metastore builder, now it should be provided as a parameter. + +Assuming you had previously used `DynamoDbMetastoreImpl`, you can replace it with new equivalent: +```java +// before 0.3.0 +final var metastore = DynamoDbMetastoreImpl.newBuilder("us-west-2").build(); + +// after +final var dynamoDbClient = DynamoDBClient.builder().region(Region.US_WEST_2).build(); +final var metastore = DynamoDbMetastoreImpl.newBuilder("us-west-2", dynamoDbClient).build(); +``` + +Region specified two times intentionally, to support Global Tables and prevent key collisions. + ## Quick Start ```java @@ -87,7 +108,11 @@ For simplicity, the DynamoDB implementation uses the builder pattern to enable c To obtain an instance of the builder, use the static factory method `newBuilder`. ```java -DynamoDbMetastoreImpl.newBuilder(); +// Create dynamo db client +DynamoDbClient dynamoDbClient = ...; + +// Build the DynamoDB Metastore +DynamoDbMetastoreImpl.newBuilder("us-west-2", dynamoDbClient).build(); ``` Once you have a builder, you can either use the `withXXX` setter methods to configure the metastore properties or simply build the metastore by calling the `build` method. @@ -95,15 +120,13 @@ build the metastore by calling the `build` method. - **withKeySuffix**: Specifies whether key suffix should be enabled for DynamoDB. **This is required to enable Global Tables.** - **withTableName**: Specifies the name of the DynamoDb table. - - **withRegion**: Specifies the region for the AWS DynamoDb client. - - **withEndPointConfiguration**: Adds an EndPoint configuration to the AWS DynamoDb client. Below is an example of a DynamoDB metastore that uses a Global Table named `TestTable` ```java Metastore dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder() - .withKeySuffix("us-west-2") .withTableName("TestTable") + .withKeySuffix() .build(); ``` @@ -129,6 +152,17 @@ Map regionMap = ImmutableMap.of("us-east-1", "arn_of_us-east-1", KeyManagementService keyManagementService = AwsKeyManagementServiceImpl.newBuilder(regionMap, "us-east-1").build(); ``` +Starting with version 0.3.0 it is possible to specify AWS KMS client factory to be used: +```java +// Define AWS KMS client factory +AwsKmsClientFactory awsKmsClientFactory = ...; + +// Build the Key Management Service using the region map and your preferred (usually current) region and custom AWS KMS client factory +KeyManagementService keyManagementService = AwsKeyManagementServiceImpl.newBuilder(regionMap, "us-east-1") + .withAwsKmsClientFactory(awsKmsClientFactory) + .build(); +``` + #### Static KMS (FOR TESTING ONLY) ```java diff --git a/java/app-encryption/pom.xml b/java/app-encryption/pom.xml index ba96d646e..f5f72e0d9 100644 --- a/java/app-encryption/pom.xml +++ b/java/app-encryption/pom.xml @@ -4,7 +4,7 @@ com.godaddy.asherah appencryption - 0.2.5 + 0.3.0 Asherah An application-layer encryption SDK that provides advanced encryption features and in depth defense against @@ -45,6 +45,7 @@ 3.12.0 1.21.1 1.12.468 + 2.20.69 1.70 3.3.0 3.1.6 @@ -299,21 +300,15 @@ - com.amazonaws - aws-java-sdk-core - ${aws.sdk.version} - - - - com.amazonaws - aws-java-sdk-kms - ${aws.sdk.version} + software.amazon.awssdk + kms + ${aws.sdk-v2.version} - com.amazonaws - aws-java-sdk-dynamodb - ${aws.sdk.version} + software.amazon.awssdk + dynamodb + ${aws.sdk-v2.version} @@ -323,6 +318,12 @@ + + com.amazonaws + aws-java-sdk-dynamodb + ${aws.sdk.version} + test + ch.qos.logback diff --git a/java/app-encryption/src/it/java/com/godaddy/asherah/TestSetup.java b/java/app-encryption/src/it/java/com/godaddy/asherah/TestSetup.java index 99c0321cd..8d21ab36f 100644 --- a/java/app-encryption/src/it/java/com/godaddy/asherah/TestSetup.java +++ b/java/app-encryption/src/it/java/com/godaddy/asherah/TestSetup.java @@ -6,23 +6,35 @@ import com.godaddy.asherah.appencryption.kms.AwsKeyManagementServiceImpl; import com.godaddy.asherah.appencryption.kms.KeyManagementService; import com.godaddy.asherah.appencryption.kms.StaticKeyManagementServiceImpl; -import com.godaddy.asherah.appencryption.persistence.DynamoDbMetastoreImpl; -import com.godaddy.asherah.appencryption.persistence.InMemoryMetastoreImpl; -import com.godaddy.asherah.appencryption.persistence.JdbcMetastoreImpl; -import com.godaddy.asherah.appencryption.persistence.Metastore; +import com.godaddy.asherah.appencryption.persistence.*; import com.google.common.base.Splitter; import com.zaxxer.hikari.HikariDataSource; import org.apache.commons.lang3.StringUtils; import org.json.JSONObject; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import java.io.File; import java.io.IOException; +import java.net.URI; import java.util.Map; import static com.godaddy.asherah.testhelpers.Constants.*; public class TestSetup { + public static DynamoDbClient createDynamoDbClient(String endpoint, String region) { + return DynamoDbClient.builder() + .region(Region.of(region)) + .endpointOverride(URI.create(endpoint)) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create("test", "test"))) + .build(); + } + public static Metastore createMetastore() { String metastoreType = configReader().getMetastoreType(); if (metastoreType.equalsIgnoreCase(METASTORE_JDBC)) { @@ -37,10 +49,6 @@ public static Metastore createMetastore() { return JdbcMetastoreImpl.newBuilder(dataSource).build(); } - if (metastoreType.equalsIgnoreCase(METASTORE_DYNAMODB)) { - return DynamoDbMetastoreImpl.newBuilder("us-west-2").build(); - } - return new InMemoryMetastoreImpl<>(); } diff --git a/java/app-encryption/src/it/java/com/godaddy/asherah/regression/DynamoDbGlobalTableIT.java b/java/app-encryption/src/it/java/com/godaddy/asherah/regression/DynamoDbGlobalTableIT.java index 939112afb..6a9828786 100644 --- a/java/app-encryption/src/it/java/com/godaddy/asherah/regression/DynamoDbGlobalTableIT.java +++ b/java/app-encryption/src/it/java/com/godaddy/asherah/regression/DynamoDbGlobalTableIT.java @@ -1,16 +1,7 @@ package com.godaddy.asherah.regression; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; -import com.amazonaws.services.dynamodbv2.document.DynamoDB; import com.amazonaws.services.dynamodbv2.local.main.ServerRunner; import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer; -import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; -import com.amazonaws.services.dynamodbv2.model.CreateTableRequest; -import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; -import com.amazonaws.services.dynamodbv2.model.KeyType; -import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput; -import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType; import com.godaddy.asherah.TestSetup; import com.godaddy.asherah.appencryption.Session; import com.godaddy.asherah.appencryption.SessionFactory; @@ -20,6 +11,8 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; import java.util.Arrays; @@ -43,22 +36,37 @@ public void setup() throws Exception { server.start(); // Setup client pointing to our local dynamodb - DynamoDB dynamoDbDocumentClient = new DynamoDB( - AmazonDynamoDBClientBuilder.standard() - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration( - "http://localhost:" + DYNAMO_DB_PORT, "us-west-2")) - .build()); + DynamoDbClient dynamoDbClient = + TestSetup.createDynamoDbClient("http://localhost:" + DYNAMO_DB_PORT, "us-west-2"); // Create table schema - dynamoDbDocumentClient.createTable(new CreateTableRequest() - .withTableName(TABLE_NAME) - .withKeySchema( - new KeySchemaElement(PARTITION_KEY, KeyType.HASH), - new KeySchemaElement(SORT_KEY, KeyType.RANGE)) - .withAttributeDefinitions( - new AttributeDefinition(PARTITION_KEY, ScalarAttributeType.S), - new AttributeDefinition(SORT_KEY, ScalarAttributeType.N)) - .withProvisionedThroughput(new ProvisionedThroughput(1L, 1L))); + dynamoDbClient.createTable(request -> + request + .tableName(TABLE_NAME) + .keySchema( + software.amazon.awssdk.services.dynamodb.model.KeySchemaElement.builder() + .attributeName(PARTITION_KEY) + .keyType(software.amazon.awssdk.services.dynamodb.model.KeyType.HASH) + .build(), + KeySchemaElement.builder() + .attributeName(SORT_KEY) + .keyType(KeyType.RANGE) + .build()) + .attributeDefinitions( + software.amazon.awssdk.services.dynamodb.model.AttributeDefinition.builder() + .attributeName(PARTITION_KEY) + .attributeType(software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType.S) + .build(), + AttributeDefinition.builder() + .attributeName(SORT_KEY) + .attributeType(ScalarAttributeType.N) + .build()) + .provisionedThroughput(ProvisionedThroughput.builder() + .readCapacityUnits(1L) + .writeCapacityUnits(1L) + .build())); + + dynamoDbClient.close(); } @AfterEach @@ -67,11 +75,12 @@ public void teardown() throws Exception { } private SessionFactory getSessionFactory(boolean withKeySuffix, String region) { - DynamoDbMetastoreImpl.BuildStep builder = DynamoDbMetastoreImpl.newBuilder(region) - .withEndPointConfiguration("http://localhost:" + DYNAMO_DB_PORT, "us-west-2"); + DynamoDbClient dynamoDbClient = + TestSetup.createDynamoDbClient("http://localhost:" + DYNAMO_DB_PORT, "us-west-2"); + DynamoDbMetastoreImpl.Builder builder = DynamoDbMetastoreImpl.newBuilder(region, dynamoDbClient); if (withKeySuffix) { - builder = builder.withKeySuffix(); + builder.withKeySuffix(); } DynamoDbMetastoreImpl dynamoDbMetastore = builder.build(); diff --git a/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/kms/AwsKeyManagementServiceImpl.java b/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/kms/AwsKeyManagementServiceImpl.java index e103526dc..c46ad9086 100644 --- a/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/kms/AwsKeyManagementServiceImpl.java +++ b/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/kms/AwsKeyManagementServiceImpl.java @@ -1,12 +1,13 @@ package com.godaddy.asherah.appencryption.kms; -import com.amazonaws.SdkBaseException; -import com.amazonaws.services.kms.AWSKMS; -import com.amazonaws.services.kms.model.DecryptRequest; -import com.amazonaws.services.kms.model.EncryptRequest; -import com.amazonaws.services.kms.model.EncryptResult; -import com.amazonaws.services.kms.model.GenerateDataKeyRequest; -import com.amazonaws.services.kms.model.GenerateDataKeyResult; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.kms.KmsClient; +import software.amazon.awssdk.services.kms.model.DecryptRequest; +import software.amazon.awssdk.services.kms.model.EncryptRequest; +import software.amazon.awssdk.services.kms.model.EncryptResponse; +import software.amazon.awssdk.services.kms.model.GenerateDataKeyRequest; +import software.amazon.awssdk.services.kms.model.GenerateDataKeyResponse; import com.godaddy.asherah.appencryption.exceptions.AppEncryptionException; import com.godaddy.asherah.appencryption.exceptions.KmsException; import com.godaddy.asherah.appencryption.utils.Json; @@ -109,7 +110,7 @@ else if (region2.equals(this.preferredRegion)) { regionPriorityComparator.compare(regionToArn1.getKey(), regionToArn2.getKey())) .forEach(regionToArn -> regionToArnAndClientMap.put(regionToArn.getKey(), new AwsKmsArnClient(regionToArn.getValue(), - this.awsKmsClientFactory.createAwsKmsClient(regionToArn.getKey())))); + this.awsKmsClientFactory.build(regionToArn.getKey())))); } /** @@ -131,8 +132,8 @@ public byte[] encryptKey(final CryptoKey key) { // We generate a KMS datakey (plaintext and encrypted) and encrypt its plaintext key against remaining regions. // This allows us to be able to decrypt from any of the regions locally later. - GenerateDataKeyResult dataKey = generateDataKey(this.regionToArnAndClientMap); - byte[] dataKeyPlainText = dataKey.getPlaintext().array(); + GenerateDataKeyResponse dataKey = generateDataKey(this.regionToArnAndClientMap); + byte[] dataKeyPlainText = dataKey.plaintext().asByteArrayUnsafe(); // Using thread pool just for lifetime of method. Keys should be cached anyway. Use daemons so we don't block // JVM shutdown @@ -146,13 +147,13 @@ public byte[] encryptKey(final CryptoKey key) { regionToArnAndClientMap.forEach((region, arnAndClient) -> tasks.add(() -> { // If the ARN is different than the datakey's, call encrypt since it's another region - if (!arnAndClient.arn.equals(dataKey.getKeyId())) { + if (!arnAndClient.arn.equals(dataKey.keyId())) { return encryptKeyAndBuildResult(arnAndClient.awsKmsClient, region, arnAndClient.arn, dataKeyPlainText); } else { // This is the datakey, so build kmsKey json for it - return Optional.of(buildKmsRegionKeyJson(region, dataKey.getKeyId(), - dataKey.getCiphertextBlob().array())); + return Optional.of(buildKmsRegionKeyJson(region, dataKey.keyId(), + dataKey.ciphertextBlob().asByteArrayUnsafe())); } }) ); @@ -184,23 +185,24 @@ public byte[] encryptKey(final CryptoKey key) { }); } - Optional encryptKeyAndBuildResult(final AWSKMS kmsClient, final String region, final String arn, + Optional encryptKeyAndBuildResult(final KmsClient kmsClient, final String region, final String arn, final byte[] dataKeyPlainText) { try { Timer encryptTimer = Metrics.timer(MetricsUtil.AEL_METRICS_PREFIX + ".kms.aws.encrypt." + region); return encryptTimer.record(() -> { // Note we can't wipe plaintext key till end of calling method since underlying buffer shared by all requests - EncryptRequest encryptRequest = new EncryptRequest() - .withKeyId(arn) - .withPlaintext(ByteBuffer.wrap(dataKeyPlainText)); - EncryptResult encryptResult = kmsClient.encrypt(encryptRequest); + EncryptRequest encryptRequest = EncryptRequest.builder() + .keyId(arn) + .plaintext(SdkBytes.fromByteBuffer(ByteBuffer.wrap(dataKeyPlainText))) + .build(); + EncryptResponse encryptResponse = kmsClient.encrypt(encryptRequest); - byte[] encryptedKeyEncryptionKey = encryptResult.getCiphertextBlob().array(); + byte[] encryptedKeyEncryptionKey = encryptResponse.ciphertextBlob().asByteArrayUnsafe(); return Optional.of(buildKmsRegionKeyJson(region, arn, encryptedKeyEncryptionKey)); }); } - catch (SdkBaseException e) { + catch (SdkException e) { logger.warn("Failed to encrypt generated data key via region {} KMS", region, e); // TODO Consider adding notification/CW alert return Optional.empty(); @@ -211,24 +213,25 @@ Optional encryptKeyAndBuildResult(final AWSKMS kmsClient, final Stri * Attempt to generate a KMS datakey using the first successful response using a sorted map of available KMS clients. * * @param sortedRegionToArnAndClient A sorted dictionary mapping regions and their arns and kms clients. - * @return A {@link GenerateDataKeyResult} object that contains the plain text key and the ciphertext for that key. + * @return A {@link GenerateDataKeyResponse} object that contains the plain text key and the ciphertext for that key. * @exception KmsException Throws a {@link KmsException} if we're unable to generate a datakey in any AWS region. */ - GenerateDataKeyResult generateDataKey(final Map sortedRegionToArnAndClient) { + GenerateDataKeyResponse generateDataKey(final Map sortedRegionToArnAndClient) { for (Map.Entry regionToArnAndClient : sortedRegionToArnAndClient.entrySet()) { try { Timer generateDataKeyTimer = Metrics.timer(MetricsUtil.AEL_METRICS_PREFIX + ".kms.aws.generatedatakey." + regionToArnAndClient.getKey()); return generateDataKeyTimer.record(() -> { - AWSKMS kmsClient = regionToArnAndClient.getValue().awsKmsClient; - GenerateDataKeyRequest dataKeyRequest = new GenerateDataKeyRequest() - .withKeyId(regionToArnAndClient.getValue().arn) - .withKeySpec("AES_256"); + KmsClient kmsClient = regionToArnAndClient.getValue().awsKmsClient; + GenerateDataKeyRequest dataKeyRequest = GenerateDataKeyRequest.builder() + .keyId(regionToArnAndClient.getValue().arn) + .keySpec("AES_256") + .build(); return kmsClient.generateDataKey(dataKeyRequest); }); } - catch (SdkBaseException e) { + catch (SdkException e) { logger.warn("Failed to generate data key via region {}, trying next region", regionToArnAndClient.getKey(), e); // TODO Consider adding notification/CW alert } @@ -273,7 +276,7 @@ public CryptoKey decryptKey(final byte[] keyCipherText, final Instant keyCreated decryptKmsEncryptedKey(arnAndClient.awsKmsClient, encryptedKey, keyCreated, kmsKeyEncryptionKey, revoked) ); } - catch (SdkBaseException e) { + catch (SdkException e) { logger.warn("Failed to decrypt via region {} KMS, trying next region", region, e); // TODO Consider adding notification/CW alert } @@ -299,11 +302,12 @@ List getPrioritizedKmsRegionKeyJsonList(final JSONArray kmsRegionKeyArray) .collect(Collectors.toList()); } - CryptoKey decryptKmsEncryptedKey(final AWSKMS awsKmsClient, final byte[] cipherText, final Instant keyCreated, + CryptoKey decryptKmsEncryptedKey(final KmsClient awsKmsClient, final byte[] cipherText, final Instant keyCreated, final byte[] kmsKeyEncryptionKey, final boolean revoked) { - DecryptRequest decryptRequest = new DecryptRequest() - .withCiphertextBlob(ByteBuffer.wrap(kmsKeyEncryptionKey)); - byte[] plaintextBackingBytes = awsKmsClient.decrypt(decryptRequest).getPlaintext().array(); + DecryptRequest decryptRequest = DecryptRequest.builder() + .ciphertextBlob(SdkBytes.fromByteBuffer(ByteBuffer.wrap(kmsKeyEncryptionKey))) + .build(); + byte[] plaintextBackingBytes = awsKmsClient.decrypt(decryptRequest).plaintext().asByteArrayUnsafe(); try { return crypto.decryptKey(cipherText, keyCreated, crypto.generateKeyFromBytes(plaintextBackingBytes), revoked); } @@ -314,9 +318,9 @@ CryptoKey decryptKmsEncryptedKey(final AWSKMS awsKmsClient, final byte[] cipherT static final class AwsKmsArnClient { private final String arn; - private final AWSKMS awsKmsClient; + private final KmsClient awsKmsClient; - private AwsKmsArnClient(final String arn, final AWSKMS awsKmsClient) { + private AwsKmsArnClient(final String arn, final KmsClient awsKmsClient) { this.arn = arn; this.awsKmsClient = awsKmsClient; } @@ -326,12 +330,23 @@ public static final class Builder { private final Map regionToArnMap; private final String preferredRegion; + private AwsKmsClientFactory kmsClientFactory = new AwsKmsClientFactory.DefaultAwsKmsClientFactory(); private Builder(final Map regionToArnMap, final String region) { this.regionToArnMap = regionToArnMap; this.preferredRegion = region; } + /** Specifies custom kms client factory to use. + * + * @param clientFactory The AWS KMS client factory. + * @return The current {@code BuildStep} instance. + */ + public Builder withKmsClientFactory(final AwsKmsClientFactory clientFactory) { + kmsClientFactory = clientFactory; + return this; + } + /** * Builds the {@link AwsKeyManagementServiceImpl} object. * @@ -339,7 +354,7 @@ private Builder(final Map regionToArnMap, final String region) { */ public AwsKeyManagementServiceImpl build() { return new AwsKeyManagementServiceImpl(regionToArnMap, preferredRegion, new BouncyAes256GcmCrypto(), - new AwsKmsClientFactory()); + kmsClientFactory); } } } diff --git a/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/kms/AwsKmsClientFactory.java b/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/kms/AwsKmsClientFactory.java index 752df0396..96aa494e8 100644 --- a/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/kms/AwsKmsClientFactory.java +++ b/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/kms/AwsKmsClientFactory.java @@ -1,15 +1,29 @@ package com.godaddy.asherah.appencryption.kms; -import com.amazonaws.services.kms.AWSKMS; -import com.amazonaws.services.kms.AWSKMSClientBuilder; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.kms.KmsClient; /** * A factory to create an AWS KMS client based on the region provided. */ -class AwsKmsClientFactory { - AWSKMS createAwsKmsClient(final String region) { - return AWSKMSClientBuilder.standard() - .withRegion(region) +interface AwsKmsClientFactory { + + KmsClient build(String region); + + static DefaultAwsKmsClientFactory defaultFactory() { + return new DefaultAwsKmsClientFactory(); + } + + /** + * Default implementation of {@link AwsKmsClientFactory}. + */ + class DefaultAwsKmsClientFactory implements AwsKmsClientFactory { + + @Override + public KmsClient build(final String region) { + return KmsClient.builder() + .region(Region.of(region)) .build(); + } } } diff --git a/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImpl.java b/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImpl.java index c32062af9..df56b721f 100644 --- a/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImpl.java +++ b/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImpl.java @@ -1,23 +1,13 @@ package com.godaddy.asherah.appencryption.persistence; import java.time.Instant; +import java.util.HashMap; import java.util.Iterator; +import java.util.Map; import java.util.Optional; -import com.amazonaws.SdkBaseException; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; -import com.amazonaws.services.dynamodbv2.document.DynamoDB; -import com.amazonaws.services.dynamodbv2.document.GetItemOutcome; -import com.amazonaws.services.dynamodbv2.document.Item; -import com.amazonaws.services.dynamodbv2.document.ItemCollection; -import com.amazonaws.services.dynamodbv2.document.QueryOutcome; -import com.amazonaws.services.dynamodbv2.document.Table; -import com.amazonaws.services.dynamodbv2.document.spec.GetItemSpec; -import com.amazonaws.services.dynamodbv2.document.spec.PutItemSpec; -import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec; -import com.amazonaws.services.dynamodbv2.model.ConditionalCheckFailedException; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import com.godaddy.asherah.appencryption.exceptions.AppEncryptionException; import com.godaddy.asherah.appencryption.utils.MetricsUtil; @@ -27,6 +17,12 @@ import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.ComparisonOperator; +import software.amazon.awssdk.services.dynamodb.model.Condition; +import software.amazon.awssdk.services.dynamodb.model.ConditionalCheckFailedException; +import software.amazon.awssdk.services.dynamodb.model.GetItemResponse; +import software.amazon.awssdk.services.dynamodb.model.QueryResponse; /** * Provides an AWS DynamoDB based implementation of {@link Metastore} to store and retrieve @@ -48,49 +44,49 @@ public class DynamoDbMetastoreImpl implements Metastore { Metrics.timer(MetricsUtil.AEL_METRICS_PREFIX + ".metastore.dynamodb.loadlatest"); private final Timer storeTimer = Metrics.timer(MetricsUtil.AEL_METRICS_PREFIX + ".metastore.dynamodb.store"); - private final DynamoDB client; + private final DynamoDbClient dynamoDbClient; private final String tableName; private final String preferredRegion; private final boolean hasKeySuffix; - // Table instance can be cached since thread-safe and no state other than description, which we don't use - private final Table table; + /** * Initialize a {@code DynamoDbMetastoreImpl} builder with the given region. - * - * @param region The preferred region for the DynamoDb. This can be overridden using the - * {@link Builder#withRegion(String)} builder step. + * @param preferredRegion The preferred region for the DynamoDb. This can be overridden in DynamoDB client. + * @param dynamoDbClient DynamoDB client. * @return The current {@code Builder} instance. */ - public static Builder newBuilder(final String region) { - return new Builder(region); + public static Builder newBuilder(final String preferredRegion, final DynamoDbClient dynamoDbClient) { + return new Builder(preferredRegion, dynamoDbClient); } DynamoDbMetastoreImpl(final Builder builder) { - this.client = new DynamoDB(builder.client); + this.dynamoDbClient = builder.dynamoDbClient; this.tableName = builder.tableName; this.preferredRegion = builder.preferredRegion; this.hasKeySuffix = builder.hasKeySuffix; - this.table = client.getTable(tableName); } @Override public Optional load(final String keyId, final Instant created) { return loadTimer.record(() -> { try { - GetItemOutcome outcome = table.getItemOutcome(new GetItemSpec() - .withPrimaryKey( - PARTITION_KEY, keyId, - SORT_KEY, created.getEpochSecond()) - .withProjectionExpression(ATTRIBUTE_KEY_RECORD) - .withConsistentRead(true)); // always use strong consistency - - Item item = outcome.getItem(); - if (item != null) { - return Optional.of(new JSONObject(item.getMap(ATTRIBUTE_KEY_RECORD))); + GetItemResponse getItemResponse = dynamoDbClient.getItem(request -> + request + .tableName(tableName) + .key(Map.of( + PARTITION_KEY, AttributeValue.fromS(keyId), + SORT_KEY, AttributeValue.fromN(Long.toString(created.getEpochSecond())) + )) + .attributesToGet(ATTRIBUTE_KEY_RECORD) + .consistentRead(true) // always use strong consistency + ); + Map item = getItemResponse.item(); + if (item != null && item.containsKey(ATTRIBUTE_KEY_RECORD)) { + return Optional.of(toJSONObject(item.get(ATTRIBUTE_KEY_RECORD).m())); } } - catch (SdkBaseException se) { + catch (SdkException se) { logger.error("Metastore error", se); } @@ -109,20 +105,27 @@ public Optional loadLatest(final String keyId) { return loadLatestTimer.record(() -> { try { // Have to use query api to use limit and reverse sort order - ItemCollection itemCollection = table.query(new QuerySpec() - .withHashKey(PARTITION_KEY, keyId) - .withProjectionExpression(ATTRIBUTE_KEY_RECORD) - .withScanIndexForward(false) // sorts descending - .withMaxResultSize(1) // limit 1 - .withConsistentRead(true)); // always use strong consistency - - Iterator iterator = itemCollection.iterator(); + QueryResponse queryResponse = dynamoDbClient.query(request -> + request + .tableName(tableName) + .keyConditions(Map.of( + PARTITION_KEY, Condition.builder() + .attributeValueList(AttributeValue.fromS(keyId)) + .comparisonOperator(ComparisonOperator.EQ) + .build() + )) + .attributesToGet(ATTRIBUTE_KEY_RECORD) + .scanIndexForward(false) // sorts descending + .limit(1) // limit 1 + .consistentRead(true) // limit 1 + ); + Iterator> iterator = queryResponse.items().iterator(); if (iterator.hasNext()) { - Item item = iterator.next(); - return Optional.of(new JSONObject(item.getMap(ATTRIBUTE_KEY_RECORD))); + Map item = iterator.next(); + return Optional.of(toJSONObject(item.get(ATTRIBUTE_KEY_RECORD).m())); } } - catch (SdkBaseException se) { + catch (SdkException se) { logger.error("Metastore error", se); } @@ -147,15 +150,16 @@ public boolean store(final String keyId, final Instant created, final JSONObject // sort key alone to guarantee primary key uniqueness. It automatically checks for existence of this item's // composite primary key and if it contains the specified attribute name, either of which is inherently // required. - Item item = new Item() - .withPrimaryKey( - PARTITION_KEY, keyId, - SORT_KEY, created.getEpochSecond()) - .withMap(ATTRIBUTE_KEY_RECORD, value.toMap()); - table.putItem(new PutItemSpec() - .withItem(item) - .withConditionExpression("attribute_not_exists(" + PARTITION_KEY + ")")); - + dynamoDbClient.putItem(request -> + request + .tableName(tableName) + .item(Map.of( + PARTITION_KEY, AttributeValue.fromS(keyId), + SORT_KEY, AttributeValue.fromN(Long.toString(created.getEpochSecond())), + ATTRIBUTE_KEY_RECORD, AttributeValue.fromM(toDynamoDbItem(value)) + )) + .conditionExpression("attribute_not_exists(" + PARTITION_KEY + ")") + ); return true; } catch (ConditionalCheckFailedException e) { @@ -163,13 +167,53 @@ public boolean store(final String keyId, final Instant created, final JSONObject logger.info("Attempted to create duplicate key: {} {}", keyId, created); return false; } - catch (SdkBaseException se) { + catch (SdkException se) { logger.error("Metastore error during store", se); throw new AppEncryptionException("Metastore error", se); } }); } + static Map toDynamoDbItem(final JSONObject value) { + Map result = new HashMap<>(); + for (final var key : value.keySet()) { + final var object = value.get(key); + if (object instanceof String stringObject) { + result.put(key, AttributeValue.fromS(stringObject)); + } + else if (object instanceof Long longObject) { + result.put(key, AttributeValue.fromN(Long.toString(longObject))); + } + else if (object instanceof Boolean booleanObject) { + result.put(key, AttributeValue.fromBool(booleanObject)); + } + else if (object instanceof JSONObject jsonObject) { + result.put(key, AttributeValue.fromM(toDynamoDbItem(jsonObject))); + } + } + return result; + } + + static JSONObject toJSONObject(final Map item) { + JSONObject result = new JSONObject(); + for (final var entry : item.entrySet()) { + final var value = entry.getValue(); + if (value.s() != null) { + result.put(entry.getKey(), value.s()); + } + else if (value.n() != null) { + result.put(entry.getKey(), Long.parseLong(value.n())); + } + else if (value.bool() != null) { + result.put(entry.getKey(), value.bool()); + } + else if (value.m() != null) { + result.put(entry.getKey(), toJSONObject(value.m())); + } + } + return result; + } + @Override public String getKeySuffix() { if (hasKeySuffix) { @@ -182,112 +226,54 @@ String getTableName() { return tableName; } - DynamoDB getClient() { - return client; + DynamoDbClient getClient() { + return dynamoDbClient; } - public static final class Builder implements BuildStep, EndPointStep, RegionStep { + public static final class Builder { static final String DEFAULT_TABLE_NAME = "EncryptionKey"; - private AmazonDynamoDB client; - private final String preferredRegion; - private final AmazonDynamoDBClientBuilder standardBuilder = AmazonDynamoDBClientBuilder.standard(); + private final DynamoDbClient dynamoDbClient; private String tableName = DEFAULT_TABLE_NAME; - private boolean hasEndPoint = false; private boolean hasKeySuffix = false; - private boolean hasRegion = false; - - private Builder(final String region) { - this.preferredRegion = region; - } - - @Override - public BuildStep withTableName(final String table) { - this.tableName = table; - return this; - } - - @Override - public BuildStep withEndPointConfiguration(final String endPoint, final String signingRegion) { - if (!hasRegion) { - hasEndPoint = true; - standardBuilder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endPoint, signingRegion)); - } - return this; - } - - @Override - public BuildStep withKeySuffix() { - this.hasKeySuffix = true; - return this; - } - @Override - public BuildStep withRegion(final String region) { - if (!hasEndPoint) { - hasRegion = true; - standardBuilder.withRegion(region); - } - return this; + private Builder(final String preferredRegion, final DynamoDbClient dynamoDbClient) { + this.preferredRegion = preferredRegion; + this.dynamoDbClient = dynamoDbClient; } - @Override - public DynamoDbMetastoreImpl build() { - if (!hasEndPoint && !hasRegion) { - standardBuilder.withRegion(preferredRegion); - } - client = standardBuilder.build(); - return new DynamoDbMetastoreImpl(this); - } - } - - public interface EndPointStep { - /** - * Adds Endpoint config to the AWS DynamoDb client. - * - * @param endPoint The service endpoint either with or without the protocol. - * @param signingRegion The region to use for SigV4 signing of requests (e.g. us-west-1). - * @return The current {@code BuildStep} instance with end point configuration. - */ - BuildStep withEndPointConfiguration(String endPoint, String signingRegion); - } - - public interface RegionStep { - /** - * Specifies the region for the AWS DynamoDb client. - * - * @param region The region for the DynamoDb client. - * @return The current {@code BuildStep} instance with a client region. This overrides the region specified as the - * {@code newBuilder} input parameter. - */ - BuildStep withRegion(String region); - } - - public interface BuildStep { /** * Specifies the name of the table. * - * @param tableName A custom name for the table. - * @return The current {@code BuildStep} instance. + * @param name A custom name for the table. + * @return The current {@code Builder} instance. */ - BuildStep withTableName(String tableName); + public Builder withTableName(final String name) { + this.tableName = name; + return this; + } /** * Specifies whether key suffix should be enabled for DynamoDB. This should be enabled for Global Table support. * Adding a suffix to keys prevents multi-region writes from clobbering each other and ensures that no keys are * lost. * - * @return The current {@code BuildStep} instance. + * @return The current {@code Builder} instance. */ - BuildStep withKeySuffix(); + public Builder withKeySuffix() { + this.hasKeySuffix = true; + return this; + } /** - * Builds the finalized {@code DynamoDbMetastoreImpl} with the parameters specified in the builder. + * Build the {@code DynamoDbMetastoreImpl} instance. * - * @return The fully instantiated {@code DynamoDbMetastoreImpl}. - */ - DynamoDbMetastoreImpl build(); + * @return The {@code DynamoDbMetastoreImpl} instance. + * */ + public DynamoDbMetastoreImpl build() { + return new DynamoDbMetastoreImpl(this); + } } } diff --git a/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/kms/AwsKeyManagementServiceImplTest.java b/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/kms/AwsKeyManagementServiceImplTest.java index 2b06ef524..e6c30c0c4 100644 --- a/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/kms/AwsKeyManagementServiceImplTest.java +++ b/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/kms/AwsKeyManagementServiceImplTest.java @@ -1,8 +1,9 @@ package com.godaddy.asherah.appencryption.kms; -import com.amazonaws.SdkBaseException; -import com.amazonaws.services.kms.AWSKMS; -import com.amazonaws.services.kms.model.*; +import software.amazon.awssdk.core.SdkBytes; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.kms.KmsClient; +import software.amazon.awssdk.services.kms.model.*; import com.godaddy.asherah.appencryption.exceptions.AppEncryptionException; import com.godaddy.asherah.appencryption.exceptions.KmsException; import com.godaddy.asherah.appencryption.utils.Json; @@ -40,7 +41,7 @@ class AwsKeyManagementServiceImplTest { String preferredRegion = US_WEST_1; @Mock - AWSKMS awsKmsClient; + KmsClient awsKmsClient; @Mock AeadEnvelopeCrypto crypto; @Mock @@ -54,7 +55,7 @@ class AwsKeyManagementServiceImplTest { void setUp() { // This will be fragile since it's used in the constructor itself. If unit tests need to mock different clients // being returned, they'll likely need to create their own new flavors of this mock and the main spy. - when(awsKmsClientFactory.createAwsKmsClient(any())).thenReturn(awsKmsClient); + when(awsKmsClientFactory.build(any())).thenReturn(awsKmsClient); awsKeyManagementServiceImpl = spy(new AwsKeyManagementServiceImpl(regionToArnMap, preferredRegion, crypto, awsKmsClientFactory)); @@ -139,7 +140,7 @@ void testDecryptKeyWithKmsFailureShouldThrowKmsException() { )); // apparently not needed since we tell the call that uses it to fail anyway? //when(awsKmsClientFactory.createAwsKmsClient(any())).thenReturn(awsKmsClient); - doThrow(SdkBaseException.class) + doThrow(SdkException.class) .when(awsKeyManagementServiceImpl).decryptKmsEncryptedKey(any(), any(), any(), any(), anyBoolean()); assertThrows(KmsException.class, @@ -175,8 +176,10 @@ void testDecryptKmsEncryptedKeySuccessful() { byte[] keyEncryptionKey = new byte[]{2, 3}; boolean revoked = false; byte[] plaintextBackingBytes = new byte[]{4, 5}; - DecryptResult decryptResult = new DecryptResult().withPlaintext(ByteBuffer.wrap(plaintextBackingBytes)); - when(awsKmsClient.decrypt(any())).thenReturn(decryptResult); + DecryptResponse decryptResponse = DecryptResponse.builder() + .plaintext(SdkBytes.fromByteArrayUnsafe(plaintextBackingBytes)) + .build(); + when(awsKmsClient.decrypt(any(DecryptRequest.class))).thenReturn(decryptResponse); when(crypto.generateKeyFromBytes(plaintextBackingBytes)).thenReturn(cryptoKey); CryptoKey expectedKey = mock(CryptoKey.class); when(crypto.decryptKey(cipherText, now, cryptoKey, revoked)).thenReturn(expectedKey); @@ -191,9 +194,9 @@ void testDecryptKmsEncryptedKeySuccessful() { void testDecryptKmsEncryptedKeyWithKmsFailureShouldThrowException() { byte[] cipherText = new byte[]{0, 1}; byte[] keyEncryptionKey = new byte[]{2, 3}; - when(awsKmsClient.decrypt(any())).thenThrow(SdkBaseException.class); + when(awsKmsClient.decrypt(any(DecryptRequest.class))).thenThrow(SdkException.class); - assertThrows(SdkBaseException.class, + assertThrows(SdkException.class, () -> awsKeyManagementServiceImpl .decryptKmsEncryptedKey(awsKmsClient, cipherText, Instant.now(), keyEncryptionKey, false)); } @@ -205,8 +208,10 @@ void testDecryptKmsEncryptedKeyWithCryptoFailureShouldThrowExceptionAndWipeBytes byte[] keyEncryptionKey = new byte[]{2, 3}; boolean revoked = false; byte[] plaintextBackingBytes = new byte[]{4, 5}; - DecryptResult decryptResult = new DecryptResult().withPlaintext(ByteBuffer.wrap(plaintextBackingBytes)); - when(awsKmsClient.decrypt(any())).thenReturn(decryptResult); + DecryptResponse decryptResponse = DecryptResponse.builder() + .plaintext(SdkBytes.fromByteArrayUnsafe(plaintextBackingBytes)) + .build(); + when(awsKmsClient.decrypt(any(DecryptRequest.class))).thenReturn(decryptResponse); when(crypto.generateKeyFromBytes(plaintextBackingBytes)).thenReturn(cryptoKey); when(crypto.decryptKey(cipherText, now, cryptoKey, revoked)).thenThrow(AppEncryptionException.class); @@ -228,20 +233,21 @@ void testPrimaryBuilderPath() { @Test void testGenerateDataKeySuccessful() { Map sortedRegionToArnAndClient = awsKeyManagementServiceImpl.getRegionToArnAndClientMap(); - GenerateDataKeyRequest expectedRequest = new GenerateDataKeyRequest() - .withKeyId(ARN_US_WEST_1) // preferred regions's ARN, verify it's the first and hence returned - .withKeySpec("AES_256"); - GenerateDataKeyResult dataKeyResult = mock(GenerateDataKeyResult.class); - - when(awsKmsClient.generateDataKey(eq(expectedRequest))).thenReturn(dataKeyResult); - GenerateDataKeyResult dataKeyResultActual = awsKeyManagementServiceImpl.generateDataKey(sortedRegionToArnAndClient); - assertEquals(dataKeyResult, dataKeyResultActual); + GenerateDataKeyRequest expectedRequest = GenerateDataKeyRequest.builder() + .keyId(ARN_US_WEST_1) // preferred regions's ARN, verify it's the first and hence returned + .keySpec("AES_256") + .build(); + GenerateDataKeyResponse dataKeyResponse = mock(GenerateDataKeyResponse.class); + + when(awsKmsClient.generateDataKey(eq(expectedRequest))).thenReturn(dataKeyResponse); + GenerateDataKeyResponse dataKeyResponseActual = awsKeyManagementServiceImpl.generateDataKey(sortedRegionToArnAndClient); + assertEquals(dataKeyResponse, dataKeyResponseActual); } @Test void testGenerateDataKeyWithKmsFailureShouldThrowKmsException() { Map sortedRegionToArnAndClient = awsKeyManagementServiceImpl.getRegionToArnAndClientMap(); - when(awsKmsClient.generateDataKey(any(GenerateDataKeyRequest.class))).thenThrow(SdkBaseException.class); + when(awsKmsClient.generateDataKey(any(GenerateDataKeyRequest.class))).thenThrow(SdkException.class); assertThrows(KmsException.class, () -> awsKeyManagementServiceImpl.generateDataKey(sortedRegionToArnAndClient)); @@ -250,9 +256,11 @@ void testGenerateDataKeyWithKmsFailureShouldThrowKmsException() { @Test void testEncryptKeyAndBuildResult() { byte[] encryptedKey = new byte[] {0,1}; - EncryptResult encryptResult = new EncryptResult().withCiphertextBlob(ByteBuffer.wrap(encryptedKey)); + EncryptResponse encryptResponse = EncryptResponse.builder() + .ciphertextBlob(SdkBytes.fromByteArrayUnsafe(encryptedKey)) + .build(); - when(awsKmsClient.encrypt(any())).thenReturn(encryptResult); + when(awsKmsClient.encrypt(any(EncryptRequest.class))).thenReturn(encryptResponse); byte[] dataKeyPlainText = new byte[] {2,3}; Optional actualResult = awsKeyManagementServiceImpl @@ -265,7 +273,7 @@ void testEncryptKeyAndBuildResult() { @Test void testEncryptKeyAndBuildResultReturnEmptyOptional() { byte[] dataKeyPlainText = new byte[] {0,1}; - when(awsKmsClient.encrypt(any())).thenThrow(SdkBaseException.class); + when(awsKmsClient.encrypt(any(EncryptRequest.class))).thenThrow(SdkException.class); Optional actualResult = awsKeyManagementServiceImpl .encryptKeyAndBuildResult(awsKmsClient, preferredRegion, ARN_US_WEST_1, dataKeyPlainText); @@ -297,18 +305,18 @@ void testEncryptKeySuccessful() { ENCRYPTED_KEY, Base64.getEncoder().encodeToString(encryptedKey) )); - GenerateDataKeyResult dataKeyMock = mock(GenerateDataKeyResult.class); + GenerateDataKeyResponse dataKeyMock = mock(GenerateDataKeyResponse.class); CryptoKey generatedDataKeyCryptoKey = mock(CryptoKey.class); when(awsKeyManagementServiceImpl.generateDataKey(awsKeyManagementServiceImpl.getRegionToArnAndClientMap())) .thenReturn(dataKeyMock); - when(dataKeyMock.getPlaintext()).thenReturn(dataKeyPlainText); + when(dataKeyMock.plaintext()).thenReturn(SdkBytes.fromByteArrayUnsafe(dataKeyPlainText.array())); when(crypto.generateKeyFromBytes(dataKeyPlainText.array())).thenReturn(generatedDataKeyCryptoKey); when(crypto.encryptKey(cryptoKey, generatedDataKeyCryptoKey)).thenReturn(encryptedKey); - when(dataKeyMock.getKeyId()).thenReturn(ARN_US_WEST_1); + when(dataKeyMock.keyId()).thenReturn(ARN_US_WEST_1); doReturn(Optional.of(encryptKeyAndBuildResultJson)).when(awsKeyManagementServiceImpl) .encryptKeyAndBuildResult(eq(awsKmsClient), eq(US_EAST_1), eq(ARN_US_EAST_1), eq(dataKeyPlainText.array())); - when(dataKeyMock.getCiphertextBlob()).thenReturn(dataKeyCipherText); + when(dataKeyMock.ciphertextBlob()).thenReturn(SdkBytes.fromByteArrayUnsafe(dataKeyCipherText.array())); byte[] encryptedResult = awsKeyManagementServiceImpl.encryptKey(cryptoKey); Json kmsKeyEnvelopeResult = new Json(encryptedResult); @@ -322,12 +330,12 @@ void testEncryptKeyShouldThrowExceptionAndWipeBytes() { ByteBuffer dataKeyPlainText = ByteBuffer.wrap(new byte[] {1,2}); byte[] encryptedKey = new byte[] {3,4}; - GenerateDataKeyResult dataKeyMock = mock(GenerateDataKeyResult.class); + GenerateDataKeyResponse dataKeyMock = mock(GenerateDataKeyResponse.class); CryptoKey generatedDataKeyCryptoKey = mock(CryptoKey.class); when(awsKeyManagementServiceImpl.generateDataKey(awsKeyManagementServiceImpl.getRegionToArnAndClientMap())) .thenReturn(dataKeyMock); - when(dataKeyMock.getPlaintext()).thenReturn(dataKeyPlainText); + when(dataKeyMock.plaintext()).thenReturn(SdkBytes.fromByteArrayUnsafe(dataKeyPlainText.array())); when(crypto.generateKeyFromBytes(dataKeyPlainText.array())).thenReturn(generatedDataKeyCryptoKey); // Inject unexpected exception so the CompleteableFuture.join throws a CompletionException doThrow(RuntimeException.class).when(awsKeyManagementServiceImpl) diff --git a/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImplTest.java b/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImplTest.java index 9f1ec4980..cb81d7b94 100644 --- a/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImplTest.java +++ b/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImplTest.java @@ -1,11 +1,20 @@ package com.godaddy.asherah.appencryption.persistence; import java.math.BigDecimal; +import java.net.URI; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Map; import java.util.Optional; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; +import com.amazonaws.services.dynamodbv2.document.DynamoDB; +import com.amazonaws.services.dynamodbv2.document.Item; +import com.amazonaws.services.dynamodbv2.document.Table; import org.json.JSONObject; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; @@ -13,23 +22,16 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.SDKGlobalConfiguration; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; -import com.amazonaws.services.dynamodbv2.document.DynamoDB; -import com.amazonaws.services.dynamodbv2.document.Item; -import com.amazonaws.services.dynamodbv2.document.Table; -import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; -import com.amazonaws.services.dynamodbv2.model.CreateTableRequest; -import com.amazonaws.services.dynamodbv2.model.KeySchemaElement; -import com.amazonaws.services.dynamodbv2.model.KeyType; -import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughput; -import com.amazonaws.services.dynamodbv2.model.ScalarAttributeType; import com.amazonaws.services.dynamodbv2.local.main.ServerRunner; import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer; import com.godaddy.asherah.appencryption.exceptions.AppEncryptionException; import com.google.common.collect.ImmutableMap; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; import static com.godaddy.asherah.appencryption.persistence.DynamoDbMetastoreImpl.*; import static org.junit.jupiter.api.Assertions.*; @@ -42,19 +44,21 @@ class DynamoDbMetastoreImplTest { static final String TEST_KEY_WITH_SUFFIX = "some_key_" + REGION; static DynamoDBProxyServer server; - // Note we need to use BigDecimals here to make the asserts play nice. SDK is always converting numbers to BigDecimal final Map keyRecord = ImmutableMap.of( "ParentKeyMeta", ImmutableMap.of( "KeyId", "_SK_api_ecomm", - "Created", new BigDecimal(1541461380)), + "Created", 1541461380L), "Key", "mWT/x4RvIFVFE2BEYV1IB9FMM8sWN1sK6YN5bS2UyGR+9RSZVTvp/bcQ6PycW6kxYEqrpA+aV4u04jOr", - "Created", new BigDecimal(1541461381)); + "Created", 1541461381L); final Instant instant = Instant.now().minus(1, ChronoUnit.DAYS); - Table table; - DynamoDB dynamoDbDocumentClient; + final AwsCredentialsProvider testCredentialsProvider = StaticCredentialsProvider.create( + AwsBasicCredentials.create("test", "test")); + DynamoDbMetastoreImpl dynamoDbMetastoreImpl; + DynamoDbClient dynamoDbClient; + String tableName; @BeforeAll public static void setupClass() throws Exception { @@ -72,46 +76,55 @@ public static void teardownClass() throws Exception { @BeforeEach void setUp() { - dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.newBuilder(REGION) - .withEndPointConfiguration("http://localhost:" + DYNAMO_DB_PORT, "us-west-2") + dynamoDbClient = DynamoDbClient.builder() + .region(Region.of(REGION)) + .endpointOverride(URI.create("http://localhost:" + DYNAMO_DB_PORT)) + .credentialsProvider(testCredentialsProvider) + .build(); + dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.newBuilder(REGION, dynamoDbClient) .build(); - dynamoDbDocumentClient = dynamoDbMetastoreImpl.getClient(); + tableName = dynamoDbMetastoreImpl.getTableName(); // Create table schema - createTableSchema(dynamoDbDocumentClient, dynamoDbMetastoreImpl.getTableName()); + createTableSchema(dynamoDbClient, dynamoDbMetastoreImpl.getTableName()); - table = dynamoDbDocumentClient.getTable(dynamoDbMetastoreImpl.getTableName()); - Item item = new Item() - .withPrimaryKey( - PARTITION_KEY, TEST_KEY, - SORT_KEY, instant.getEpochSecond()) - .withMap(ATTRIBUTE_KEY_RECORD, keyRecord); - table.putItem(item); - Item itemWithSuffix = new Item() - .withPrimaryKey( - PARTITION_KEY, TEST_KEY_WITH_SUFFIX, - SORT_KEY, instant.getEpochSecond()) - .withMap(ATTRIBUTE_KEY_RECORD, keyRecord); - table.putItem(itemWithSuffix); + putItem(TEST_KEY, instant.getEpochSecond(), keyRecord); + putItem(TEST_KEY_WITH_SUFFIX, instant.getEpochSecond(), keyRecord); } - public void createTableSchema(final DynamoDB client,final String tableName) { + public void createTableSchema(final DynamoDbClient dynamoDbClient, final String tableName) { // Create table schema - client.createTable(new CreateTableRequest() - .withTableName(tableName) - .withKeySchema( - new KeySchemaElement(PARTITION_KEY, KeyType.HASH), - new KeySchemaElement(SORT_KEY, KeyType.RANGE)) - .withAttributeDefinitions( - new AttributeDefinition(PARTITION_KEY, ScalarAttributeType.S), - new AttributeDefinition(SORT_KEY, ScalarAttributeType.N)) - .withProvisionedThroughput(new ProvisionedThroughput(1L, 1L))); + dynamoDbClient.createTable(request -> + request + .tableName(tableName) + .keySchema( + KeySchemaElement.builder() + .attributeName(PARTITION_KEY) + .keyType(KeyType.HASH) + .build(), + KeySchemaElement.builder() + .attributeName(SORT_KEY) + .keyType(KeyType.RANGE) + .build()) + .attributeDefinitions( + AttributeDefinition.builder() + .attributeName(PARTITION_KEY) + .attributeType(ScalarAttributeType.S) + .build(), + AttributeDefinition.builder() + .attributeName(SORT_KEY) + .attributeType(ScalarAttributeType.N) + .build()) + .provisionedThroughput(ProvisionedThroughput.builder() + .readCapacityUnits(1L) + .writeCapacityUnits(1L) + .build())); } @AfterEach void tearDown() { // Blow out the whole table so we have clean slate each time - dynamoDbDocumentClient.getTable(dynamoDbMetastoreImpl.getTableName()).delete(); + dynamoDbClient.deleteTable(request -> request.tableName(dynamoDbMetastoreImpl.getTableName())); } @Test @@ -145,8 +158,7 @@ void testLoadLatestWithSingleRecord() { @Test void testLoadLatestWithSingleRecordAndSuffix() { - DynamoDbMetastoreImpl dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.newBuilder(REGION) - .withEndPointConfiguration("http://localhost:" + DYNAMO_DB_PORT, "us-west-2") + DynamoDbMetastoreImpl dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.newBuilder(REGION, dynamoDbClient) .withKeySuffix() .build(); Optional actualJsonObject = dynamoDbMetastoreImpl.loadLatest(TEST_KEY); @@ -161,35 +173,16 @@ void testLoadLatestWithMultipleRecords() { Instant instantPlusOneHour = instant.plus(1, ChronoUnit.HOURS); Instant instantMinusOneDay = instant.minus(1, ChronoUnit.DAYS); Instant instantPlusOneDay = instant.plus(1, ChronoUnit.DAYS); - Item itemMinusOneHour = new Item() - .withPrimaryKey( - PARTITION_KEY, TEST_KEY, - SORT_KEY, instantMinusOneHour.getEpochSecond()) - .withMap(ATTRIBUTE_KEY_RECORD, ImmutableMap.of( - "mytime", instantMinusOneHour.getEpochSecond())); - Item itemPlusOneHour = new Item() - .withPrimaryKey( - PARTITION_KEY, TEST_KEY, - SORT_KEY, instantPlusOneHour.getEpochSecond()) - .withMap(ATTRIBUTE_KEY_RECORD, ImmutableMap.of( - "mytime", instantPlusOneHour.getEpochSecond())); - Item itemMinusOneDay = new Item() - .withPrimaryKey( - PARTITION_KEY, TEST_KEY, - SORT_KEY, instantMinusOneDay.getEpochSecond()) - .withMap(ATTRIBUTE_KEY_RECORD, ImmutableMap.of( - "mytime", instantMinusOneDay.getEpochSecond())); - Item itemPlusOneDay = new Item() - .withPrimaryKey( - PARTITION_KEY, TEST_KEY, - SORT_KEY, instantPlusOneDay.getEpochSecond()) - .withMap(ATTRIBUTE_KEY_RECORD, ImmutableMap.of( - "mytime", instantPlusOneDay.getEpochSecond())); + // intentionally mixing up insertion order - table.putItem(itemPlusOneHour); - table.putItem(itemPlusOneDay); - table.putItem(itemMinusOneHour); - table.putItem(itemMinusOneDay); + putItem(TEST_KEY, instantPlusOneHour.getEpochSecond(), ImmutableMap.of( + "mytime", instantPlusOneHour.getEpochSecond())); + putItem(TEST_KEY, instantMinusOneDay.getEpochSecond(), ImmutableMap.of( + "mytime", instantMinusOneDay.getEpochSecond())); + putItem(TEST_KEY, instantMinusOneHour.getEpochSecond(), ImmutableMap.of( + "mytime", instantMinusOneHour.getEpochSecond())); + putItem(TEST_KEY, instantPlusOneDay.getEpochSecond(), ImmutableMap.of( + "mytime", instantPlusOneDay.getEpochSecond())); Optional actualJsonObject = dynamoDbMetastoreImpl.loadLatest(TEST_KEY); @@ -213,8 +206,7 @@ void testLoadLatestWithFailureShouldReturnEmpty() { @Test void testStoreSuccess() { - DynamoDbMetastoreImpl dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.newBuilder(REGION) - .withEndPointConfiguration("http://localhost:" + DYNAMO_DB_PORT, "us-west-2") + DynamoDbMetastoreImpl dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.newBuilder(REGION, dynamoDbClient) .withKeySuffix() .build(); boolean actualValue = dynamoDbMetastoreImpl.store(TEST_KEY, Instant.now(), new JSONObject(keyRecord)); @@ -245,35 +237,9 @@ void testStoreWithFailureShouldThrowException() { () -> dynamoDbMetastoreImpl.store(null, Instant.now(), new JSONObject())); } - @Test - void testPrimaryBuilderPath() { - // Hack to inject default region since we don't explicitly require one be specified as we do in KMS impl - System.setProperty(SDKGlobalConfiguration.AWS_REGION_SYSTEM_PROPERTY, "us-west-2"); - DynamoDbMetastoreImpl dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.newBuilder(REGION).build(); - - assertNotNull(dynamoDbMetastoreImpl); - } - - @Test - void testBuilderPathWithRegion() { - DynamoDbMetastoreImpl dynamoDbMetastoreImpl = - DynamoDbMetastoreImpl.newBuilder(REGION).withRegion("us-west-1").build(); - - assertNotNull(dynamoDbMetastoreImpl); - } - - @Test - void testBuilderPathWithEndPointConfiguration() { - DynamoDbMetastoreImpl dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.newBuilder(REGION) - .withEndPointConfiguration("http://localhost:" + DYNAMO_DB_PORT, "us-west-2") - .build(); - - assertNotNull(dynamoDbMetastoreImpl); - } - @Test void testBuilderPathWithKeySuffix() { - DynamoDbMetastoreImpl dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder(REGION) + DynamoDbMetastoreImpl dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder(REGION, dynamoDbClient) .withKeySuffix() .build(); @@ -282,7 +248,7 @@ void testBuilderPathWithKeySuffix() { @Test void testBuilderPathWithoutKeySuffix() { - DynamoDbMetastoreImpl dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder(REGION) + DynamoDbMetastoreImpl dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder(REGION, dynamoDbClient) .build(); assertEquals("", dynamoDbMetastore.getKeySuffix()); @@ -292,26 +258,13 @@ void testBuilderPathWithoutKeySuffix() { void testBuilderPathWithTableName() { String tableName = "DummyTable"; - // Use AWS SDK to create client - AmazonDynamoDB client = AmazonDynamoDBClientBuilder.standard() - .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://localhost:8000", "us-west-2")) - .build(); - DynamoDB dynamoDBclient = new DynamoDB(client); - - createTableSchema(dynamoDBclient, tableName); + createTableSchema(dynamoDbClient, tableName); // Put the object in DummyTable - Table table = dynamoDBclient.getTable(tableName); - Item item = new Item() - .withPrimaryKey( - PARTITION_KEY, TEST_KEY, - SORT_KEY, instant.getEpochSecond()) - .withMap(ATTRIBUTE_KEY_RECORD, keyRecord); - table.putItem(item); + putItem(tableName, TEST_KEY, instant.getEpochSecond(), keyRecord); // Create a metastore object using the withTableName step - DynamoDbMetastoreImpl dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder(REGION) - .withEndPointConfiguration("http://localhost:8000", "us-west-2") + DynamoDbMetastoreImpl dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder(REGION, dynamoDbClient) .withTableName(tableName) .build(); Optional actualJsonObject = dynamoDbMetastore.load(TEST_KEY, instant); @@ -321,4 +274,56 @@ void testBuilderPathWithTableName() { assertTrue(actualJsonObject.isPresent()); assertEquals(keyRecord, actualJsonObject.get().toMap()); } + + @Test + void testBackwardCompatibility() { + // use sdk v1 to store keyrecord + AmazonDynamoDB client = AmazonDynamoDBClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider( + new BasicAWSCredentials("test", "test"))) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration("http://localhost:8000", "us-west-2")) + .build(); + + Table table = new DynamoDB(client).getTable(tableName); + + final Map keyRecordBeforeUpgrade = ImmutableMap.of( + "ParentKeyMeta", ImmutableMap.of( + "KeyId", "_SK_api_ecomm", + "Created", new BigDecimal(1541461380L)), + "Key", "mWT/x4RvIFVFE2BEYV1IB9FMM8sWN1sK6YN5bS2UyGR+9RSZVTvp/bcQ6PycW6kxYEqrpA+aV4u04jOr", + "Created", new BigDecimal(1541461380L)); + + Item item = new Item() + .withPrimaryKey( + PARTITION_KEY, "backward-test-key", + SORT_KEY, instant.getEpochSecond()) + .withMap(ATTRIBUTE_KEY_RECORD, keyRecordBeforeUpgrade); + table.putItem(item); + + // use sdk v2 to load keyrecord + JSONObject actualJsonObject = dynamoDbMetastoreImpl.load("backward-test-key", instant).orElse(null); + + // verify + assertNotNull(actualJsonObject); + assertEquals(actualJsonObject.get("Key"), keyRecordBeforeUpgrade.get("Key")); + assertEquals(actualJsonObject.get("Created"), ((BigDecimal) keyRecordBeforeUpgrade.get("Created")).longValueExact()); + } + + + private void putItem(String partitionKey, Long sortKey, Map keyRecord) { + putItem(tableName, partitionKey, sortKey, keyRecord); + } + + private void putItem(String tableName, String partitionKey, Long sortKey, Map keyRecord) { + dynamoDbClient.putItem(request -> + request + .tableName(tableName) + .item(Map.of( + PARTITION_KEY, AttributeValue.fromS(partitionKey), + SORT_KEY, AttributeValue.fromN(Long.toString(sortKey)), + ATTRIBUTE_KEY_RECORD, AttributeValue.fromM(DynamoDbMetastoreImpl.toDynamoDbItem(new JSONObject(keyRecord))) + )) + ); + } } diff --git a/samples/java/reference-app/pom.xml b/samples/java/reference-app/pom.xml index c29bc46fa..c2c2e8f16 100644 --- a/samples/java/reference-app/pom.xml +++ b/samples/java/reference-app/pom.xml @@ -12,7 +12,7 @@ 1.8 UTF-8 - 0.2.5 + 0.3.0 5.9.3 1.9.3 0.8.10 diff --git a/samples/java/reference-app/src/main/java/com/godaddy/asherah/referenceapp/App.java b/samples/java/reference-app/src/main/java/com/godaddy/asherah/referenceapp/App.java index 574ccf33b..f4f0e93e1 100644 --- a/samples/java/reference-app/src/main/java/com/godaddy/asherah/referenceapp/App.java +++ b/samples/java/reference-app/src/main/java/com/godaddy/asherah/referenceapp/App.java @@ -1,5 +1,6 @@ package com.godaddy.asherah.referenceapp; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Map; @@ -40,6 +41,9 @@ import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.Option; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder; @Command(mixinStandardHelpOptions = true, description = "Runs an example end-to-end encryption and decryption round " + "trip. NOTE: Some metastore and kms types depend on required " @@ -125,17 +129,20 @@ private Metastore setupMetastore() { dynamoDbRegion = "us-west-2"; logger.info("configuring metastore with default region: {}.", dynamoDbRegion); } - DynamoDbMetastoreImpl.Builder builder = DynamoDbMetastoreImpl.newBuilder(dynamoDbRegion); // Check for region and endpoint configuration. // The client can use either withRegion or withEndPointConfiguration but not both + DynamoDbClientBuilder dynamoDbClientBuilder = DynamoDbClient.builder(); if (dynamoDbEndpoint != null) { if (dynamoDbRegion == null || dynamoDbRegion.trim().isEmpty() || dynamoDbEndpoint.trim().isEmpty()) { logger.error("One or more parameter(s) for endpoint configuration missing."); return null; } - builder.withEndPointConfiguration(dynamoDbEndpoint, dynamoDbRegion); + dynamoDbClientBuilder.endpointOverride(URI.create(dynamoDbEndpoint)).region(Region.of(dynamoDbRegion)); } + + DynamoDbMetastoreImpl.Builder builder = + DynamoDbMetastoreImpl.newBuilder(dynamoDbRegion, dynamoDbClientBuilder.build()); // Check for table name if (dynamoDbTableName != null) { if (dynamoDbTableName.trim().isEmpty()) { diff --git a/server/java/pom.xml b/server/java/pom.xml index fe7cd081d..f8acf3a77 100644 --- a/server/java/pom.xml +++ b/server/java/pom.xml @@ -17,7 +17,7 @@ 5.9.3 1.54.1 3.22.3 - 0.2.5 + 0.3.0 4.7.3 2.10.1 20230227 diff --git a/server/java/src/main/java/com/godaddy/asherah/grpc/AppEncryptionConfig.java b/server/java/src/main/java/com/godaddy/asherah/grpc/AppEncryptionConfig.java index 90bf8f734..49711146e 100644 --- a/server/java/src/main/java/com/godaddy/asherah/grpc/AppEncryptionConfig.java +++ b/server/java/src/main/java/com/godaddy/asherah/grpc/AppEncryptionConfig.java @@ -13,7 +13,11 @@ import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder; +import java.net.URI; import java.util.Map; class AppEncryptionConfig { @@ -82,17 +86,17 @@ Metastore setupMetastore(final String metastoreType, final String jd logger.info("configuring metastore with default region: {}.", region); } - DynamoDbMetastoreImpl.Builder builder = DynamoDbMetastoreImpl.newBuilder(region); + DynamoDbClientBuilder dynamoDbClientBuilder = DynamoDbClient.builder() + .region(Region.of(region)); // Check for region and endpoint configuration. - // The client can use either withRegion or withEndPointConfiguration but not both if (endPoint != null) { - if (region == null || region.trim().isEmpty() || endPoint.trim().isEmpty()) { - logger.error("One or more parameter(s) for endpoint configuration missing."); - return null; - } - builder.withEndPointConfiguration(endPoint, region); + dynamoDbClientBuilder.endpointOverride(URI.create(endPoint)); } + + DynamoDbMetastoreImpl.Builder builder = + DynamoDbMetastoreImpl.newBuilder(region, dynamoDbClientBuilder.build()); + //Check for table name if (tableName != null) { if (tableName.trim().isEmpty()) { diff --git a/server/java/src/test/java/com/godaddy/asherah/grpc/AppEncryptionConfigTest.java b/server/java/src/test/java/com/godaddy/asherah/grpc/AppEncryptionConfigTest.java index 0067f038a..48112045d 100644 --- a/server/java/src/test/java/com/godaddy/asherah/grpc/AppEncryptionConfigTest.java +++ b/server/java/src/test/java/com/godaddy/asherah/grpc/AppEncryptionConfigTest.java @@ -69,7 +69,7 @@ void testDynamoDbWithEmptyEndpointConfigurationSetupMetastore() { @Test void testDynamoDbWithEndpointConfigurationSetupMetastore() { - DynamoDbConfig dynamoDbConfig = new DynamoDbConfig("endPoint", "us-west-2", false, null); + DynamoDbConfig dynamoDbConfig = new DynamoDbConfig("https://endPoint", "us-west-2", false, null); Metastore metastore = appEncryptionConfig.setupMetastore("dyNaModb", null, dynamoDbConfig); assertNotNull(metastore); @@ -78,7 +78,7 @@ void testDynamoDbWithEndpointConfigurationSetupMetastore() { @Test void testDynamoDbWithTableNameSetupMetastore() { - DynamoDbConfig dynamoDbConfig = new DynamoDbConfig("endPoint", "us-west-2", false, "CustomTableName"); + DynamoDbConfig dynamoDbConfig = new DynamoDbConfig("https://endPoint", "us-west-2", false, "CustomTableName"); Metastore metastore = appEncryptionConfig.setupMetastore("dyNaModb", null, dynamoDbConfig); assertNotNull(metastore); @@ -87,7 +87,7 @@ void testDynamoDbWithTableNameSetupMetastore() { @Test void testDynamoDbWithEmptyTableNameSetupMetastoreReturnsNull() { - DynamoDbConfig dynamoDbConfig = new DynamoDbConfig("endPoint", "us-west-2", false, ""); + DynamoDbConfig dynamoDbConfig = new DynamoDbConfig("https://endPoint", "us-west-2", false, ""); Metastore metastore = appEncryptionConfig.setupMetastore("dyNaModb", null, dynamoDbConfig); assertNull(metastore); From 174db0d686381fe06bd6c3143e859413ebc0cbc9 Mon Sep 17 00:00:00 2001 From: Artyom Gabeev Date: Thu, 25 May 2023 16:17:41 +0300 Subject: [PATCH 2/6] Failfast on conversion --- java/app-encryption/README.md | 3 ++- .../appencryption/persistence/DynamoDbMetastoreImpl.java | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/java/app-encryption/README.md b/java/app-encryption/README.md index 95469c139..a5384173d 100644 --- a/java/app-encryption/README.md +++ b/java/app-encryption/README.md @@ -42,7 +42,8 @@ You can specify the current release of Asherah as a project dependency using the Starting with version 0.3.0 Asherah uses AWS SDK v2. AWS SDK both major versions can be used in the same project, and v2 dependencies will be provided as transitive. There is single breaking change around DynamoDB metastore configuration: -Previous DynamoDB client was initialized withing metastore builder, now it should be provided as a parameter. + +Previously DynamoDB client was initialized withing metastore builder, now it should be provided as a parameter. Assuming you had previously used `DynamoDbMetastoreImpl`, you can replace it with new equivalent: ```java diff --git a/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImpl.java b/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImpl.java index df56b721f..5202e510d 100644 --- a/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImpl.java +++ b/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImpl.java @@ -190,6 +190,9 @@ else if (object instanceof Boolean booleanObject) { else if (object instanceof JSONObject jsonObject) { result.put(key, AttributeValue.fromM(toDynamoDbItem(jsonObject))); } + else { + throw new IllegalArgumentException("Unsupported type: " + object.getClass().getName()); + } } return result; } @@ -210,6 +213,9 @@ else if (value.bool() != null) { else if (value.m() != null) { result.put(entry.getKey(), toJSONObject(value.m())); } + else { + throw new IllegalArgumentException("Unsupported type: " + value.type()); + } } return result; } From ef7e5855fd760f5c20c1b271075db4f8eb04ed37 Mon Sep 17 00:00:00 2001 From: Artyom Gabeev Date: Thu, 25 May 2023 16:22:22 +0300 Subject: [PATCH 3/6] Refactor --- .../regression/DynamoDbGlobalTableIT.java | 16 +++++++----- .../DynamoDbMetastoreImplTest.java | 26 ++++++++++++------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/java/app-encryption/src/it/java/com/godaddy/asherah/regression/DynamoDbGlobalTableIT.java b/java/app-encryption/src/it/java/com/godaddy/asherah/regression/DynamoDbGlobalTableIT.java index 6a9828786..bea0f4038 100644 --- a/java/app-encryption/src/it/java/com/godaddy/asherah/regression/DynamoDbGlobalTableIT.java +++ b/java/app-encryption/src/it/java/com/godaddy/asherah/regression/DynamoDbGlobalTableIT.java @@ -8,11 +8,15 @@ import com.godaddy.asherah.appencryption.persistence.DynamoDbMetastoreImpl; import com.godaddy.asherah.utils.PayloadGenerator; import com.godaddy.asherah.utils.SessionFactoryGenerator; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; +import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement; +import software.amazon.awssdk.services.dynamodb.model.KeyType; +import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput; +import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.*; import java.util.Arrays; @@ -44,18 +48,18 @@ public void setup() throws Exception { request .tableName(TABLE_NAME) .keySchema( - software.amazon.awssdk.services.dynamodb.model.KeySchemaElement.builder() + KeySchemaElement.builder() .attributeName(PARTITION_KEY) - .keyType(software.amazon.awssdk.services.dynamodb.model.KeyType.HASH) + .keyType(KeyType.HASH) .build(), KeySchemaElement.builder() .attributeName(SORT_KEY) .keyType(KeyType.RANGE) .build()) .attributeDefinitions( - software.amazon.awssdk.services.dynamodb.model.AttributeDefinition.builder() + AttributeDefinition.builder() .attributeName(PARTITION_KEY) - .attributeType(software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType.S) + .attributeType(ScalarAttributeType.S) .build(), AttributeDefinition.builder() .attributeName(SORT_KEY) diff --git a/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImplTest.java b/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImplTest.java index cb81d7b94..721523b0b 100644 --- a/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImplTest.java +++ b/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImplTest.java @@ -15,23 +15,29 @@ import com.amazonaws.services.dynamodbv2.document.DynamoDB; import com.amazonaws.services.dynamodbv2.document.Item; import com.amazonaws.services.dynamodbv2.document.Table; -import org.json.JSONObject; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - import com.amazonaws.services.dynamodbv2.local.main.ServerRunner; import com.amazonaws.services.dynamodbv2.local.server.DynamoDBProxyServer; -import com.godaddy.asherah.appencryption.exceptions.AppEncryptionException; -import com.google.common.collect.ImmutableMap; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.*; +import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement; +import software.amazon.awssdk.services.dynamodb.model.KeyType; +import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput; +import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; +import com.godaddy.asherah.appencryption.exceptions.AppEncryptionException; +import com.google.common.collect.ImmutableMap; +import org.json.JSONObject; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + + import static com.godaddy.asherah.appencryption.persistence.DynamoDbMetastoreImpl.*; import static org.junit.jupiter.api.Assertions.*; From 36dd456b6e1d94fa223eea27f4b7c5982321838d Mon Sep 17 00:00:00 2001 From: Artyom Gabeev Date: Thu, 1 Jun 2023 15:36:54 +0300 Subject: [PATCH 4/6] Install during ci --- .github/workflows/ci.yml | 4 ++++ java/app-encryption/scripts/install.sh | 5 +++++ 2 files changed, 9 insertions(+) create mode 100755 java/app-encryption/scripts/install.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95717a446..3279ba946 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,10 @@ jobs: sudo prlimit --pid $$ --core=-1 sudo prlimit --pid $$ --memlock=-1:-1 ./scripts/integration_test.sh + - name: Install + run: | + cd java/app-encryption + ./scripts/install.sh release-java-app-Encryption: if: startsWith( github.ref, 'refs/heads/release-' ) && github.repository == 'godaddy/asherah' diff --git a/java/app-encryption/scripts/install.sh b/java/app-encryption/scripts/install.sh new file mode 100755 index 000000000..6cd7181e1 --- /dev/null +++ b/java/app-encryption/scripts/install.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -e + +mvn --no-transfer-progress install -DskipTests From 83a285efac3fce4688e1d44aef30842f9eb1ca4a Mon Sep 17 00:00:00 2001 From: Artyom Gabeev Date: Fri, 2 Jun 2023 12:28:28 +0300 Subject: [PATCH 5/6] Code review changes --- java/app-encryption/README.md | 37 +---- java/app-encryption/scripts/install.sh | 5 - .../regression/DynamoDbGlobalTableIT.java | 3 +- .../persistence/DynamoDbMetastoreImpl.java | 138 ++++++++++++++---- .../DynamoDbMetastoreImplTest.java | 18 ++- 5 files changed, 133 insertions(+), 68 deletions(-) delete mode 100755 java/app-encryption/scripts/install.sh diff --git a/java/app-encryption/README.md b/java/app-encryption/README.md index a5384173d..e67d1f8eb 100644 --- a/java/app-encryption/README.md +++ b/java/app-encryption/README.md @@ -4,7 +4,6 @@ Application level envelope encryption SDK for Java with support for cloud-agnost [![Version](https://img.shields.io/maven-central/v/com.godaddy.asherah/appencryption)](https://mvnrepository.com/artifact/com.godaddy.asherah/appencryption) * [Installation](#installation) - * [Upgrade](#upgrade) * [Quick Start](#quick-start) * [How to Use Asherah](#how-to-use-asherah) * [Define the Metastore](#define-the-metastore) @@ -36,27 +35,6 @@ You can specify the current release of Asherah as a project dependency using the ``` -## Upgrade - -### 0.3.x - -Starting with version 0.3.0 Asherah uses AWS SDK v2. AWS SDK both major versions can be used in the same project, -and v2 dependencies will be provided as transitive. There is single breaking change around DynamoDB metastore configuration: - -Previously DynamoDB client was initialized withing metastore builder, now it should be provided as a parameter. - -Assuming you had previously used `DynamoDbMetastoreImpl`, you can replace it with new equivalent: -```java -// before 0.3.0 -final var metastore = DynamoDbMetastoreImpl.newBuilder("us-west-2").build(); - -// after -final var dynamoDbClient = DynamoDBClient.builder().region(Region.US_WEST_2).build(); -final var metastore = DynamoDbMetastoreImpl.newBuilder("us-west-2", dynamoDbClient).build(); -``` - -Region specified two times intentionally, to support Global Tables and prevent key collisions. - ## Quick Start ```java @@ -109,11 +87,7 @@ For simplicity, the DynamoDB implementation uses the builder pattern to enable c To obtain an instance of the builder, use the static factory method `newBuilder`. ```java -// Create dynamo db client -DynamoDbClient dynamoDbClient = ...; - -// Build the DynamoDB Metastore -DynamoDbMetastoreImpl.newBuilder("us-west-2", dynamoDbClient).build(); +DynamoDbMetastoreImpl.newBuilder(); ``` Once you have a builder, you can either use the `withXXX` setter methods to configure the metastore properties or simply build the metastore by calling the `build` method. @@ -121,11 +95,14 @@ build the metastore by calling the `build` method. - **withKeySuffix**: Specifies whether key suffix should be enabled for DynamoDB. **This is required to enable Global Tables.** - **withTableName**: Specifies the name of the DynamoDb table. + - **withRegion**: Specifies the region for the AWS DynamoDb client. + - **withEndPointConfiguration**: Adds an EndPoint configuration to the AWS DynamoDb client. + - **withClientOverride**: Specifies a custom AWS DynamoDb client. Region and endpoint configuration will be ignored if custom client is provided. Below is an example of a DynamoDB metastore that uses a Global Table named `TestTable` ```java -Metastore dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder() +Metastore dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder("us-west-2") .withTableName("TestTable") .withKeySuffix() .build(); @@ -153,7 +130,7 @@ Map regionMap = ImmutableMap.of("us-east-1", "arn_of_us-east-1", KeyManagementService keyManagementService = AwsKeyManagementServiceImpl.newBuilder(regionMap, "us-east-1").build(); ``` -Starting with version 0.3.0 it is possible to specify AWS KMS client factory to be used: +It is possible to specify AWS KMS client factory to be used, instead of default one: ```java // Define AWS KMS client factory AwsKmsClientFactory awsKmsClientFactory = ...; @@ -162,7 +139,7 @@ AwsKmsClientFactory awsKmsClientFactory = ...; KeyManagementService keyManagementService = AwsKeyManagementServiceImpl.newBuilder(regionMap, "us-east-1") .withAwsKmsClientFactory(awsKmsClientFactory) .build(); -``` +``` #### Static KMS (FOR TESTING ONLY) diff --git a/java/app-encryption/scripts/install.sh b/java/app-encryption/scripts/install.sh deleted file mode 100755 index 6cd7181e1..000000000 --- a/java/app-encryption/scripts/install.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -set -e - -mvn --no-transfer-progress install -DskipTests diff --git a/java/app-encryption/src/it/java/com/godaddy/asherah/regression/DynamoDbGlobalTableIT.java b/java/app-encryption/src/it/java/com/godaddy/asherah/regression/DynamoDbGlobalTableIT.java index bea0f4038..477282b6b 100644 --- a/java/app-encryption/src/it/java/com/godaddy/asherah/regression/DynamoDbGlobalTableIT.java +++ b/java/app-encryption/src/it/java/com/godaddy/asherah/regression/DynamoDbGlobalTableIT.java @@ -81,7 +81,8 @@ public void teardown() throws Exception { private SessionFactory getSessionFactory(boolean withKeySuffix, String region) { DynamoDbClient dynamoDbClient = TestSetup.createDynamoDbClient("http://localhost:" + DYNAMO_DB_PORT, "us-west-2"); - DynamoDbMetastoreImpl.Builder builder = DynamoDbMetastoreImpl.newBuilder(region, dynamoDbClient); + DynamoDbMetastoreImpl.Builder builder = DynamoDbMetastoreImpl.newBuilder(region); + builder.withClientOverride(dynamoDbClient); if (withKeySuffix) { builder.withKeySuffix(); diff --git a/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImpl.java b/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImpl.java index 5202e510d..1ff87a45c 100644 --- a/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImpl.java +++ b/java/app-encryption/src/main/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImpl.java @@ -1,5 +1,6 @@ package com.godaddy.asherah.appencryption.persistence; +import java.net.URI; import java.time.Instant; import java.util.HashMap; import java.util.Iterator; @@ -7,6 +8,7 @@ import java.util.Optional; import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import com.godaddy.asherah.appencryption.exceptions.AppEncryptionException; import com.godaddy.asherah.appencryption.utils.MetricsUtil; @@ -17,6 +19,7 @@ import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.ComparisonOperator; import software.amazon.awssdk.services.dynamodb.model.Condition; @@ -52,12 +55,13 @@ public class DynamoDbMetastoreImpl implements Metastore { /** * Initialize a {@code DynamoDbMetastoreImpl} builder with the given region. - * @param preferredRegion The preferred region for the DynamoDb. This can be overridden in DynamoDB client. - * @param dynamoDbClient DynamoDB client. + * + * @param region The preferred region for the DynamoDb. This can be overridden using the + * {@link Builder#withRegion(String)} builder step. * @return The current {@code Builder} instance. */ - public static Builder newBuilder(final String preferredRegion, final DynamoDbClient dynamoDbClient) { - return new Builder(preferredRegion, dynamoDbClient); + public static Builder newBuilder(final String region) { + return new Builder(region); } DynamoDbMetastoreImpl(final Builder builder) { @@ -236,50 +240,132 @@ DynamoDbClient getClient() { return dynamoDbClient; } - public static final class Builder { + public static final class Builder implements BuildStep, EndPointStep, RegionStep { static final String DEFAULT_TABLE_NAME = "EncryptionKey"; + private DynamoDbClient dynamoDbClient; + private final String preferredRegion; - private final DynamoDbClient dynamoDbClient; + private final DynamoDbClientBuilder clientBuilder = DynamoDbClient.builder(); private String tableName = DEFAULT_TABLE_NAME; + private boolean hasEndPoint = false; private boolean hasKeySuffix = false; + private boolean hasRegion = false; + + private DynamoDbClient clientOverride; + + private Builder(final String region) { + this.preferredRegion = region; + } + + @Override + public BuildStep withTableName(final String table) { + this.tableName = table; + return this; + } + + @Override + public BuildStep withEndPointConfiguration(final String endPoint, final String signingRegion) { + if (!hasRegion) { + hasEndPoint = true; + clientBuilder + .endpointOverride(URI.create(endPoint)) + .region(Region.of(signingRegion)); + } + return this; + } + + @Override + public BuildStep withKeySuffix() { + this.hasKeySuffix = true; + return this; + } + + @Override + public BuildStep withRegion(final String region) { + if (!hasEndPoint) { + hasRegion = true; + clientBuilder.region(Region.of(region)); + } + return this; + } - private Builder(final String preferredRegion, final DynamoDbClient dynamoDbClient) { - this.preferredRegion = preferredRegion; - this.dynamoDbClient = dynamoDbClient; + @Override + public BuildStep withClientOverride(final DynamoDbClient client) { + this.clientOverride = client; + return this; + } + + @Override + public DynamoDbMetastoreImpl build() { + if (clientOverride != null) { + dynamoDbClient = clientOverride; + return new DynamoDbMetastoreImpl(this); + } + if (!hasEndPoint && !hasRegion) { + clientBuilder.region(Region.of(preferredRegion)); + } + dynamoDbClient = clientBuilder.build(); + return new DynamoDbMetastoreImpl(this); } + } + + public interface EndPointStep { + /** + * Adds Endpoint config to the AWS DynamoDb client. + * + * @param endPoint The service endpoint either with or without the protocol. + * @param signingRegion The region to use for SigV4 signing of requests (e.g. us-west-1). + * @return The current {@code BuildStep} instance with end point configuration. + */ + BuildStep withEndPointConfiguration(String endPoint, String signingRegion); + } + public interface RegionStep { + /** + * Specifies the region for the AWS DynamoDb client. + * + * @param region The region for the DynamoDb client. + * @return The current {@code BuildStep} instance with a client region. This overrides the region specified as the + * {@code newBuilder} input parameter. + */ + BuildStep withRegion(String region); + } + + public interface BuildStep { /** * Specifies the name of the table. * - * @param name A custom name for the table. - * @return The current {@code Builder} instance. + * @param tableName A custom name for the table. + * @return The current {@code BuildStep} instance. */ - public Builder withTableName(final String name) { - this.tableName = name; - return this; - } + BuildStep withTableName(String tableName); /** * Specifies whether key suffix should be enabled for DynamoDB. This should be enabled for Global Table support. * Adding a suffix to keys prevents multi-region writes from clobbering each other and ensures that no keys are * lost. * - * @return The current {@code Builder} instance. + * @return The current {@code BuildStep} instance. */ - public Builder withKeySuffix() { - this.hasKeySuffix = true; - return this; - } + BuildStep withKeySuffix(); + /** - * Build the {@code DynamoDbMetastoreImpl} instance. + * Specifies a custom DynamoDb client to be used instead of the default. Custom client will ignore any region + * or endpoint configuration specified in the builder. * - * @return The {@code DynamoDbMetastoreImpl} instance. - * */ - public DynamoDbMetastoreImpl build() { - return new DynamoDbMetastoreImpl(this); - } + * @param client A custom DynamoDb client. + * @return The current {@code BuildStep} instance. + */ + BuildStep withClientOverride(DynamoDbClient client); + + /** + * Builds the finalized {@code DynamoDbMetastoreImpl} with the parameters specified in the builder. + * + * @return The fully instantiated {@code DynamoDbMetastoreImpl}. + */ + DynamoDbMetastoreImpl build(); } } diff --git a/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImplTest.java b/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImplTest.java index 721523b0b..b6993373f 100644 --- a/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImplTest.java +++ b/java/app-encryption/src/test/java/com/godaddy/asherah/appencryption/persistence/DynamoDbMetastoreImplTest.java @@ -87,7 +87,8 @@ void setUp() { .endpointOverride(URI.create("http://localhost:" + DYNAMO_DB_PORT)) .credentialsProvider(testCredentialsProvider) .build(); - dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.newBuilder(REGION, dynamoDbClient) + dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.newBuilder(REGION) + .withClientOverride(dynamoDbClient) .build(); tableName = dynamoDbMetastoreImpl.getTableName(); @@ -164,7 +165,8 @@ void testLoadLatestWithSingleRecord() { @Test void testLoadLatestWithSingleRecordAndSuffix() { - DynamoDbMetastoreImpl dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.newBuilder(REGION, dynamoDbClient) + DynamoDbMetastoreImpl dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.newBuilder(REGION) + .withClientOverride(dynamoDbClient) .withKeySuffix() .build(); Optional actualJsonObject = dynamoDbMetastoreImpl.loadLatest(TEST_KEY); @@ -212,7 +214,8 @@ void testLoadLatestWithFailureShouldReturnEmpty() { @Test void testStoreSuccess() { - DynamoDbMetastoreImpl dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.newBuilder(REGION, dynamoDbClient) + DynamoDbMetastoreImpl dynamoDbMetastoreImpl = DynamoDbMetastoreImpl.newBuilder(REGION) + .withClientOverride(dynamoDbClient) .withKeySuffix() .build(); boolean actualValue = dynamoDbMetastoreImpl.store(TEST_KEY, Instant.now(), new JSONObject(keyRecord)); @@ -245,7 +248,8 @@ void testStoreWithFailureShouldThrowException() { @Test void testBuilderPathWithKeySuffix() { - DynamoDbMetastoreImpl dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder(REGION, dynamoDbClient) + DynamoDbMetastoreImpl dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder(REGION) + .withClientOverride(dynamoDbClient) .withKeySuffix() .build(); @@ -254,7 +258,8 @@ void testBuilderPathWithKeySuffix() { @Test void testBuilderPathWithoutKeySuffix() { - DynamoDbMetastoreImpl dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder(REGION, dynamoDbClient) + DynamoDbMetastoreImpl dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder(REGION) + .withClientOverride(dynamoDbClient) .build(); assertEquals("", dynamoDbMetastore.getKeySuffix()); @@ -270,7 +275,8 @@ void testBuilderPathWithTableName() { putItem(tableName, TEST_KEY, instant.getEpochSecond(), keyRecord); // Create a metastore object using the withTableName step - DynamoDbMetastoreImpl dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder(REGION, dynamoDbClient) + DynamoDbMetastoreImpl dynamoDbMetastore = DynamoDbMetastoreImpl.newBuilder(REGION) + .withClientOverride(dynamoDbClient) .withTableName(tableName) .build(); Optional actualJsonObject = dynamoDbMetastore.load(TEST_KEY, instant); From 953e40193640c65b2437ab47d2e52bf30df94690 Mon Sep 17 00:00:00 2001 From: Artyom Gabeev Date: Fri, 2 Jun 2023 12:29:17 +0300 Subject: [PATCH 6/6] Revert ci and samples --- .github/workflows/ci.yml | 4 ---- samples/java/reference-app/pom.xml | 2 +- .../com/godaddy/asherah/referenceapp/App.java | 11 ++--------- server/java/pom.xml | 2 +- .../asherah/grpc/AppEncryptionConfig.java | 18 +++++++----------- .../asherah/grpc/AppEncryptionConfigTest.java | 6 +++--- 6 files changed, 14 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3279ba946..95717a446 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,10 +118,6 @@ jobs: sudo prlimit --pid $$ --core=-1 sudo prlimit --pid $$ --memlock=-1:-1 ./scripts/integration_test.sh - - name: Install - run: | - cd java/app-encryption - ./scripts/install.sh release-java-app-Encryption: if: startsWith( github.ref, 'refs/heads/release-' ) && github.repository == 'godaddy/asherah' diff --git a/samples/java/reference-app/pom.xml b/samples/java/reference-app/pom.xml index c2c2e8f16..c29bc46fa 100644 --- a/samples/java/reference-app/pom.xml +++ b/samples/java/reference-app/pom.xml @@ -12,7 +12,7 @@ 1.8 UTF-8 - 0.3.0 + 0.2.5 5.9.3 1.9.3 0.8.10 diff --git a/samples/java/reference-app/src/main/java/com/godaddy/asherah/referenceapp/App.java b/samples/java/reference-app/src/main/java/com/godaddy/asherah/referenceapp/App.java index f4f0e93e1..574ccf33b 100644 --- a/samples/java/reference-app/src/main/java/com/godaddy/asherah/referenceapp/App.java +++ b/samples/java/reference-app/src/main/java/com/godaddy/asherah/referenceapp/App.java @@ -1,6 +1,5 @@ package com.godaddy.asherah.referenceapp; -import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Map; @@ -41,9 +40,6 @@ import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.Option; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder; @Command(mixinStandardHelpOptions = true, description = "Runs an example end-to-end encryption and decryption round " + "trip. NOTE: Some metastore and kms types depend on required " @@ -129,20 +125,17 @@ private Metastore setupMetastore() { dynamoDbRegion = "us-west-2"; logger.info("configuring metastore with default region: {}.", dynamoDbRegion); } + DynamoDbMetastoreImpl.Builder builder = DynamoDbMetastoreImpl.newBuilder(dynamoDbRegion); // Check for region and endpoint configuration. // The client can use either withRegion or withEndPointConfiguration but not both - DynamoDbClientBuilder dynamoDbClientBuilder = DynamoDbClient.builder(); if (dynamoDbEndpoint != null) { if (dynamoDbRegion == null || dynamoDbRegion.trim().isEmpty() || dynamoDbEndpoint.trim().isEmpty()) { logger.error("One or more parameter(s) for endpoint configuration missing."); return null; } - dynamoDbClientBuilder.endpointOverride(URI.create(dynamoDbEndpoint)).region(Region.of(dynamoDbRegion)); + builder.withEndPointConfiguration(dynamoDbEndpoint, dynamoDbRegion); } - - DynamoDbMetastoreImpl.Builder builder = - DynamoDbMetastoreImpl.newBuilder(dynamoDbRegion, dynamoDbClientBuilder.build()); // Check for table name if (dynamoDbTableName != null) { if (dynamoDbTableName.trim().isEmpty()) { diff --git a/server/java/pom.xml b/server/java/pom.xml index f8acf3a77..fe7cd081d 100644 --- a/server/java/pom.xml +++ b/server/java/pom.xml @@ -17,7 +17,7 @@ 5.9.3 1.54.1 3.22.3 - 0.3.0 + 0.2.5 4.7.3 2.10.1 20230227 diff --git a/server/java/src/main/java/com/godaddy/asherah/grpc/AppEncryptionConfig.java b/server/java/src/main/java/com/godaddy/asherah/grpc/AppEncryptionConfig.java index 49711146e..90bf8f734 100644 --- a/server/java/src/main/java/com/godaddy/asherah/grpc/AppEncryptionConfig.java +++ b/server/java/src/main/java/com/godaddy/asherah/grpc/AppEncryptionConfig.java @@ -13,11 +13,7 @@ import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.DynamoDbClientBuilder; -import java.net.URI; import java.util.Map; class AppEncryptionConfig { @@ -86,17 +82,17 @@ Metastore setupMetastore(final String metastoreType, final String jd logger.info("configuring metastore with default region: {}.", region); } - DynamoDbClientBuilder dynamoDbClientBuilder = DynamoDbClient.builder() - .region(Region.of(region)); + DynamoDbMetastoreImpl.Builder builder = DynamoDbMetastoreImpl.newBuilder(region); // Check for region and endpoint configuration. + // The client can use either withRegion or withEndPointConfiguration but not both if (endPoint != null) { - dynamoDbClientBuilder.endpointOverride(URI.create(endPoint)); + if (region == null || region.trim().isEmpty() || endPoint.trim().isEmpty()) { + logger.error("One or more parameter(s) for endpoint configuration missing."); + return null; + } + builder.withEndPointConfiguration(endPoint, region); } - - DynamoDbMetastoreImpl.Builder builder = - DynamoDbMetastoreImpl.newBuilder(region, dynamoDbClientBuilder.build()); - //Check for table name if (tableName != null) { if (tableName.trim().isEmpty()) { diff --git a/server/java/src/test/java/com/godaddy/asherah/grpc/AppEncryptionConfigTest.java b/server/java/src/test/java/com/godaddy/asherah/grpc/AppEncryptionConfigTest.java index 48112045d..0067f038a 100644 --- a/server/java/src/test/java/com/godaddy/asherah/grpc/AppEncryptionConfigTest.java +++ b/server/java/src/test/java/com/godaddy/asherah/grpc/AppEncryptionConfigTest.java @@ -69,7 +69,7 @@ void testDynamoDbWithEmptyEndpointConfigurationSetupMetastore() { @Test void testDynamoDbWithEndpointConfigurationSetupMetastore() { - DynamoDbConfig dynamoDbConfig = new DynamoDbConfig("https://endPoint", "us-west-2", false, null); + DynamoDbConfig dynamoDbConfig = new DynamoDbConfig("endPoint", "us-west-2", false, null); Metastore metastore = appEncryptionConfig.setupMetastore("dyNaModb", null, dynamoDbConfig); assertNotNull(metastore); @@ -78,7 +78,7 @@ void testDynamoDbWithEndpointConfigurationSetupMetastore() { @Test void testDynamoDbWithTableNameSetupMetastore() { - DynamoDbConfig dynamoDbConfig = new DynamoDbConfig("https://endPoint", "us-west-2", false, "CustomTableName"); + DynamoDbConfig dynamoDbConfig = new DynamoDbConfig("endPoint", "us-west-2", false, "CustomTableName"); Metastore metastore = appEncryptionConfig.setupMetastore("dyNaModb", null, dynamoDbConfig); assertNotNull(metastore); @@ -87,7 +87,7 @@ void testDynamoDbWithTableNameSetupMetastore() { @Test void testDynamoDbWithEmptyTableNameSetupMetastoreReturnsNull() { - DynamoDbConfig dynamoDbConfig = new DynamoDbConfig("https://endPoint", "us-west-2", false, ""); + DynamoDbConfig dynamoDbConfig = new DynamoDbConfig("endPoint", "us-west-2", false, ""); Metastore metastore = appEncryptionConfig.setupMetastore("dyNaModb", null, dynamoDbConfig); assertNull(metastore);