From abe0adb330065f44eaacc22d6fa53fedb8d8579a Mon Sep 17 00:00:00 2001 From: Jan Duzinkiewicz Date: Mon, 18 Mar 2024 12:51:05 -0700 Subject: [PATCH 1/3] support for s3 express one zone --- gradle/verification-metadata.xml | 2596 +++++++++++++++-- .../build/gradle/s3buildcache/S3BuildCache.kt | 1 + .../gradle/s3buildcache/S3StorageService.kt | 22 +- 3 files changed, 2387 insertions(+), 232 deletions(-) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 5041f98..ba5bd0e 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1,258 +1,2404 @@ - true - true - armored - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + false + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - + + + + + + - - - - + + + - - - - + + + - - - + + + + + + - - - + + + - - - + + + - - - + + + + + + - - - + + + + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3BuildCache.kt b/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3BuildCache.kt index ee637ed..2240bbc 100644 --- a/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3BuildCache.kt +++ b/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3BuildCache.kt @@ -31,6 +31,7 @@ abstract class S3BuildCache : RemoteGradleBuildCache() { /** * Whether to use reduced redundancy. + * When using S3 Express One Zone, set to false * @see Reduced Redundancy * */ var reducedRedundancy: Boolean = true diff --git a/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3StorageService.kt b/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3StorageService.kt index 46d9530..9185c2e 100644 --- a/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3StorageService.kt +++ b/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3StorageService.kt @@ -21,13 +21,15 @@ import androidx.build.gradle.core.FileHandleInputStream import androidx.build.gradle.core.FileHandleInputStream.Companion.handleInputStream import androidx.build.gradle.core.StorageService import org.gradle.api.logging.Logging +import software.amazon.awssdk.core.exception.SdkException import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.model.DeleteObjectRequest import software.amazon.awssdk.services.s3.model.GetObjectRequest +import software.amazon.awssdk.services.s3.model.HeadBucketRequest +import software.amazon.awssdk.services.s3.model.NoSuchBucketException import software.amazon.awssdk.services.s3.model.PutObjectRequest import software.amazon.awssdk.services.s3.model.StorageClass.REDUCED_REDUNDANCY -import software.amazon.awssdk.services.s3.model.StorageClass.STANDARD import java.io.InputStream import kotlin.io.path.outputStream @@ -74,7 +76,11 @@ class S3StorageService( val request = PutObjectRequest.builder() .bucket(bucketName) .key(cacheKey) - .storageClass(if (reducedRedundancy) REDUCED_REDUNDANCY else STANDARD) + .apply { + if (reducedRedundancy) { + storageClass(REDUCED_REDUNDANCY) + } + } .build() logger.info("Storing $cacheKey via $request") return store(client, request, contents) @@ -101,11 +107,13 @@ class S3StorageService( override fun validateConfiguration() { try { - val buckets = client.listBuckets().buckets() - if (buckets.none { bucket -> bucket.name() == bucketName }) { - throw Exception("Bucket $bucketName under project $region cannot be found or is not accessible using the provided credentials") - } - } catch (e: Exception) { + client.headBucket(HeadBucketRequest.builder().bucket(bucketName).build()) + } catch(e: NoSuchBucketException) { + throw Exception("Bucket $bucketName in $region cannot be found: ${e.message}") + } catch (e: SdkException ) { + throw Exception("AWS SDK exception on validating access to $bucketName in $region: ${e.message}") + } + catch (e: Exception) { logger.warn("Couldn't validate S3 client config: ${e.message}") } } From 577e732741d147f1164546ce450d603a0d65e7fd Mon Sep 17 00:00:00 2001 From: Jan Duzinkiewicz Date: Fri, 5 Apr 2024 13:35:23 -0700 Subject: [PATCH 2/3] narrowing down aws sdk exception types that hard stop a build so that it can function offline --- .../androidx/build/gradle/s3buildcache/S3StorageService.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3StorageService.kt b/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3StorageService.kt index 9185c2e..21354f6 100644 --- a/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3StorageService.kt +++ b/s3buildcache/src/main/kotlin/androidx/build/gradle/s3buildcache/S3StorageService.kt @@ -21,7 +21,9 @@ import androidx.build.gradle.core.FileHandleInputStream import androidx.build.gradle.core.FileHandleInputStream.Companion.handleInputStream import androidx.build.gradle.core.StorageService import org.gradle.api.logging.Logging +import software.amazon.awssdk.core.exception.SdkClientException import software.amazon.awssdk.core.exception.SdkException +import software.amazon.awssdk.core.exception.SdkServiceException import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.model.DeleteObjectRequest @@ -110,8 +112,8 @@ class S3StorageService( client.headBucket(HeadBucketRequest.builder().bucket(bucketName).build()) } catch(e: NoSuchBucketException) { throw Exception("Bucket $bucketName in $region cannot be found: ${e.message}") - } catch (e: SdkException ) { - throw Exception("AWS SDK exception on validating access to $bucketName in $region: ${e.message}") + } catch (e: SdkServiceException ) { + throw Exception("AWS SDK exception on validating access to $bucketName in $region: ${e::class.simpleName} - ${e.message}") } catch (e: Exception) { logger.warn("Couldn't validate S3 client config: ${e.message}") From d5713da14c962ec21b0dc3a064fba3f2c44f4aef Mon Sep 17 00:00:00 2001 From: Jan Duzinkiewicz Date: Thu, 11 Apr 2024 12:38:51 -0700 Subject: [PATCH 3/3] finishing main rebase --- gradle/verification-metadata.xml | 2592 +++--------------------------- 1 file changed, 223 insertions(+), 2369 deletions(-) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index ba5bd0e..5041f98 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1,2404 +1,258 @@ + true - false + true + armored + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - + + + + - - - - - - + + + + - - - - - - + + + + - - - - - - + + + - - - - - - + + + - - - + + + - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +