From 32be7eb445d8e1d9661bf7de2495e6669f712384 Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Sun, 8 May 2022 19:33:44 -0400 Subject: [PATCH 1/4] update --- build.gradle.kts | 2 +- .../s3/s34k/minio/BucketObjectContainer.kt | 29 ++++++++++++++++++- .../lib/s3/s34k/minio/util/minio-builders.kt | 3 +- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index ed5a529..efbdc2e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -38,7 +38,7 @@ dependencies { implementation("org.slf4j:slf4j-api:1.7.36") implementation("io.minio:minio:8.3.8") - api("org.veupathdb.lib.s3:s34k-core:0.2.0+s34k-0.3.0") + api("org.veupathdb.lib.s3:s34k-core:0.4.0-SNAPSHOT") { isChanging = true } testImplementation(kotlin("test")) } diff --git a/src/main/kotlin/org/veupathdb/lib/s3/s34k/minio/BucketObjectContainer.kt b/src/main/kotlin/org/veupathdb/lib/s3/s34k/minio/BucketObjectContainer.kt index 31765d4..19539ef 100644 --- a/src/main/kotlin/org/veupathdb/lib/s3/s34k/minio/BucketObjectContainer.kt +++ b/src/main/kotlin/org/veupathdb/lib/s3/s34k/minio/BucketObjectContainer.kt @@ -235,7 +235,7 @@ internal class BucketObjectContainer( return out } - override fun listAll(params: ObjectListParams): ObjectList { + override fun listAll(params: ObjectListAllParams): ObjectList { log.debug("Attempting to list all objects in {}", bucket) val out = try { @@ -260,6 +260,33 @@ internal class BucketObjectContainer( return out } + override fun list(params: ObjectListParams): ObjectList { + log.debug("Attempting to list objects in {}", bucket) + + val res = try { + minio.listObjects(ListObjectsArgs.builder() + .bucket(bucket) + .region(params, bucket) + .recursive(true) + .optPrefix(params.region) + .headers(params.headers) + .queryParams(params.queryParams) + .build()) + .toStream() + .map(Result::get) + } catch (e: Throwable) { + e.throwCorrect { "Failed to fetch object list from $bucket" } + } + + // If they set the recursive flag, then we are gonna given them all of it. + if (params.recursive) { + return BasicObjectList(res.map{ MObject(it.objectName(), bucket.region, MHeaders(), bucket, minio) }.toIterable()) + } + + // They didn't set the recursive flag, so we have some work to do + val pref = if (params.prefix.isNullOrBlank()) 0 + } + override fun open(path: String, params: ObjectOpenParams): StreamObject? { log.debug("Attempting to open object '{}' in {}", path, bucket) diff --git a/src/main/kotlin/org/veupathdb/lib/s3/s34k/minio/util/minio-builders.kt b/src/main/kotlin/org/veupathdb/lib/s3/s34k/minio/util/minio-builders.kt index c2aee47..8cfc900 100644 --- a/src/main/kotlin/org/veupathdb/lib/s3/s34k/minio/util/minio-builders.kt +++ b/src/main/kotlin/org/veupathdb/lib/s3/s34k/minio/util/minio-builders.kt @@ -55,7 +55,8 @@ internal inline fun , A: ObjectArgs> B.region(params // endregion Object Args -internal inline fun ListObjectsArgs.Builder.optPrefix(prefix: String?) = also { prefix?.let(this::prefix) } +internal inline fun ListObjectsArgs.Builder.optPrefix(prefix: String?) = + also { if (!prefix.isNullOrBlank()) prefix(prefix) } internal inline fun PutObjectArgs.Builder.optContentType(ct: String?) = also { ct?.let(this::contentType) } From ddb78ee053ac48ce91d35380f3526ac5f147e5c1 Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Mon, 9 May 2022 09:23:44 -0400 Subject: [PATCH 2/4] simplify --- .../s3/s34k/minio/BucketObjectContainer.kt | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/src/main/kotlin/org/veupathdb/lib/s3/s34k/minio/BucketObjectContainer.kt b/src/main/kotlin/org/veupathdb/lib/s3/s34k/minio/BucketObjectContainer.kt index 19539ef..a7acfd0 100644 --- a/src/main/kotlin/org/veupathdb/lib/s3/s34k/minio/BucketObjectContainer.kt +++ b/src/main/kotlin/org/veupathdb/lib/s3/s34k/minio/BucketObjectContainer.kt @@ -263,28 +263,23 @@ internal class BucketObjectContainer( override fun list(params: ObjectListParams): ObjectList { log.debug("Attempting to list objects in {}", bucket) - val res = try { - minio.listObjects(ListObjectsArgs.builder() - .bucket(bucket) - .region(params, bucket) - .recursive(true) - .optPrefix(params.region) - .headers(params.headers) - .queryParams(params.queryParams) - .build()) - .toStream() - .map(Result::get) + return try { + BasicObjectList( + minio.listObjects(ListObjectsArgs.builder() + .bucket(bucket) + .region(params, bucket) + .recursive(true) + .optPrefix(params.region) + .headers(params.headers) + .queryParams(params.queryParams) + .build()) + .toStream() + .map(Result::get) + .map { MObject(it.objectName(), bucket.region, MHeaders(), bucket, minio) } + .toIterable()) } catch (e: Throwable) { e.throwCorrect { "Failed to fetch object list from $bucket" } } - - // If they set the recursive flag, then we are gonna given them all of it. - if (params.recursive) { - return BasicObjectList(res.map{ MObject(it.objectName(), bucket.region, MHeaders(), bucket, minio) }.toIterable()) - } - - // They didn't set the recursive flag, so we have some work to do - val pref = if (params.prefix.isNullOrBlank()) 0 } override fun open(path: String, params: ObjectOpenParams): StreamObject? { From 1e2e98b0bbd004db49eae161a7e028d2f3b4be8c Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Mon, 9 May 2022 09:36:04 -0400 Subject: [PATCH 3/4] version --- build.gradle.kts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index efbdc2e..11e8aae 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { } group = "org.veupathdb.lib.s3" -version = "0.1.0+s34k-0.3.0" +version = "0.2.0+s34k-0.4.0" java { sourceCompatibility = JavaVersion.VERSION_1_8 @@ -36,9 +36,10 @@ dependencies { implementation(kotlin("stdlib")) implementation(kotlin("stdlib-jdk8")) implementation("org.slf4j:slf4j-api:1.7.36") - implementation("io.minio:minio:8.3.8") + implementation("io.minio:minio:8.3.9") - api("org.veupathdb.lib.s3:s34k-core:0.4.0-SNAPSHOT") { isChanging = true } + @Suppress("GradlePackageVersionRange") + api("org.veupathdb.lib.s3:s34k-core:0.3.0+s34k-0.4.0") testImplementation(kotlin("test")) } From f17a65cdc95550fcb7aa1cc2ef6a52787516b45c Mon Sep 17 00:00:00 2001 From: Elizabeth Date: Mon, 9 May 2022 09:49:43 -0400 Subject: [PATCH 4/4] update test build --- test/build.gradle.kts | 3 ++- test/src/main/resources/log4j2.xml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/build.gradle.kts b/test/build.gradle.kts index 1109ad7..e0d8ac2 100644 --- a/test/build.gradle.kts +++ b/test/build.gradle.kts @@ -25,8 +25,9 @@ dependencies { implementation(kotlin("stdlib")) implementation(kotlin("stdlib-jdk8")) implementation("org.slf4j:slf4j-api:1.7.36") + implementation("io.minio:minio:8.3.9") - implementation("org.veupathdb.lib.s3:s34k-minio:0.1.0") + implementation("org.veupathdb.lib.s3:s34k-minio:0.2.0+s34k-0.4.0") implementation("org.apache.logging.log4j:log4j-core:2.17.2") implementation("org.apache.logging.log4j:log4j-slf4j-impl:2.17.2") diff --git a/test/src/main/resources/log4j2.xml b/test/src/main/resources/log4j2.xml index 0441ce4..d0bc191 100644 --- a/test/src/main/resources/log4j2.xml +++ b/test/src/main/resources/log4j2.xml @@ -12,7 +12,7 @@ - +