From 544eec73f9186c111b4216cacbf464491157c477 Mon Sep 17 00:00:00 2001 From: Michael Herdt Date: Fri, 12 Mar 2021 13:08:14 +0100 Subject: [PATCH 1/8] Open S3Object InputStream when creating an instance of S3Artifact to make it reusable and prevent a wrong usage because of an unused open InputStream. Signed-off-by: Michael Herdt --- .../artifact/repository/S3Artifact.java | 112 +++++++++++++++++- .../S3ArtifactNotFoundException.java | 90 ++++++++++++++ .../artifact/repository/S3Repository.java | 42 ++----- 3 files changed, 211 insertions(+), 33 deletions(-) create mode 100644 hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3ArtifactNotFoundException.java diff --git a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java index 9fe130c..08556b2 100644 --- a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java +++ b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java @@ -10,8 +10,13 @@ import java.io.InputStream; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3Object; +import com.google.common.io.BaseEncoding; import org.eclipse.hawkbit.artifact.repository.model.AbstractDbArtifact; import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.util.Assert; import com.amazonaws.services.s3.AmazonS3; @@ -22,11 +27,22 @@ */ public class S3Artifact extends AbstractDbArtifact { + private static final Logger LOG = LoggerFactory.getLogger(S3Artifact.class); + private final AmazonS3 amazonS3; private final S3RepositoryProperties s3Properties; private final String key; + private S3Object s3Object; + + + private S3Artifact(final S3Object s3Object, final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, + final String key, final String artifactId, final DbArtifactHash hashes, final Long size, + final String contentType) { + this(amazonS3, s3Properties, key, artifactId, hashes, size, contentType); + this.s3Object = s3Object; + } - S3Artifact(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String key, + private S3Artifact(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String key, final String artifactId, final DbArtifactHash hashes, final Long size, final String contentType) { super(artifactId, hashes, size, contentType); Assert.notNull(amazonS3, "S3 cannot be null"); @@ -37,9 +53,62 @@ public class S3Artifact extends AbstractDbArtifact { this.key = key; } - @Override - public InputStream getFileInputStream() { - return amazonS3.getObject(s3Properties.getBucketName(), key).getObjectContent(); + /** + * Get an S3Artifact for an already existing binary in the repository based on + * the given key. + * + * @param amazonS3 + * connection to the AmazonS3 + * @param s3Properties + * used to retrieve the bucket name + * @param key + * of the artifact + * @param artifactId + * sha1Hash to create the {@link DbArtifactHash} + * @return an instance of {@link S3Artifact} + * @throws S3ArtifactNotFoundException + * in case that no artifact could be found for the given values + */ + public static S3Artifact get(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String key, + final String artifactId) throws S3ArtifactNotFoundException { + final S3Object s3Object = getS3ObjectOrThrowException(amazonS3, s3Properties.getBucketName(), key); + + final ObjectMetadata objectMetadata = s3Object.getObjectMetadata(); + final DbArtifactHash artifactHash = createArtifactHash(artifactId, objectMetadata); + return new S3Artifact(s3Object, amazonS3, s3Properties, key, artifactId, artifactHash, + objectMetadata.getContentLength(), objectMetadata.getContentType()); + } + + /** + * Create a new instance of {@link S3Artifact}. In this case it is not checked + * if an artifact with the given values exists. The S3 object is empty. + * + * @param amazonS3 + * connection to the AmazonS3 + * @param s3Properties + * used to retrieve the bucket name + * @param key + * of the artifact + * @param hashes + * instance of {@link DbArtifactHash} + * @param size + * of the artifact + * @param contentType + * of the artifact + * @return an instance of {@link S3Artifact} with an empty {@link S3Object} + */ + public static S3Artifact create(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, + final String key, final DbArtifactHash hashes, final Long size, final String contentType) { + return new S3Artifact(amazonS3, s3Properties, key, hashes.getSha1(), hashes, size, contentType); + } + + /** + * Verify if the {@link S3Object} exists + * + * @return result of {@link AmazonS3}#doesObjectExist + */ + public boolean exists(){ + return amazonS3.doesObjectExist(s3Properties.getBucketName(), key); } @Override @@ -47,4 +116,39 @@ public String toString() { return "S3Artifact [key=" + key + ", getArtifactId()=" + getArtifactId() + ", getHashes()=" + getHashes() + ", getSize()=" + getSize() + ", getContentType()=" + getContentType() + "]"; } + + @Override + public InputStream getFileInputStream() { + LOG.debug("Get file input stream for s3 object with key {}", key); + return getS3Object().getObjectContent(); + } + + private S3Object getS3Object() { + if (s3Object == null) { + LOG.debug("Initialize S3Object in bucket {} with key {}", s3Properties.getBucketName(), key); + s3Object = amazonS3.getObject(s3Properties.getBucketName(), key); + } + return s3Object; + } + + private static S3Object getS3ObjectOrThrowException(AmazonS3 amazonS3, String bucketName, String key) + throws S3ArtifactNotFoundException { + final S3Object s3Object = amazonS3.getObject(bucketName, key); + if (s3Object == null) { + throw new S3ArtifactNotFoundException("Cannot find s3 object by given arguments.", bucketName, key); + } + return s3Object; + } + + private static DbArtifactHash createArtifactHash(final String artifactId, ObjectMetadata metadata) { + return new DbArtifactHash(artifactId, BaseEncoding.base16().lowerCase() + .encode(BaseEncoding.base64().decode(sanitizeEtag(metadata.getETag()))), null); + } + + private static String sanitizeEtag(final String etag) { + // base64 alphabet consist of alphanumeric characters and + / = (see RFC + // 4648) + return etag.trim().replaceAll("[^A-Za-z0-9+/=]", ""); + } + } diff --git a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3ArtifactNotFoundException.java b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3ArtifactNotFoundException.java new file mode 100644 index 0000000..f8cb73c --- /dev/null +++ b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3ArtifactNotFoundException.java @@ -0,0 +1,90 @@ +/** + * Copyright (c) 2021 Bosch.IO GmbH and others. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + */ +package org.eclipse.hawkbit.artifact.repository; + +/** + * An exception that is thrown as soon as an S3 object could not be found in a S3 bucket. + */ +public class S3ArtifactNotFoundException extends RuntimeException { + + private final String bucket; + private final String key; + + /** + * Constructor with individual error message and information about the searched + * artifact. + * + * @param message + * use an individual error message here. + * + * @param bucket + * the bucket of the searched artifact. + * @param key + * the key of the searched artifact (mostly kind of + * 'tenant/sha1hash'). + */ + public S3ArtifactNotFoundException(final String message, final String bucket, final String key) { + super(message); + this.bucket = bucket; + this.key = key; + } + + /** + * Constructor with individual error message with a cause and information about + * the searched artifact. + * + * @param message + * use an individual error message here. + * + * @param cause + * the cause of the exception. + * @param bucket + * the bucket of the searched artifact. + * @param key + * the key of the searched artifact (mostly kind of + * 'tenant/sha1hash'). + */ + public S3ArtifactNotFoundException(final String message, final Throwable cause, final String bucket, + final String key) { + super(message, cause); + this.bucket = bucket; + this.key = key; + } + + /** + * Constructor with a cause and information about the searched artifact. + * + * @param cause + * the cause of the exception. + * @param bucket + * the bucket of the searched artifact. + * @param key + * the key of the searched artifact (mostly kind of + * 'tenant/sha1hash'). + */ + public S3ArtifactNotFoundException(final Throwable cause, final String bucket, final String key) { + super(cause); + this.bucket = bucket; + this.key = key; + } + + /** + * @return key (mostly kind of 'tenant/sha1hash'). + */ + public String getKey() { + return key; + } + + /** + * @return the bucket name + */ + public String getBucket() { + return bucket; + } +} diff --git a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java index 34bc4d3..a40d823 100644 --- a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java +++ b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java @@ -28,7 +28,6 @@ import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectResult; -import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.common.io.BaseEncoding; @@ -77,7 +76,7 @@ protected AbstractDbArtifact store(final String tenant, final DbArtifactHash bas LOG.info("Storing file {} with length {} to AWS S3 bucket {} with key {}", file.getName(), file.length(), s3Properties.getBucketName(), key); - if (existsByTenantAndSha1(tenant, base16Hashes.getSha1())) { + if (s3Artifact.exists()) { LOG.debug("Artifact {} already exists on S3 bucket {}, don't need to upload twice", key, s3Properties.getBucketName()); return s3Artifact; @@ -100,8 +99,8 @@ protected AbstractDbArtifact store(final String tenant, final DbArtifactHash bas private S3Artifact createS3Artifact(final String tenant, final DbArtifactHash hashes, final String contentType, final File file) { - return new S3Artifact(amazonS3, s3Properties, objectKey(tenant, hashes.getSha1()), hashes.getSha1(), hashes, - file.length(), contentType); + return S3Artifact.create(amazonS3, s3Properties, objectKey(tenant, hashes.getSha1()), hashes, file.length(), + contentType); } private ObjectMetadata createObjectMetadata(final String mdMD5Hash16, final String contentType, final File file) { @@ -131,37 +130,22 @@ private static String objectKey(final String tenant, final String sha1Hash) { @Override public AbstractDbArtifact getArtifactBySha1(final String tenant, final String sha1Hash) { final String key = objectKey(tenant, sha1Hash); - - LOG.info("Retrieving S3 object from bucket {} and key {}", s3Properties.getBucketName(), key); - try (final S3Object s3Object = amazonS3.getObject(s3Properties.getBucketName(), key)) { - if (s3Object == null) { - return null; - } - - final ObjectMetadata s3ObjectMetadata = s3Object.getObjectMetadata(); - - // the MD5Content is stored in the ETag - return new S3Artifact(amazonS3, s3Properties, key, sha1Hash, - new DbArtifactHash(sha1Hash, - BaseEncoding.base16().lowerCase().encode( - BaseEncoding.base64().decode(sanitizeEtag(s3ObjectMetadata.getETag()))), - null), - s3ObjectMetadata.getContentLength(), s3ObjectMetadata.getContentType()); - } catch (final IOException e) { - LOG.error("Could not verify S3Object", e); + LOG.debug("Retrieving S3 object from bucket {} and key {}", s3Properties.getBucketName(), key); + try { + return S3Artifact.get(amazonS3, s3Properties, key, sha1Hash); + } catch (final S3ArtifactNotFoundException e) { + LOG.debug("Cannot find artifact for bucket {} with key {}", e.getBucket(), e.getKey()); return null; } } - private static String sanitizeEtag(final String etag) { - // base64 alphabet consist of alphanumeric characters and + / = (see RFC - // 4648) - return etag.trim().replaceAll("[^A-Za-z0-9+/=]", ""); - } - @Override public boolean existsByTenantAndSha1(final String tenant, final String sha1Hash) { - return amazonS3.doesObjectExist(s3Properties.getBucketName(), objectKey(tenant, sha1Hash)); + final boolean exists = amazonS3.doesObjectExist(s3Properties.getBucketName(), objectKey(tenant, sha1Hash)); + if (LOG.isDebugEnabled()) { + LOG.debug("Search for artifact with sha1Hash {} results in status: {}", sha1Hash, exists); + } + return exists; } @Override From 5e8cbc3254471b7cd6bfe23697fb9ef0a934888a Mon Sep 17 00:00:00 2001 From: Michael Herdt Date: Fri, 12 Mar 2021 13:20:19 +0100 Subject: [PATCH 2/8] Make S3Artifact a final class. Signed-off-by: Michael Herdt --- .../org/eclipse/hawkbit/artifact/repository/S3Artifact.java | 2 +- .../org/eclipse/hawkbit/artifact/repository/S3Repository.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java index 08556b2..24b1720 100644 --- a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java +++ b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java @@ -25,7 +25,7 @@ * An {@link AbstractDbArtifact} implementation which retrieves the * {@link InputStream} from the {@link AmazonS3} client. */ -public class S3Artifact extends AbstractDbArtifact { +public final class S3Artifact extends AbstractDbArtifact { private static final Logger LOG = LoggerFactory.getLogger(S3Artifact.class); diff --git a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java index a40d823..1c3549c 100644 --- a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java +++ b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Repository.java @@ -134,7 +134,7 @@ public AbstractDbArtifact getArtifactBySha1(final String tenant, final String sh try { return S3Artifact.get(amazonS3, s3Properties, key, sha1Hash); } catch (final S3ArtifactNotFoundException e) { - LOG.debug("Cannot find artifact for bucket {} with key {}", e.getBucket(), e.getKey()); + LOG.debug("Cannot find artifact for bucket {} with key {}", e.getBucket(), e.getKey(), e); return null; } } From 0544b342e75c69fe93f0f547611db94ac3ec469e Mon Sep 17 00:00:00 2001 From: Michael Herdt Date: Fri, 12 Mar 2021 15:32:36 +0100 Subject: [PATCH 3/8] Add new bosch license header for 2021. Signed-off-by: Michael Herdt --- licenses/LICENSE_HEADER_TEMPLATE_BOSCH_21.txt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 licenses/LICENSE_HEADER_TEMPLATE_BOSCH_21.txt diff --git a/licenses/LICENSE_HEADER_TEMPLATE_BOSCH_21.txt b/licenses/LICENSE_HEADER_TEMPLATE_BOSCH_21.txt new file mode 100644 index 0000000..a1239d1 --- /dev/null +++ b/licenses/LICENSE_HEADER_TEMPLATE_BOSCH_21.txt @@ -0,0 +1,6 @@ +Copyright (c) 2021 Bosch.IO GmbH and others. + +All rights reserved. This program and the accompanying materials +are made available under the terms of the Eclipse Public License v1.0 +which accompanies this distribution, and is available at +http://www.eclipse.org/legal/epl-v10.html From f1c319ced754dfbbf21fb5bdd9569b975816386b Mon Sep 17 00:00:00 2001 From: Michael Herdt Date: Fri, 12 Mar 2021 15:55:42 +0100 Subject: [PATCH 4/8] add new license header file to pom. Signed-off-by: Michael Herdt --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 73e460e..23cc6a2 100644 --- a/pom.xml +++ b/pom.xml @@ -90,12 +90,13 @@ com.mycila license-maven-plugin -
licenses/LICENSE_HEADER_TEMPLATE_BOSCH_20.txt
+
licenses/LICENSE_HEADER_TEMPLATE_BOSCH_21.txt
licenses/LICENSE_HEADER_TEMPLATE_SIEMENS.txt licenses/LICENSE_HEADER_TEMPLATE_SIEMENS_18.txt licenses/LICENSE_HEADER_TEMPLATE_BOSCH.txt licenses/LICENSE_HEADER_TEMPLATE_BOSCH_18.txt + licenses/LICENSE_HEADER_TEMPLATE_BOSCH_20.txt licenses/LICENSE_HEADER_TEMPLATE_MICROSOFT_20.txt licenses/LICENSE_HEADER_TEMPLATE_MICROSOFT_18.txt licenses/LICENSE_HEADER_TEMPLATE_RICO_PAHLISCH.txt From 584ea572b706c0cb6bf84ff6db62c2383cd34701 Mon Sep 17 00:00:00 2001 From: Michael Herdt Date: Tue, 16 Mar 2021 14:16:14 +0100 Subject: [PATCH 5/8] abort S3Object input stream on cancel. Signed-off-by: Michael Herdt --- .../org/eclipse/hawkbit/artifact/repository/S3Artifact.java | 6 ++++++ pom.xml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java index 24b1720..c48f3e7 100644 --- a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java +++ b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java @@ -123,6 +123,12 @@ public InputStream getFileInputStream() { return getS3Object().getObjectContent(); } + @Override + public void abortIfNeeded() { + LOG.info("Abort requested for artifact input stream with key {}", key); + getS3Object().getObjectContent().abort(); + } + private S3Object getS3Object() { if (s3Object == null) { LOG.debug("Initialize S3Object in bucket {} with key {}", s3Properties.getBucketName(), key); diff --git a/pom.xml b/pom.xml index 23cc6a2..d4699ee 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ - 0.3.0M7 + 0.3.0-SNAPSHOT 1.11.415 0.77.0-alpha From aeced15924424da6475175a25bd38439a7bc8158 Mon Sep 17 00:00:00 2001 From: Michael Herdt Date: Fri, 19 Mar 2021 13:23:31 +0100 Subject: [PATCH 6/8] Introduce a wrapper for the S3ObjectInputStream to abort the connection before closing it. Signed-off-by: Michael Herdt --- .../artifact/repository/S3Artifact.java | 57 ++++++++++++++----- pom.xml | 2 +- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java index c48f3e7..bc818cc 100644 --- a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java +++ b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java @@ -8,11 +8,14 @@ */ package org.eclipse.hawkbit.artifact.repository; +import java.io.IOException; import java.io.InputStream; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.google.common.io.BaseEncoding; +import org.apache.http.client.methods.HttpRequestBase; import org.eclipse.hawkbit.artifact.repository.model.AbstractDbArtifact; import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash; import org.slf4j.Logger; @@ -33,13 +36,14 @@ public final class S3Artifact extends AbstractDbArtifact { private final S3RepositoryProperties s3Properties; private final String key; private S3Object s3Object; - + private WrappedS3InputStream s3InputStream; private S3Artifact(final S3Object s3Object, final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String key, final String artifactId, final DbArtifactHash hashes, final Long size, final String contentType) { this(amazonS3, s3Properties, key, artifactId, hashes, size, contentType); this.s3Object = s3Object; + this.s3InputStream = WrappedS3InputStream.wrap(s3Object.getObjectContent()); } private S3Artifact(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String key, @@ -107,7 +111,7 @@ public static S3Artifact create(final AmazonS3 amazonS3, final S3RepositoryPrope * * @return result of {@link AmazonS3}#doesObjectExist */ - public boolean exists(){ + public boolean exists() { return amazonS3.doesObjectExist(s3Properties.getBucketName(), key); } @@ -120,21 +124,16 @@ public String toString() { @Override public InputStream getFileInputStream() { LOG.debug("Get file input stream for s3 object with key {}", key); - return getS3Object().getObjectContent(); - } - - @Override - public void abortIfNeeded() { - LOG.info("Abort requested for artifact input stream with key {}", key); - getS3Object().getObjectContent().abort(); + refreshS3ObjectIfNeeded(); + return s3InputStream; } - - private S3Object getS3Object() { - if (s3Object == null) { - LOG.debug("Initialize S3Object in bucket {} with key {}", s3Properties.getBucketName(), key); + + private void refreshS3ObjectIfNeeded() { + if (s3Object == null || s3InputStream == null) { + LOG.info("Initialize S3Object in bucket {} with key {}", s3Properties.getBucketName(), key); s3Object = amazonS3.getObject(s3Properties.getBucketName(), key); + s3InputStream = WrappedS3InputStream.wrap(s3Object.getObjectContent()); } - return s3Object; } private static S3Object getS3ObjectOrThrowException(AmazonS3 amazonS3, String bucketName, String key) @@ -157,4 +156,34 @@ private static String sanitizeEtag(final String etag) { return etag.trim().replaceAll("[^A-Za-z0-9+/=]", ""); } + /** + * Wrapper to abort the http request of the S3 input stream before closing it + */ + static class WrappedS3InputStream extends S3ObjectInputStream { + + /** + * Constructor + */ + private WrappedS3InputStream(InputStream in, HttpRequestBase httpRequest) { + super(in, httpRequest); + } + + /** + * Wrap an input stream of type {@link S3ObjectInputStream} to abort a + * connection before closing the stream + * + * @param inputStream + * the {@link S3ObjectInputStream} + * @return an instance of {@link WrappedS3InputStream} + */ + public static WrappedS3InputStream wrap(final S3ObjectInputStream inputStream) { + return new WrappedS3InputStream(inputStream, inputStream.getHttpRequest()); + } + + @Override + public void close() throws IOException { + super.abort(); + super.close(); + } + } } diff --git a/pom.xml b/pom.xml index d4699ee..23cc6a2 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ - 0.3.0-SNAPSHOT + 0.3.0M7 1.11.415 0.77.0-alpha From f6619af4d1ade77ba416ae20fee6cc8d0ed3fa70 Mon Sep 17 00:00:00 2001 From: Michael Herdt Date: Fri, 19 Mar 2021 13:50:50 +0100 Subject: [PATCH 7/8] Fix tests Signed-off-by: Michael Herdt --- .../eclipse/hawkbit/artifact/repository/S3Artifact.java | 2 +- .../hawkbit/artifact/repository/S3RepositoryTest.java | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java index bc818cc..4c12c49 100644 --- a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java +++ b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java @@ -159,7 +159,7 @@ private static String sanitizeEtag(final String etag) { /** * Wrapper to abort the http request of the S3 input stream before closing it */ - static class WrappedS3InputStream extends S3ObjectInputStream { + static final class WrappedS3InputStream extends S3ObjectInputStream { /** * Constructor diff --git a/hawkbit-extension-artifact-repository-s3/src/test/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryTest.java b/hawkbit-extension-artifact-repository-s3/src/test/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryTest.java index 87d4d00..7a1300c 100644 --- a/hawkbit-extension-artifact-repository-s3/src/test/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryTest.java +++ b/hawkbit-extension-artifact-repository-s3/src/test/java/org/eclipse/hawkbit/artifact/repository/S3RepositoryTest.java @@ -28,6 +28,8 @@ import java.security.NoSuchAlgorithmException; import java.util.Random; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import org.apache.http.client.methods.HttpRequestBase; import org.eclipse.hawkbit.artifact.repository.model.AbstractDbArtifact; import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash; import org.junit.jupiter.api.Test; @@ -70,6 +72,9 @@ public class S3RepositoryTest { @Mock private ObjectMetadata s3ObjectMetadataMock; + @Mock + private S3ObjectInputStream s3ObjectInputStream; + @Mock private PutObjectResult putObjectResultMock; @@ -121,6 +126,7 @@ public void getArtifactBySHA1Hash() { final String knownMdBase16 = BaseEncoding.base16().lowerCase().encode(knownMd5.getBytes()); final String knownMd5Base64 = BaseEncoding.base64().encode(knownMd5.getBytes()); + when(s3ObjectMock.getObjectContent()).thenReturn(s3ObjectInputStream); when(amazonS3Mock.getObject(anyString(), anyString())).thenReturn(s3ObjectMock); when(s3ObjectMock.getObjectMetadata()).thenReturn(s3ObjectMetadataMock); when(s3ObjectMetadataMock.getContentLength()).thenReturn(knownContentLength); @@ -146,6 +152,7 @@ public void getArtifactBySHA1SanitizeEtag() { final String knownMdBase16 = BaseEncoding.base16().lowerCase().encode(knownMd5.getBytes()); final String knownMd5Base64 = BaseEncoding.base64().encode(knownMd5.getBytes()); + when(s3ObjectMock.getObjectContent()).thenReturn(s3ObjectInputStream); when(amazonS3Mock.getObject(anyString(), anyString())).thenReturn(s3ObjectMock); when(s3ObjectMock.getObjectMetadata()).thenReturn(s3ObjectMetadataMock); // add special characters to etag From 9c0ce6ec9821dc28ec2a653643f973af8ea4bf76 Mon Sep 17 00:00:00 2001 From: Michael Herdt Date: Wed, 24 Mar 2021 08:58:58 +0100 Subject: [PATCH 8/8] Make use of java.util.Base64. Signed-off-by: Michael Herdt --- .../eclipse/hawkbit/artifact/repository/S3Artifact.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java index 4c12c49..1330f41 100644 --- a/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java +++ b/hawkbit-extension-artifact-repository-s3/src/main/java/org/eclipse/hawkbit/artifact/repository/S3Artifact.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Base64; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3Object; @@ -74,7 +75,7 @@ private S3Artifact(final AmazonS3 amazonS3, final S3RepositoryProperties s3Prope * in case that no artifact could be found for the given values */ public static S3Artifact get(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties, final String key, - final String artifactId) throws S3ArtifactNotFoundException { + final String artifactId) { final S3Object s3Object = getS3ObjectOrThrowException(amazonS3, s3Properties.getBucketName(), key); final ObjectMetadata objectMetadata = s3Object.getObjectMetadata(); @@ -136,8 +137,7 @@ private void refreshS3ObjectIfNeeded() { } } - private static S3Object getS3ObjectOrThrowException(AmazonS3 amazonS3, String bucketName, String key) - throws S3ArtifactNotFoundException { + private static S3Object getS3ObjectOrThrowException(AmazonS3 amazonS3, String bucketName, String key) { final S3Object s3Object = amazonS3.getObject(bucketName, key); if (s3Object == null) { throw new S3ArtifactNotFoundException("Cannot find s3 object by given arguments.", bucketName, key); @@ -147,7 +147,7 @@ private static S3Object getS3ObjectOrThrowException(AmazonS3 amazonS3, String bu private static DbArtifactHash createArtifactHash(final String artifactId, ObjectMetadata metadata) { return new DbArtifactHash(artifactId, BaseEncoding.base16().lowerCase() - .encode(BaseEncoding.base64().decode(sanitizeEtag(metadata.getETag()))), null); + .encode(Base64.getDecoder().decode(sanitizeEtag(metadata.getETag()))), null); } private static String sanitizeEtag(final String etag) {