From 0aac2c83b0ffc99e77e5ca7fa04b374a26ca9678 Mon Sep 17 00:00:00 2001 From: Andrew Sung Date: Fri, 31 Jul 2020 14:20:09 -0400 Subject: [PATCH 1/6] Update and clean up dependencies. --- build.gradle | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index be325f22..827adf52 100644 --- a/build.gradle +++ b/build.gradle @@ -59,15 +59,15 @@ configurations { dependencies { api 'gyro:gyro-core:0.99.1-SNAPSHOT' - implementation enforcedPlatform('com.google.cloud:libraries-bom:5.3.0') - implementation 'com.google.auth:google-auth-library-oauth2-http:0.18.0' - implementation 'com.google.apis:google-api-services-iam:v1-rev309-1.25.0' - implementation 'com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200311-1.30.9' + implementation enforcedPlatform('com.google.cloud:libraries-bom:8.1.0') + // XXX: com.google.apis:google-api-services-iam:v1-rev316-1.25.0 uses older version of com.google.api-client:google-api-client + implementation 'com.google.api-client:google-api-client:1.30.9' + implementation 'com.google.apis:google-api-services-iam:v1-rev316-1.25.0' implementation 'com.google.cloud:google-cloud-compute' implementation 'com.google.cloud:google-cloud-dns' - implementation 'com.google.cloud:google-cloud-storage' implementation 'com.google.cloud:google-cloud-kms' - implementation 'com.google.guava:guava:23.0' + implementation 'com.google.cloud:google-cloud-resourcemanager' + implementation 'com.google.cloud:google-cloud-storage' implementation 'com.psddev:dari-util:3.3.607-xe0f27a' gyroDoclet "gyro:gyro-doclet:0.99.0-SNAPSHOT" From 5ca53fa70cd59ce6b2b9330cbd5bac6657326103 Mon Sep 17 00:00:00 2001 From: Andrew Sung Date: Fri, 31 Jul 2020 14:23:56 -0400 Subject: [PATCH 2/6] Refactors to use storage SDK. --- .../gyro/google/GoogleStorageFileBackend.java | 60 ++++++--------- .../gyro/google/StorageObjectIterator.java | 74 ------------------- 2 files changed, 23 insertions(+), 111 deletions(-) delete mode 100644 src/main/java/gyro/google/StorageObjectIterator.java diff --git a/src/main/java/gyro/google/GoogleStorageFileBackend.java b/src/main/java/gyro/google/GoogleStorageFileBackend.java index d222c8fa..0d2b1b3e 100644 --- a/src/main/java/gyro/google/GoogleStorageFileBackend.java +++ b/src/main/java/gyro/google/GoogleStorageFileBackend.java @@ -16,17 +16,17 @@ package gyro.google; -import java.io.ByteArrayOutputStream; import java.io.InputStream; import java.io.OutputStream; -import java.util.Spliterators; +import java.nio.channels.Channels; +import java.util.Optional; import java.util.stream.Stream; import java.util.stream.StreamSupport; -import com.google.api.client.googleapis.json.GoogleJsonResponseException; -import com.google.api.client.http.ByteArrayContent; -import com.google.api.services.storage.Storage; -import com.google.api.services.storage.model.StorageObject; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; import com.psddev.dari.util.ObjectUtils; import gyro.core.FileBackend; import gyro.core.GyroCore; @@ -72,10 +72,8 @@ public String getCredentials() { @Override public Stream list() throws Exception { if (this.equals(GyroCore.getStateBackend(getName()))) { - return StreamSupport.stream( - Spliterators.spliteratorUnknownSize(new StorageObjectIterator(getBucket(), prefixed(""), client()), 0), - false) - .map(StorageObject::getName) + return StreamSupport.stream(service().list(getBucket()).iterateAll().spliterator(), false) + .map(Blob::getName) .filter(f -> f.endsWith(".gyro")) .map(this::removePrefix); } @@ -85,43 +83,31 @@ public Stream list() throws Exception { @Override public InputStream openInput(String file) throws Exception { - return client().objects().get(getBucket(), prefixed(file)).executeMediaAsInputStream(); + return Channels.newInputStream(service().reader(getBucket(), prefixed(file))); } @Override public OutputStream openOutput(String file) throws Exception { - return new ByteArrayOutputStream() { - - public void close() { - try { - StorageObject upload = new StorageObject(); - upload.setName(prefixed(file)); - client().objects().insert(getBucket(), upload, new ByteArrayContent(null, toByteArray())).execute(); - } catch (Exception e) { - throw new GyroException(String.format("Could not upload file %s.", prefixed(file))); - } - } - }; + return Channels.newOutputStream(service().writer(BlobInfo.newBuilder(getBucket(), prefixed(file)).build())); } @Override public void delete(String file) throws Exception { - // Delete if exists. - try { - client().objects().delete(getBucket(), prefixed(file)).execute(); - } catch (GoogleJsonResponseException e) { - if (e.getStatusCode() != 404) { - throw e; - } - } + service().delete(getBucket(), prefixed(file)); } - private Storage client() { - GoogleCredentials credentials = (GoogleCredentials) getRootScope().getSettings(CredentialsSettings.class) - .getCredentialsByName() - .get("google::" + getCredentials()); - - return credentials.createClient(Storage.class); + private Storage service() { + return Optional.ofNullable(getRootScope()) + .map(e -> e.getSettings(CredentialsSettings.class)) + .map(CredentialsSettings::getCredentialsByName) + .map(e -> e.get("google::" + getCredentials())) + .filter(GoogleCredentials.class::isInstance) + .map(GoogleCredentials.class::cast) + .map(e -> StorageOptions.newBuilder() + .setCredentials(e.getGoogleCredentials()) + .build() + .getService()) + .orElseThrow(() -> new GyroException("No storage service available!")); } private String prefixed(String file) { diff --git a/src/main/java/gyro/google/StorageObjectIterator.java b/src/main/java/gyro/google/StorageObjectIterator.java deleted file mode 100644 index adf6ecd7..00000000 --- a/src/main/java/gyro/google/StorageObjectIterator.java +++ /dev/null @@ -1,74 +0,0 @@ -package gyro.google; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; - -import com.google.api.services.storage.Storage; -import com.google.api.services.storage.model.Objects; -import com.google.api.services.storage.model.StorageObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -class StorageObjectIterator implements Iterator { - - private static final Logger LOGGER = LoggerFactory.getLogger(StorageObjectIterator.class); - - private String bucket; - private String prefix; - private Storage client; - - private List storageObjects = new ArrayList<>(); - private int index; - private String nextPageToken; - - public StorageObjectIterator(String bucket, String prefix, Storage client) { - this.bucket = bucket; - this.prefix = prefix; - this.client = client; - } - - @Override - public boolean hasNext() { - if (index < storageObjects.size()) { - return true; - } - - if (index > 0 && nextPageToken == null) { - return false; - } - - try { - Objects objects = client.objects() - .list(bucket) - .setPrefix(prefix) - .setPageToken(nextPageToken) - .setMaxResults(100L) - .execute(); - - if (objects.getItems() != null) { - index = 0; - storageObjects.clear(); - storageObjects.addAll(objects.getItems()); - } - nextPageToken = objects.getNextPageToken(); - - } catch (IOException e) { - LOGGER.error("Failed to retrieve storage objects: {} {} {}", bucket, prefix, nextPageToken, e); - return false; - } - - return !storageObjects.isEmpty(); - - } - - @Override - public StorageObject next() { - if (hasNext()) { - return storageObjects.get(index++); - } - throw new NoSuchElementException(); - } -} From 8f5e2c126537b9a38c85d4c37147e222735f8603 Mon Sep 17 00:00:00 2001 From: Andrew Sung Date: Mon, 3 Aug 2020 10:49:00 -0400 Subject: [PATCH 3/6] Implements exists and copy methods in file backend. --- .../java/gyro/google/GoogleStorageFileBackend.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/gyro/google/GoogleStorageFileBackend.java b/src/main/java/gyro/google/GoogleStorageFileBackend.java index 0d2b1b3e..e7c9df34 100644 --- a/src/main/java/gyro/google/GoogleStorageFileBackend.java +++ b/src/main/java/gyro/google/GoogleStorageFileBackend.java @@ -96,6 +96,16 @@ public void delete(String file) throws Exception { service().delete(getBucket(), prefixed(file)); } + @Override + public boolean exists(String file) throws Exception { + return service().get(getBucket(), prefixed(file)) != null; + } + + @Override + public void copy(String source, String destination) throws Exception { + service().copy(Storage.CopyRequest.of(getBucket(), prefixed(source), prefixed(destination))).getResult(); + } + private Storage service() { return Optional.ofNullable(getRootScope()) .map(e -> e.getSettings(CredentialsSettings.class)) From d5961cbe02bc5ae902453eff1e2b1c8ca3ae53ba Mon Sep 17 00:00:00 2001 From: Andrew Sung Date: Mon, 3 Aug 2020 10:52:30 -0400 Subject: [PATCH 4/6] Fixes to use a snapshot version of gyro in development. --- build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 827adf52..a5fae36e 100644 --- a/build.gradle +++ b/build.gradle @@ -58,7 +58,8 @@ configurations { } dependencies { - api 'gyro:gyro-core:0.99.1-SNAPSHOT' + api 'gyro:gyro-core:0.99.5' + (releaseBuild ? '' : '-SNAPSHOT') + implementation enforcedPlatform('com.google.cloud:libraries-bom:8.1.0') // XXX: com.google.apis:google-api-services-iam:v1-rev316-1.25.0 uses older version of com.google.api-client:google-api-client implementation 'com.google.api-client:google-api-client:1.30.9' From 8e88cdc9f5ab199ed260442079eda1998557876e Mon Sep 17 00:00:00 2001 From: Andrew Sung Date: Fri, 7 Aug 2020 17:21:59 -0400 Subject: [PATCH 5/6] Fixes to support ACL. --- .../gyro/google/GoogleStorageFileBackend.java | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/main/java/gyro/google/GoogleStorageFileBackend.java b/src/main/java/gyro/google/GoogleStorageFileBackend.java index e7c9df34..130b001c 100644 --- a/src/main/java/gyro/google/GoogleStorageFileBackend.java +++ b/src/main/java/gyro/google/GoogleStorageFileBackend.java @@ -29,6 +29,7 @@ import com.google.cloud.storage.StorageOptions; import com.psddev.dari.util.ObjectUtils; import gyro.core.FileBackend; +import gyro.core.FileBackendAccess; import gyro.core.GyroCore; import gyro.core.GyroException; import gyro.core.Type; @@ -87,8 +88,10 @@ public InputStream openInput(String file) throws Exception { } @Override - public OutputStream openOutput(String file) throws Exception { - return Channels.newOutputStream(service().writer(BlobInfo.newBuilder(getBucket(), prefixed(file)).build())); + public OutputStream openOutput(String file, FileBackendAccess acl) throws Exception { + return Channels.newOutputStream(service().writer( + BlobInfo.newBuilder(getBucket(), prefixed(file)).build(), + Storage.BlobWriteOption.predefinedAcl(predefinedAcl(acl)))); } @Override @@ -102,8 +105,14 @@ public boolean exists(String file) throws Exception { } @Override - public void copy(String source, String destination) throws Exception { - service().copy(Storage.CopyRequest.of(getBucket(), prefixed(source), prefixed(destination))).getResult(); + public void copy(String source, String destination, FileBackendAccess acl) throws Exception { + String bucket = getBucket(); + service().copy(Storage.CopyRequest.newBuilder() + .setSource(bucket, prefixed(source)) + .setTarget( + BlobInfo.newBuilder(bucket, prefixed(destination)).build(), + Storage.BlobTargetOption.predefinedAcl(predefinedAcl(acl))) + .build()).getResult(); } private Storage service() { @@ -131,4 +140,13 @@ private String removePrefix(String file) { return file; } + + private Storage.PredefinedAcl predefinedAcl(FileBackendAccess acl) { + Storage.PredefinedAcl predefinedAcl = Storage.PredefinedAcl.PRIVATE; + + if (acl == FileBackendAccess.PUBLIC) { + predefinedAcl = Storage.PredefinedAcl.PUBLIC_READ; + } + return predefinedAcl; + } } From c4fc14f88b3d6412aa951ae1ca8c59ecad0553b9 Mon Sep 17 00:00:00 2001 From: Andrew Sung Date: Mon, 10 Aug 2020 16:37:55 -0400 Subject: [PATCH 6/6] Fixes to make files always private. --- .../gyro/google/GoogleStorageFileBackend.java | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/main/java/gyro/google/GoogleStorageFileBackend.java b/src/main/java/gyro/google/GoogleStorageFileBackend.java index 130b001c..771b0301 100644 --- a/src/main/java/gyro/google/GoogleStorageFileBackend.java +++ b/src/main/java/gyro/google/GoogleStorageFileBackend.java @@ -29,7 +29,6 @@ import com.google.cloud.storage.StorageOptions; import com.psddev.dari.util.ObjectUtils; import gyro.core.FileBackend; -import gyro.core.FileBackendAccess; import gyro.core.GyroCore; import gyro.core.GyroException; import gyro.core.Type; @@ -88,10 +87,10 @@ public InputStream openInput(String file) throws Exception { } @Override - public OutputStream openOutput(String file, FileBackendAccess acl) throws Exception { + public OutputStream openOutput(String file) throws Exception { return Channels.newOutputStream(service().writer( BlobInfo.newBuilder(getBucket(), prefixed(file)).build(), - Storage.BlobWriteOption.predefinedAcl(predefinedAcl(acl)))); + Storage.BlobWriteOption.predefinedAcl(Storage.PredefinedAcl.PRIVATE))); } @Override @@ -105,13 +104,14 @@ public boolean exists(String file) throws Exception { } @Override - public void copy(String source, String destination, FileBackendAccess acl) throws Exception { + public void copy(String source, String destination) throws Exception { String bucket = getBucket(); service().copy(Storage.CopyRequest.newBuilder() .setSource(bucket, prefixed(source)) .setTarget( BlobInfo.newBuilder(bucket, prefixed(destination)).build(), - Storage.BlobTargetOption.predefinedAcl(predefinedAcl(acl))) + + Storage.BlobTargetOption.predefinedAcl(Storage.PredefinedAcl.PRIVATE)) .build()).getResult(); } @@ -140,13 +140,4 @@ private String removePrefix(String file) { return file; } - - private Storage.PredefinedAcl predefinedAcl(FileBackendAccess acl) { - Storage.PredefinedAcl predefinedAcl = Storage.PredefinedAcl.PRIVATE; - - if (acl == FileBackendAccess.PUBLIC) { - predefinedAcl = Storage.PredefinedAcl.PUBLIC_READ; - } - return predefinedAcl; - } }