diff --git a/build.gradle b/build.gradle index be325f22..a5fae36e 100644 --- a/build.gradle +++ b/build.gradle @@ -58,16 +58,17 @@ configurations { } dependencies { - api 'gyro:gyro-core:0.99.1-SNAPSHOT' - implementation enforcedPlatform('com.google.cloud:libraries-bom:5.3.0') - implementation 'com.google.auth:google-auth-library-oauth2-http:0.18.0' - implementation 'com.google.apis:google-api-services-iam:v1-rev309-1.25.0' - implementation 'com.google.apis:google-api-services-cloudresourcemanager:v1-rev20200311-1.30.9' + api 'gyro:gyro-core:0.99.5' + (releaseBuild ? '' : '-SNAPSHOT') + + implementation enforcedPlatform('com.google.cloud:libraries-bom:8.1.0') + // XXX: com.google.apis:google-api-services-iam:v1-rev316-1.25.0 uses older version of com.google.api-client:google-api-client + implementation 'com.google.api-client:google-api-client:1.30.9' + implementation 'com.google.apis:google-api-services-iam:v1-rev316-1.25.0' implementation 'com.google.cloud:google-cloud-compute' implementation 'com.google.cloud:google-cloud-dns' - implementation 'com.google.cloud:google-cloud-storage' implementation 'com.google.cloud:google-cloud-kms' - implementation 'com.google.guava:guava:23.0' + implementation 'com.google.cloud:google-cloud-resourcemanager' + implementation 'com.google.cloud:google-cloud-storage' implementation 'com.psddev:dari-util:3.3.607-xe0f27a' gyroDoclet "gyro:gyro-doclet:0.99.0-SNAPSHOT" diff --git a/src/main/java/gyro/google/GoogleStorageFileBackend.java b/src/main/java/gyro/google/GoogleStorageFileBackend.java index d222c8fa..771b0301 100644 --- a/src/main/java/gyro/google/GoogleStorageFileBackend.java +++ b/src/main/java/gyro/google/GoogleStorageFileBackend.java @@ -16,17 +16,17 @@ package gyro.google; -import java.io.ByteArrayOutputStream; import java.io.InputStream; import java.io.OutputStream; -import java.util.Spliterators; +import java.nio.channels.Channels; +import java.util.Optional; import java.util.stream.Stream; import java.util.stream.StreamSupport; -import com.google.api.client.googleapis.json.GoogleJsonResponseException; -import com.google.api.client.http.ByteArrayContent; -import com.google.api.services.storage.Storage; -import com.google.api.services.storage.model.StorageObject; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; import com.psddev.dari.util.ObjectUtils; import gyro.core.FileBackend; import gyro.core.GyroCore; @@ -72,10 +72,8 @@ public String getCredentials() { @Override public Stream list() throws Exception { if (this.equals(GyroCore.getStateBackend(getName()))) { - return StreamSupport.stream( - Spliterators.spliteratorUnknownSize(new StorageObjectIterator(getBucket(), prefixed(""), client()), 0), - false) - .map(StorageObject::getName) + return StreamSupport.stream(service().list(getBucket()).iterateAll().spliterator(), false) + .map(Blob::getName) .filter(f -> f.endsWith(".gyro")) .map(this::removePrefix); } @@ -85,43 +83,50 @@ public Stream list() throws Exception { @Override public InputStream openInput(String file) throws Exception { - return client().objects().get(getBucket(), prefixed(file)).executeMediaAsInputStream(); + return Channels.newInputStream(service().reader(getBucket(), prefixed(file))); } @Override public OutputStream openOutput(String file) throws Exception { - return new ByteArrayOutputStream() { - - public void close() { - try { - StorageObject upload = new StorageObject(); - upload.setName(prefixed(file)); - client().objects().insert(getBucket(), upload, new ByteArrayContent(null, toByteArray())).execute(); - } catch (Exception e) { - throw new GyroException(String.format("Could not upload file %s.", prefixed(file))); - } - } - }; + return Channels.newOutputStream(service().writer( + BlobInfo.newBuilder(getBucket(), prefixed(file)).build(), + Storage.BlobWriteOption.predefinedAcl(Storage.PredefinedAcl.PRIVATE))); } @Override public void delete(String file) throws Exception { - // Delete if exists. - try { - client().objects().delete(getBucket(), prefixed(file)).execute(); - } catch (GoogleJsonResponseException e) { - if (e.getStatusCode() != 404) { - throw e; - } - } + service().delete(getBucket(), prefixed(file)); + } + + @Override + public boolean exists(String file) throws Exception { + return service().get(getBucket(), prefixed(file)) != null; } - private Storage client() { - GoogleCredentials credentials = (GoogleCredentials) getRootScope().getSettings(CredentialsSettings.class) - .getCredentialsByName() - .get("google::" + getCredentials()); + @Override + public void copy(String source, String destination) throws Exception { + String bucket = getBucket(); + service().copy(Storage.CopyRequest.newBuilder() + .setSource(bucket, prefixed(source)) + .setTarget( + BlobInfo.newBuilder(bucket, prefixed(destination)).build(), + + Storage.BlobTargetOption.predefinedAcl(Storage.PredefinedAcl.PRIVATE)) + .build()).getResult(); + } - return credentials.createClient(Storage.class); + private Storage service() { + return Optional.ofNullable(getRootScope()) + .map(e -> e.getSettings(CredentialsSettings.class)) + .map(CredentialsSettings::getCredentialsByName) + .map(e -> e.get("google::" + getCredentials())) + .filter(GoogleCredentials.class::isInstance) + .map(GoogleCredentials.class::cast) + .map(e -> StorageOptions.newBuilder() + .setCredentials(e.getGoogleCredentials()) + .build() + .getService()) + .orElseThrow(() -> new GyroException("No storage service available!")); } private String prefixed(String file) { diff --git a/src/main/java/gyro/google/StorageObjectIterator.java b/src/main/java/gyro/google/StorageObjectIterator.java deleted file mode 100644 index adf6ecd7..00000000 --- a/src/main/java/gyro/google/StorageObjectIterator.java +++ /dev/null @@ -1,74 +0,0 @@ -package gyro.google; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; - -import com.google.api.services.storage.Storage; -import com.google.api.services.storage.model.Objects; -import com.google.api.services.storage.model.StorageObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -class StorageObjectIterator implements Iterator { - - private static final Logger LOGGER = LoggerFactory.getLogger(StorageObjectIterator.class); - - private String bucket; - private String prefix; - private Storage client; - - private List storageObjects = new ArrayList<>(); - private int index; - private String nextPageToken; - - public StorageObjectIterator(String bucket, String prefix, Storage client) { - this.bucket = bucket; - this.prefix = prefix; - this.client = client; - } - - @Override - public boolean hasNext() { - if (index < storageObjects.size()) { - return true; - } - - if (index > 0 && nextPageToken == null) { - return false; - } - - try { - Objects objects = client.objects() - .list(bucket) - .setPrefix(prefix) - .setPageToken(nextPageToken) - .setMaxResults(100L) - .execute(); - - if (objects.getItems() != null) { - index = 0; - storageObjects.clear(); - storageObjects.addAll(objects.getItems()); - } - nextPageToken = objects.getNextPageToken(); - - } catch (IOException e) { - LOGGER.error("Failed to retrieve storage objects: {} {} {}", bucket, prefix, nextPageToken, e); - return false; - } - - return !storageObjects.isEmpty(); - - } - - @Override - public StorageObject next() { - if (hasNext()) { - return storageObjects.get(index++); - } - throw new NoSuchElementException(); - } -}