Merge pull request #191 from perfectsense/feature/remote-backend-fix

Enables Remote State Storage

src/main/java/gyro/google/GoogleStorageFileBackend.java

@@ -19,14 +19,17 @@
 import java.io.ByteArrayOutputStream;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.util.Spliterators;
 import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
 
 import com.google.api.client.googleapis.json.GoogleJsonResponseException;
 import com.google.api.client.http.ByteArrayContent;
 import com.google.api.services.storage.Storage;
 import com.google.api.services.storage.model.StorageObject;
 import com.psddev.dari.util.ObjectUtils;
 import gyro.core.FileBackend;
+import gyro.core.GyroCore;
 import gyro.core.GyroException;
 import gyro.core.Type;
 import gyro.core.auth.CredentialsSettings;
@@ -68,7 +71,16 @@ public String getCredentials() {
 
     @Override
     public Stream<String> list() throws Exception {
-        return null;
+        if (this.equals(GyroCore.getStateBackend(getName()))) {
+            return StreamSupport.stream(
+                Spliterators.spliteratorUnknownSize(new StorageObjectIterator(getBucket(), prefixed(""), client()), 0),
+                false)
+                .map(StorageObject::getName)
+                .filter(f -> f.endsWith(".gyro"))
+                .map(this::removePrefix);
+        }
+
+        return Stream.empty();
     }
 
     @Override
@@ -79,6 +91,7 @@ public InputStream openInput(String file) throws Exception {
     @Override
     public OutputStream openOutput(String file) throws Exception {
         return new ByteArrayOutputStream() {
+
             public void close() {
                 try {
                     StorageObject upload = new StorageObject();
@@ -105,8 +118,8 @@ public void delete(String file) throws Exception {
 
     private Storage client() {
         GoogleCredentials credentials = (GoogleCredentials) getRootScope().getSettings(CredentialsSettings.class)
-                .getCredentialsByName()
-                .get("google::" + getCredentials());
+            .getCredentialsByName()
+            .get("google::" + getCredentials());
 
         return credentials.createClient(Storage.class);
     }
@@ -115,4 +128,11 @@ private String prefixed(String file) {
         return getPrefix() != null ? getPrefix() + '/' + file : file;
     }
 
+    private String removePrefix(String file) {
+        if (getPrefix() != null && file.startsWith(getPrefix() + "/")) {
+            return file.substring(getPrefix().length() + 1);
+        }
+
+        return file;
+    }
 }

src/main/java/gyro/google/StorageObjectIterator.java

@@ -0,0 +1,74 @@
+package gyro.google;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.model.Objects;
+import com.google.api.services.storage.model.StorageObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+class StorageObjectIterator implements Iterator<StorageObject> {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(StorageObjectIterator.class);
+
+    private String bucket;
+    private String prefix;
+    private Storage client;
+
+    private List<StorageObject> storageObjects = new ArrayList<>();
+    private int index;
+    private String nextPageToken;
+
+    public StorageObjectIterator(String bucket, String prefix, Storage client) {
+        this.bucket = bucket;
+        this.prefix = prefix;
+        this.client = client;
+    }
+
+    @Override
+    public boolean hasNext() {
+        if (index < storageObjects.size()) {
+            return true;
+        }
+
+        if (index > 0 && nextPageToken == null) {
+            return false;
+        }
+
+        try {
+            Objects objects = client.objects()
+                .list(bucket)
+                .setPrefix(prefix)
+                .setPageToken(nextPageToken)
+                .setMaxResults(100L)
+                .execute();
+
+            if (objects.getItems() != null) {
+                index = 0;
+                storageObjects.clear();
+                storageObjects.addAll(objects.getItems());
+            }
+            nextPageToken = objects.getNextPageToken();
+
+        } catch (IOException e) {
+            LOGGER.error("Failed to retrieve storage objects: {} {} {}", bucket, prefix, nextPageToken, e);
+            return false;
+        }
+
+        return !storageObjects.isEmpty();
+
+    }
+
+    @Override
+    public StorageObject next() {
+        if (hasNext()) {
+            return storageObjects.get(index++);
+        }
+        throw new NoSuchElementException();
+    }
+}