Add option to exclude instance from resume/suspend control. Fix initi…

…alization issues.
defold · Aug 14, 2024 · 3ba048e · 3ba048e
1 parent 312fac1
commit 3ba048e
Show file tree

Hide file tree

Showing 6 changed files with 40 additions and 32 deletions.
diff --git a/server/configs/application-local-dev-app.yml b/server/configs/application-local-dev-app.yml
@@ -4,34 +4,35 @@ extender:
         platforms:
             android-ndk25:
                 url: http://android-ndk25:9000
-                instance: android-ndk25
+                instanceId: android-ndk25
+                alwaysOn: true
             emsdk-2011:
                 url: http://emsdk-2011:9000
-                instance: emsdk-2011
+                instanceId: emsdk-2011
             emsdk-3155:
                 url: http://emsdk-3155:9000
-                instance: emsdk-3155
+                instanceId: emsdk-3155
             linux-latest:
                 url: http://linux:9000
-                instance: linux
+                instanceId: linux
             nssdk-1532:
                 url: http://10.0.0.3:9000
-                instance: nssdk-1532
+                instanceId: nssdk-1532
             nssdk-1753:
                 url: http://10.0.0.4:9000
-                instance: nssdk-1753
+                instanceId: nssdk-1753
             ps4-10500:
                 url: http://ps4-10500:9000
-                instance: ps4-10500
+                instanceId: ps4-10500
             ps4-11000:
                 url: http://ps4-11000:9000
-                instance: ps4-11000
+                instanceId: ps4-11000
             ps5-8000:
                 url: http://ps5-8000:9000
-                instance: ps5-8000
+                instanceId: ps5-8000
             winsdk-2019:
                 url: http://winsdk-2019:9000
-                instance: winsdk-2019
+                instanceId: winsdk-2019
             winsdk-2022:
                 url: http://winsdk-2022:9000
-                instance: winsdk-2022
+                instanceId: winsdk-2022
diff --git a/server/src/main/java/com/defold/extender/remote/RemoteEngineBuilder.java b/server/src/main/java/com/defold/extender/remote/RemoteEngineBuilder.java
@@ -85,10 +85,10 @@ public void build(final RemoteInstanceConfig remoteInstanceConfig,
 
 
         try {
-            touchInstance(remoteInstanceConfig.getInstance());
+            touchInstance(remoteInstanceConfig.getInstanceId());
             final HttpResponse response = sendRequest(remoteInstanceConfig.getUrl(), platform, sdkVersion, httpEntity);
 
-            touchInstance(remoteInstanceConfig.getInstance());
+            touchInstance(remoteInstanceConfig.getInstanceId());
             LOGGER.info("Remote builder response status: {}", response.getStatusLine());
 
             if (isClientError(response)) {
@@ -137,7 +137,7 @@ public void buildAsync(final RemoteInstanceConfig remoteInstanceConfig,
 
             final HttpClient client  = HttpClientBuilder.create().build();
 
-            touchInstance(remoteInstanceConfig.getInstance());
+            touchInstance(remoteInstanceConfig.getInstanceId());
             HttpResponse response = client.execute(request);
             // copied from ExtenderClient. Think about code deduplication.
             if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
@@ -147,7 +147,7 @@ public void buildAsync(final RemoteInstanceConfig remoteInstanceConfig,
                 Integer jobStatus = 0;
                 Thread.sleep(buildSleepTimeout);
                 while (System.currentTimeMillis() - currentTime < buildResultWaitTimeout) {
-                    touchInstance(remoteInstanceConfig.getInstance());
+                    touchInstance(remoteInstanceConfig.getInstanceId());
                     HttpGet statusRequest = new HttpGet(String.format("%s/job_status?jobId=%s", remoteInstanceConfig.getUrl(), jobId));
                     response = client.execute(statusRequest);
                     jobStatus = Integer.valueOf(EntityUtils.toString(response.getEntity()));
@@ -163,7 +163,7 @@ public void buildAsync(final RemoteInstanceConfig remoteInstanceConfig,
                     writer.write(String.format("Job %s result cannot be defined during %d", jobId, buildResultWaitTimeout));
                     writer.close();
                 }
-                touchInstance(remoteInstanceConfig.getInstance());
+                touchInstance(remoteInstanceConfig.getInstanceId());
                 HttpGet resultRequest = new HttpGet(String.format("%s/job_result?jobId=%s", remoteInstanceConfig.getUrl(), jobId));
                 response = client.execute(resultRequest);
                 LOGGER.info(String.format("Job %s result got.", jobId));
@@ -256,7 +256,7 @@ private void touchInstance(String instanceId) {
             try {
                 instanceService.touchInstance(instanceId);
             } catch(Exception exc) {
-
+                LOGGER.error(String.format("Exception during touch instance '%s'", instanceId), exc);
             }
         }
     }

diff --git a/server/src/main/java/com/defold/extender/remote/RemoteInstanceConfig.java b/server/src/main/java/com/defold/extender/remote/RemoteInstanceConfig.java
@@ -2,26 +2,24 @@
 
 public class RemoteInstanceConfig {
     private String url;
-    private String instance;
+    private String instanceId;
+    private boolean alwaysOn;
 
-    public RemoteInstanceConfig(String id, String url) {
-        this.instance = id;
+    public RemoteInstanceConfig(String url, String instanceId, boolean alwaysOn) {
         this.url = url;
+        this.instanceId = instanceId;
+        this.alwaysOn = alwaysOn;
     }
 
     public String getUrl() {
         return url;
     }
 
-    public String getInstance() {
-        return instance;
+    public String getInstanceId() {
+        return instanceId;
     }
 
-    public void setInstance(String instance) {
-        this.instance = instance;
-    }
-
-    public void setUrl(String url) {
-        this.url = url;
+    public boolean getAlwaysOn() {
+        return alwaysOn;
     }
 }
diff --git a/server/src/main/java/com/defold/extender/services/GCPInstanceService.java b/server/src/main/java/com/defold/extender/services/GCPInstanceService.java
@@ -17,6 +17,7 @@
 import com.google.cloud.compute.v1.InstancesClient;
 import com.google.cloud.compute.v1.Operation;
 import com.defold.extender.remote.RemoteHostConfiguration;
+import com.defold.extender.remote.RemoteInstanceConfig;
 import com.defold.extender.services.data.GCPInstanceState;
 import com.google.cloud.compute.v1.Instance.Status;
 
@@ -29,7 +30,7 @@ public class GCPInstanceService {
     private InstancesClient instancesClient = null;
     @Value("${extender.gcp.controller.project_id}") private String projectId;
     @Value("${extender.gcp.controller.zone}") private String computeZone;
-    @Value("${extender.gcp.controller.wait-timeout:30000}") private long operationWaitTimeout;
+    @Value("${extender.gcp.controller.wait-timeout:45000}") private long operationWaitTimeout;
     @Value("${extender.gcp.controller.retry-attempts:3}") private int retryAttempts;
     @Value("${extender.gcp.controller.retry-timeout:10000}") private long retryTimeout;
     @Value("${extender.gcp.controller.time-before-suspend:1800000}") private long timeBeforeSuspend;
@@ -39,6 +40,11 @@ public class GCPInstanceService {
 
     public GCPInstanceService(RemoteHostConfiguration remoteHostConfiguration) throws IOException{
         instancesClient = InstancesClient.create();
+        for (Map.Entry<String, RemoteInstanceConfig> entry : remoteHostConfiguration.getPlatforms().entrySet()) {
+            if (!entry.getValue().getAlwaysOn()) {
+                instanceState.put(entry.getValue().getInstanceId(), new GCPInstanceState());
+            }
+        }
     }
 
     private void suspendInstance(final String instanceId) throws InterruptedException, ExecutionException, TimeoutException {
@@ -84,6 +90,10 @@ public String getInstanceStatus(final String instanceId) {
     }
 
     public void touchInstance(String instanceId) throws InterruptedException, ExecutionException, TimeoutException {
+        // if we instance was marked as alwaysOn we skip adding it during initialization
+        if (!instanceState.containsKey(instanceId)) {
+            return;
+        }
         // update last touched time to prevent suspending
         instanceState.get(instanceId).lastTimeTouched = System.currentTimeMillis();
         String instanceStatus = getInstanceStatus(instanceId);

diff --git a/server/src/main/java/com/defold/extender/services/data/GCPInstanceState.java b/server/src/main/java/com/defold/extender/services/data/GCPInstanceState.java
@@ -1,5 +1,5 @@
 package com.defold.extender.services.data;
 
 public class GCPInstanceState {
-    public volatile long lastTimeTouched = 0;
+    public volatile long lastTimeTouched = System.currentTimeMillis();
 }
diff --git a/server/src/test/java/com/defold/extender/remote/RemoteEngineBuilderTest.java b/server/src/test/java/com/defold/extender/remote/RemoteEngineBuilderTest.java
@@ -1,7 +1,6 @@
 package com.defold.extender.remote;
 
 import com.defold.extender.ExtenderException;
-import com.defold.extender.services.GCPInstanceService;
 
 import org.apache.http.HttpEntity;
 import org.apache.http.StatusLine;
@@ -26,7 +25,7 @@
 public class RemoteEngineBuilderTest {
 
     private RemoteEngineBuilder remoteEngineBuilder;
-    final RemoteInstanceConfig remoteBuilderConfig = new RemoteInstanceConfig("osx-latest", "https://test.darwin-build.defold.com");
+    final RemoteInstanceConfig remoteBuilderConfig = new RemoteInstanceConfig("https://test.darwin-build.defold.com", "osx-latest", true);
 
     @Before
     public void setUp() throws IOException {