Skip to content

Commit

Permalink
Add GCPInstanceService which used to monitor and suspend gcp instance…
Browse files Browse the repository at this point in the history 
…s if instance not used for a while.
  • Loading branch information
@ekharkunov
ekharkunov committed Aug 13, 2024
1 parent 4b4b730 commit 0b2ca03
Show file tree
Hide file tree
Showing 10 changed files with 229 additions and 42 deletions.
1 change: 1 addition & 0 deletions server/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ dependencies {
}

implementation('com.google.cloud:spring-cloud-gcp-starter-storage')
implementation('com.google.cloud:google-cloud-compute:1.57.0')
implementation('com.google.guava:guava:33.2.1-jre')
implementation('com.samskivert:jmustache:1.15')
implementation('commons-chain:commons-chain:1.2')
Expand Down
50 changes: 33 additions & 17 deletions server/configs/application-local-dev-app.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,36 @@ extender:
remote-builder:
enabled: true
platforms:
android-ndk25: http://android-ndk25:9000
android-latest: http://android-ndk25:9000
emsdk-2011: http://emsdk-2011:9000
emsdk-3155: http://emsdk-3155:9000
emsdk-latest: http://emsdk-3155:9000
linux-latest: http://linux:9000
nssdk-1532: http://nssdk-1532:9000
nssdk-1753: http://nssdk-1753:9000
nssdk-latest: http://nssdk-1753:9000
ps4-10500: http://ps4-10500:9000
ps4-11000: http://ps4-11000:9000
ps4-latest: http://ps4-11000:9000
ps5-8000: http://ps5-8000:9000
ps5-latest: http://ps5-8000:9000
winsdk-2019: http://winsdk-2019:9000
winsdk-2022: http://winsdk-2022:9000
winsdk-latest: http://winsdk-2022:9000
android-ndk25:
url: http://android-ndk25:9000
instance: android-ndk25
emsdk-2011:
url: http://emsdk-2011:9000
instance: emsdk-2011
emsdk-3155:
url: http://emsdk-3155:9000
instance: emsdk-3155
linux-latest:
url: http://linux:9000
instance: linux
nssdk-1532:
url: http://10.0.0.3:9000
instance: nssdk-1532
nssdk-1753:
url: http://10.0.0.4:9000
instance: nssdk-1753
ps4-10500:
url: http://ps4-10500:9000
instance: ps4-10500
ps4-11000:
url: http://ps4-11000:9000
instance: ps4-11000
ps5-8000:
url: http://ps5-8000:9000
instance: ps5-8000
winsdk-2019:
url: http://winsdk-2019:9000
instance: winsdk-2019
winsdk-2022:
url: http://winsdk-2022:9000
instance: winsdk-2022
13 changes: 7 additions & 6 deletions server/src/main/java/com/defold/extender/ExtenderController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.defold.extender.remote.RemoteEngineBuilder;
import com.defold.extender.remote.RemoteHostConfiguration;
import com.defold.extender.remote.RemoteInstanceConfig;
import com.defold.extender.metrics.MetricsWriter;
import com.defold.extender.services.DefoldSdkService;
import com.defold.extender.services.DataCacheService;
Expand Down Expand Up @@ -65,7 +66,7 @@ public class ExtenderController {
private final HealthReporterService healthReporter;

private final RemoteEngineBuilder remoteEngineBuilder;
private Map<String, String> remoteBuilderPlatformMappings;
private Map<String, RemoteInstanceConfig> remoteBuilderPlatformMappings;
private final boolean remoteBuilderEnabled;

private static long maxPackageSize = 1024*1024*1024;
Expand Down Expand Up @@ -212,8 +213,8 @@ public void buildEngine(HttpServletRequest _request,
// Build engine locally or on remote builder
if (remoteBuilderEnabled && isRemotePlatform(buildEnvDescription[0], buildEnvDescription[1])) {
LOGGER.info("Building engine on remote builder");
String remoteUrl = getRemoteBuilderUrl(buildEnvDescription[0], buildEnvDescription[1]);
this.remoteEngineBuilder.build(remoteUrl, uploadDirectory, platform, sdkVersion, response.getOutputStream());
RemoteInstanceConfig remoteInstanceConfig = getRemoteBuilderConfig(buildEnvDescription[0], buildEnvDescription[1]);
this.remoteEngineBuilder.build(remoteInstanceConfig, uploadDirectory, platform, sdkVersion, response.getOutputStream());
metricsWriter.measureRemoteEngineBuild(platform);
} else {
LOGGER.info("Building engine locally");
Expand Down Expand Up @@ -344,8 +345,8 @@ public void buildEngineAsync(HttpServletRequest _request,
// Build engine locally or on remote builder
if (remoteBuilderEnabled && isRemotePlatform(buildEnvDescription[0], buildEnvDescription[1])) {
LOGGER.info("Building engine on remote builder");
String remoteUrl = getRemoteBuilderUrl(buildEnvDescription[0], buildEnvDescription[1]);
this.remoteEngineBuilder.buildAsync(remoteUrl, uploadDirectory, platform, sdkVersion, jobDirectory, buildDirectory, metricsWriter);
RemoteInstanceConfig remoteInstanceConfig = getRemoteBuilderConfig(buildEnvDescription[0], buildEnvDescription[1]);
this.remoteEngineBuilder.buildAsync(remoteInstanceConfig, uploadDirectory, platform, sdkVersion, jobDirectory, buildDirectory, metricsWriter);
} else {
asyncBuilder.asyncBuildEngine(metricsWriter, platform, sdkVersion, jobDirectory, uploadDirectory, buildDirectory);
}
Expand Down Expand Up @@ -536,7 +537,7 @@ private boolean isRemotePlatform(final String platform, String platformVersion)
|| this.remoteBuilderPlatformMappings.containsKey(String.format("%s-%s", platform, ExtenderController.LATEST));
}

private String getRemoteBuilderUrl(String platform, String platformVersion) throws ExtenderException{
private RemoteInstanceConfig getRemoteBuilderConfig(String platform, String platformVersion) throws ExtenderException{
String fullKey = String.format("%s-%s", platform, platformVersion);
String fallbackKey = String.format("%s-%s", platform, ExtenderController.LATEST);
if (this.remoteBuilderPlatformMappings.containsKey(fullKey)) {
Expand Down
39 changes: 29 additions & 10 deletions server/src/main/java/com/defold/extender/remote/RemoteEngineBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.defold.extender.BuilderConstants;
import com.defold.extender.ExtenderException;
import com.defold.extender.metrics.MetricsWriter;
import com.defold.extender.services.GCPInstanceService;
import com.defold.extender.Timer;

import org.apache.commons.io.FileUtils;
Expand Down Expand Up @@ -34,42 +35,45 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.util.Optional;
import java.nio.charset.StandardCharsets;

@Service
public class RemoteEngineBuilder {

private static final Logger LOGGER = LoggerFactory.getLogger(RemoteEngineBuilder.class);

private GCPInstanceService instanceService;
private File jobResultLocation;
private long buildSleepTimeout;
private long buildResultWaitTimeout;
private boolean keepJobDirectory = false;

public RemoteEngineBuilder(@Value("${extender.job-result.location}") String jobResultLocation,
public RemoteEngineBuilder(Optional<GCPInstanceService> instanceService,
@Value("${extender.job-result.location}") String jobResultLocation,
@Value("${extender.remote-builder.build-sleep-timeout:5000}") long buildSleepTimeout,
@Value("${extender.remote-builder.build-result-wait-timeout:1200000}") long buildResultWaitTimeout) {
instanceService.ifPresent(val -> { LOGGER.info("Instance client is initialized"); this.instanceService = val; });
this.buildSleepTimeout = buildSleepTimeout;
this.buildResultWaitTimeout = buildResultWaitTimeout;
this.jobResultLocation = new File(jobResultLocation);
this.keepJobDirectory = System.getenv("DM_DEBUG_KEEP_JOB_FOLDER") != null || System.getenv("DM_DEBUG_JOB_FOLDER") != null;
}

private String getErrorString(HttpResponse response) throws IOException {

ByteArrayOutputStream bos = new ByteArrayOutputStream();
response.getEntity().writeTo(bos);
final byte[] bytes = bos.toByteArray();
return new String(bytes, StandardCharsets.UTF_8);
}

public void build(final String remoteBuilderUrl,
public void build(final RemoteInstanceConfig remoteInstanceConfig,
final File projectDirectory,
final String platform,
final String sdkVersion,
final OutputStream out) throws ExtenderException {

LOGGER.info("Building engine remotely at {}", remoteBuilderUrl);
LOGGER.info("Building engine remotely at {}", remoteInstanceConfig.getUrl());

final HttpEntity httpEntity;

Expand All @@ -81,8 +85,10 @@ public void build(final String remoteBuilderUrl,


try {
final HttpResponse response = sendRequest(remoteBuilderUrl, platform, sdkVersion, httpEntity);
touchInstance(remoteInstanceConfig.getInstance());
final HttpResponse response = sendRequest(remoteInstanceConfig.getUrl(), platform, sdkVersion, httpEntity);

touchInstance(remoteInstanceConfig.getInstance());
LOGGER.info("Remote builder response status: {}", response.getStatusLine());

if (isClientError(response)) {
Expand All @@ -102,13 +108,13 @@ public void build(final String remoteBuilderUrl,
}

@Async
public void buildAsync(final String remoteBuilderUrl,
public void buildAsync(final RemoteInstanceConfig remoteInstanceConfig,
final File projectDirectory,
final String platform,
final String sdkVersion,
File jobDirectory, File buildDirectory, MetricsWriter metricsWriter) throws FileNotFoundException, IOException {

LOGGER.info("Building engine remotely at {}", remoteBuilderUrl);
LOGGER.info("Building engine remotely at {}", remoteInstanceConfig.getUrl());
String jobName = jobDirectory.getName();
Thread.currentThread().setName(String.format("async-build-%s", jobName));
File resultDir = new File(jobResultLocation.getAbsolutePath(), jobName);
Expand All @@ -125,12 +131,13 @@ public void buildAsync(final String remoteBuilderUrl,
}

try {
final String serverUrl = String.format("%s/build_async/%s/%s", remoteBuilderUrl, platform, sdkVersion);
final String serverUrl = String.format("%s/build_async/%s/%s", remoteInstanceConfig.getUrl(), platform, sdkVersion);
final HttpPost request = new HttpPost(serverUrl);
request.setEntity(httpEntity);

final HttpClient client = HttpClientBuilder.create().build();

touchInstance(remoteInstanceConfig.getInstance());
HttpResponse response = client.execute(request);
// copied from ExtenderClient. Think about code deduplication.
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
Expand All @@ -140,7 +147,8 @@ public void buildAsync(final String remoteBuilderUrl,
Integer jobStatus = 0;
Thread.sleep(buildSleepTimeout);
while (System.currentTimeMillis() - currentTime < buildResultWaitTimeout) {
HttpGet statusRequest = new HttpGet(String.format("%s/job_status?jobId=%s", remoteBuilderUrl, jobId));
touchInstance(remoteInstanceConfig.getInstance());
HttpGet statusRequest = new HttpGet(String.format("%s/job_status?jobId=%s", remoteInstanceConfig.getUrl(), jobId));
response = client.execute(statusRequest);
jobStatus = Integer.valueOf(EntityUtils.toString(response.getEntity()));
if (jobStatus != 0) {
Expand All @@ -155,7 +163,8 @@ public void buildAsync(final String remoteBuilderUrl,
writer.write(String.format("Job %s result cannot be defined during %d", jobId, buildResultWaitTimeout));
writer.close();
}
HttpGet resultRequest = new HttpGet(String.format("%s/job_result?jobId=%s", remoteBuilderUrl, jobId));
touchInstance(remoteInstanceConfig.getInstance());
HttpGet resultRequest = new HttpGet(String.format("%s/job_result?jobId=%s", remoteInstanceConfig.getUrl(), jobId));
response = client.execute(resultRequest);
LOGGER.info(String.format("Job %s result got.", jobId));
if (jobStatus == BuilderConstants.JobStatus.SUCCESS.ordinal()) {
Expand Down Expand Up @@ -241,4 +250,14 @@ private boolean isServerError(final HttpResponse response) {
private String getStatusReason(final HttpResponse response) {
return response.getStatusLine().getReasonPhrase();
}

private void touchInstance(String instanceId) {
if (instanceService != null) {
try {
instanceService.touchInstance(instanceId);
} catch(Exception exc) {

}
}
}
}
4 changes: 2 additions & 2 deletions server/src/main/java/com/defold/extender/remote/RemoteHostConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@
@Component
@ConfigurationProperties(prefix = "extender.remote-builder")
public class RemoteHostConfiguration {
private Map<String, String> platforms = new HashMap<>();
private Map<String, RemoteInstanceConfig> platforms = new HashMap<>();

public Map<String, String> getPlatforms() {
public Map<String, RemoteInstanceConfig> getPlatforms() {
return platforms;
}
}
27 changes: 27 additions & 0 deletions server/src/main/java/com/defold/extender/remote/RemoteInstanceConfig.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package com.defold.extender.remote;

public class RemoteInstanceConfig {
private String url;
private String instance;

public RemoteInstanceConfig(String id, String url) {
this.instance = id;
this.url = url;
}

public String getUrl() {
return url;
}

public String getInstance() {
return instance;
}

public void setInstance(String instance) {
this.instance = instance;
}

public void setUrl(String url) {
this.url = url;
}
}
Loading

0 comments on commit 0b2ca03

Please sign in to comment.