Skip to content

Commit

Permalink
remove unused params, send identifiers and paths, try scheduler, etc.
Browse files Browse the repository at this point in the history
  • Loading branch information
@qqmyers
qqmyers committed Oct 20, 2023
1 parent 8f53aff commit 56192f8
Show file tree
Hide file tree
Showing 5 changed files with 75 additions and 22 deletions.
28 changes: 18 additions & 10 deletions src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3451,15 +3451,25 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
if(!DataAccess.getDriverType(storeId).equals(DataAccess.GLOBUS)) {
return badRequest(BundleUtil.getStringFromBundle("datasets.api.globusuploaddisabled"));
}

URLTokenUtil tokenUtil = new URLTokenUtil(dataset, authSvc.findApiTokenByUser(authUser), locale);

boolean managed = GlobusOverlayAccessIO.isDataverseManaged(storeId);
String endpoint = GlobusOverlayAccessIO.getEndpointId(storeId);

JsonObjectBuilder queryParams = Json.createObjectBuilder();
queryParams.add("queryParameters",
Json.createArrayBuilder().add(Json.createObjectBuilder().add("datasetId", "{datasetId}"))
.add(Json.createObjectBuilder().add("siteUrl", "{siteUrl}"))
.add(Json.createObjectBuilder().add("datasetVersion", "{datasetVersion}"))
.add(Json.createObjectBuilder().add("dvLocale", "{localeCode}")).add(Json.createObjectBuilder()
.add("datasetPid", "{datasetPid}").add("managed", managed).add("endpoint", endpoint)));
.add(Json.createObjectBuilder().add("dvLocale", "{localeCode}"))
.add(Json.createObjectBuilder().add("datasetPid", "{datasetPid}")));
JsonObject substitutedParams = tokenUtil.getParams(queryParams.build());
JsonObjectBuilder params = Json.createObjectBuilder();
substitutedParams.keySet().forEach((key) -> {
params.add(key, substitutedParams.get(key));
});
params.add("managed", Boolean.toString(managed)).add("endpoint", endpoint);

JsonArrayBuilder allowedApiCalls = Json.createArrayBuilder();
allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "requestGlobusTransferPaths")
Expand All @@ -3470,14 +3480,12 @@ public Response getGlobusUploadParams(@Context ContainerRequestContext crc, @Pat
.add(URLTokenUtil.HTTP_METHOD, "POST")
.add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/addGlobusFiles")
.add(URLTokenUtil.TIMEOUT, 300));
allowedApiCalls.add(Json.createObjectBuilder().add(URLTokenUtil.NAME, "getFileListing")
.add(URLTokenUtil.HTTP_METHOD, "GET")
.add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}/files")
.add(URLTokenUtil.TIMEOUT, 300));

allowedApiCalls.add(
Json.createObjectBuilder().add(URLTokenUtil.NAME, "getFileListing").add(URLTokenUtil.HTTP_METHOD, "GET")
.add(URLTokenUtil.URL_TEMPLATE, "/api/v1/datasets/{datasetId}/versions/{datasetVersion}/files")
.add(URLTokenUtil.TIMEOUT, 300));

URLTokenUtil tokenUtil = new URLTokenUtil(dataset, authSvc.findApiTokenByUser(authUser), locale);
return ok(tokenUtil.createPostBody(tokenUtil.getParams(queryParams.build()), allowedApiCalls.build()));
return ok(tokenUtil.createPostBody(params.build(), allowedApiCalls.build()));
}

/** Requests permissions for a given globus user to upload to the dataset
Expand Down Expand Up @@ -3544,7 +3552,7 @@ public Response requestGlobusUpload(@Context ContainerRequestContext crc, @PathP
JsonObject response = globusService.requestAccessiblePaths(principal, dataset, numberOfPaths);
switch (response.getInt("status")) {
case 201:
return ok(response.getJsonArray("paths"));
return ok(response.getJsonObject("paths"));
case 400:
return badRequest("Unable to grant permission");
case 409:
Expand Down
24 changes: 24 additions & 0 deletions src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -403,4 +403,28 @@ public static boolean isValidDirectStorageIdentifier(String storageId) {
}
return false;
}



public static String getNewStorageIdentifier(String driverId) {
String storageType = DataAccess.getDriverType(driverId);
if (storageType.equals("tmp") || storageType.equals("Undefined")) {
return null;
}
switch (storageType) {
case FILE:
return FileAccessIO.getNewIdentifier(driverId);
case SWIFT:
return SwiftAccessIO.getNewIdentifier(driverId);
case S3:
return S3AccessIO.getNewIdentifier(driverId);
case REMOTE:
return RemoteOverlayAccessIO.getNewIdentifier(driverId);
case GLOBUS:
return GlobusOverlayAccessIO.getNewIdentifier(driverId);
default:
logger.warning("Request to validate for storage driver: " + driverId);
}
return null;
}
}
4 changes: 4 additions & 0 deletions src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1400,4 +1400,8 @@ public long retrieveSizeFromMedia() throws IOException {
}
return objectMetadata.getContentLength();
}

public static String getNewIdentifier(String driverId) {
return driverId + DataAccess.SEPARATOR + getConfigParamForDriver(driverId, BUCKET_NAME) + ":" + FileUtil.generateStorageIdentifier();
}
}
5 changes: 5 additions & 0 deletions src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import edu.harvard.iq.dataverse.Dataverse;
import edu.harvard.iq.dataverse.DvObject;
import edu.harvard.iq.dataverse.datavariable.DataVariable;
import edu.harvard.iq.dataverse.util.FileUtil;

import java.io.IOException;
import java.io.InputStream;
Expand Down Expand Up @@ -670,5 +671,9 @@ protected static String getConfigParamForDriver(String driverId, String paramete
protected static String getConfigParamForDriver(String driverId, String parameterName, String defaultValue) {
return System.getProperty("dataverse.files." + driverId + "." + parameterName, defaultValue);
}

public static String getNewIdentifier(String driverId) {
return driverId + DataAccess.SEPARATOR + FileUtil.generateStorageIdentifier();
}

}
36 changes: 24 additions & 12 deletions src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.Scheduler;
import com.google.gson.FieldNamingPolicy;
import com.google.gson.GsonBuilder;
import edu.harvard.iq.dataverse.*;
Expand Down Expand Up @@ -36,6 +37,7 @@
import java.time.Duration;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
Expand Down Expand Up @@ -257,12 +259,16 @@ public JsonObject requestAccessiblePaths(String principal, Dataset dataset, int
permissions.setId(globusResponse.getString("access_id"));
monitorTemporaryPermissions(permissions.getId(), dataset.getId());
logger.info("Access rule " + permissions.getId() + " was created successfully");
JsonArrayBuilder pathArray = Json.createArrayBuilder();

String driverId = dataset.getEffectiveStorageDriverId();
JsonObjectBuilder paths = Json.createObjectBuilder();
for(int i=0;i<numberOfPaths;i++) {
pathArray.add(getUniqueFilePath(endpoint));
String storageIdentifier = DataAccess.getNewStorageIdentifier(driverId);
int lastIndex = Math.max(storageIdentifier.lastIndexOf("/"), storageIdentifier.lastIndexOf(":"));
paths.add(storageIdentifier, endpoint.getBasePath() + "/" + storageIdentifier.substring(lastIndex + 1));

}
response.add("paths", pathArray.build());
response.add("paths", paths.build());

} else {
//Shouldn't happen!
Expand All @@ -277,18 +283,23 @@ public JsonObject requestAccessiblePaths(String principal, Dataset dataset, int
return response.build();
}

private String getUniqueFilePath(GlobusEndpoint endpoint) {
private Entry<String,String> getUniqueFilePath(GlobusEndpoint endpoint) {
// TODO See if generated identifier exists at globus endpoint
return endpoint.getBasePath() + "/" + FileUtil.generateStorageIdentifier();
String sid=FileUtil.generateStorageIdentifier();
String path = endpoint.getBasePath() + "/" + FileUtil.generateStorageIdentifier();
return null;
}

//Single cache of open rules/permission requests
private final Cache<String, Long> rulesCache = Caffeine.newBuilder()
.expireAfterWrite(Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))
// .expireAfterWrite(Duration.of(JvmSettings.GLOBUS_RULES_CACHE_MAXAGE.lookup(Integer.class), ChronoUnit.MINUTES))
.expireAfterWrite(Duration.of(1, ChronoUnit.MINUTES))
.scheduler(Scheduler.systemScheduler())
.evictionListener((ruleId, datasetId, cause) -> {
//Delete rules that expire
logger.info("Rule " + ruleId + " expired");
Dataset dataset = datasetSvc.find(datasetId);
deletePermission((String) ruleId, dataset, null);
deletePermission((String) ruleId, dataset, logger);
})

.build();
Expand Down Expand Up @@ -583,7 +594,7 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, DataFile df)
String appUrl;
if (upload) {
appUrl = settingsSvc.getValueForKey(SettingsServiceBean.Key.GlobusAppUrl, "http://localhost")
+ "/upload?datasetPid={datasetPid}&siteUrl={siteUrl}&datasetId={datasetId}&datasetVersion={datasetVersion}&dvLocale={localeCode}";
+ "/upload?dvLocale={localeCode}";
String callback = SystemConfig.getDataverseSiteUrlStatic() + "/api/v1/datasets/" + d.getId()
+ "/globusUploadParameters?locale=" + localeCode;
if (apiToken != null) {
Expand Down Expand Up @@ -611,7 +622,7 @@ public String getGlobusAppUrlForDataset(Dataset d, boolean upload, DataFile df)
+ rawStorageId + "&fileName=" + df.getCurrentName();
}
}
String finalUrl = tokenUtil.replaceTokensWithValues(appUrl) + "&storePrefix=" + storePrefix;
String finalUrl = tokenUtil.replaceTokensWithValues(appUrl);
logger.info("Calling app: " + finalUrl);
return finalUrl;
}
Expand Down Expand Up @@ -827,9 +838,7 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
globusLogger.info("Files failures: " + countError.toString());
globusLogger.info("Finished upload via Globus job.");

if (fileHandlerSuceeded) {
fileHandler.close();
}


} catch (Exception e) {
logger.info("Exception from globusUpload call ");
Expand All @@ -842,6 +851,9 @@ public void globusUpload(JsonObject jsonData, ApiToken token, Dataset dataset, S
deletePermission(ruleId, dataset, globusLogger);
globusLogger.info("Removed upload permission: " + ruleId);
}
if (fileHandlerSuceeded) {
fileHandler.close();
}
}

public String addFilesAsync(String curlCommand, Logger globusLogger)
Expand Down

0 comments on commit 56192f8

Please sign in to comment.