From 3d8c5cbf7ca260b2f1cf985c71f9c6b879f43629 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 18 Apr 2023 18:18:52 -0400 Subject: [PATCH] avoid repeat failures when server is overloaded --- .../sead/uploader/dataverse/DVUploader.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/sead/uploader/dataverse/DVUploader.java b/src/main/java/org/sead/uploader/dataverse/DVUploader.java index 6a88c5d..7049cc8 100644 --- a/src/main/java/org/sead/uploader/dataverse/DVUploader.java +++ b/src/main/java/org/sead/uploader/dataverse/DVUploader.java @@ -493,10 +493,23 @@ public void addDatasetMetadata(String newSubject, String type, JSONObject relati protected void postProcessChildren(Resource dir) { if (!singleFile && directUpload) { //Have to register all the files in this dir with Dataverse - + int retries = 5; + //In case of prior 504 (or other) errors, make sure the dataset is OK before adding files + int total = 0; + // For new servers, wait up to maxWaitTime for a dataset lock to expire. + try { + while (isLocked() && (total < maxWaitTime)) { + TimeUnit.SECONDS.sleep(1); + total = total + 1; + } + } catch(InterruptedException ie) { + println("Error waiting for Dataverse dataset lock - skipping: " + dir.getAbsolutePath()); + retries=0; + } + String urlString = server + "/api/datasets/:persistentId/addFiles"; urlString = urlString + "?persistentId=" + datasetPID + "&key=" + apiKey; - int retries = 5; + while (retries > 0) { HttpPost httppost = new HttpPost(urlString); JSONArray jsonData = new JSONArray(); @@ -551,7 +564,7 @@ protected void postProcessChildren(Resource dir) { } } retries = 0; - int total = 0; + total = 0; // For new servers, wait up to maxWaitTime for a dataset lock to expire. while (isLocked() && (total < maxWaitTime)) { TimeUnit.SECONDS.sleep(1);