Support for multipart uploads

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>DVUploader</groupId>
     <artifactId>DVUploader</artifactId>
-    <version>1.0.9</version>
+    <version>1.1.0</version>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     </properties>

src/main/java/org/sead/uploader/AbstractUploader.java

@@ -22,6 +22,7 @@
 import java.io.PrintWriter;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.text.DecimalFormat;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -104,6 +105,13 @@ public static void println(String s) {
         }
         return;
     }
+    static DecimalFormat decimalFormat = new DecimalFormat("#.00");
+
+    public static void printStatus(float s) {
+        System.out.print("\rProgress: " + decimalFormat.format(s*100) + "%");
+        System.out.flush();
+        return;
+    }
 
     public void parseArgs(String[] args) {
 

src/main/java/org/sead/uploader/dataverse/HttpPartUploadJob.java

@@ -0,0 +1,107 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.sead.uploader.dataverse;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.entity.InputStreamEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
+import static org.sead.uploader.AbstractUploader.println;
+import org.sead.uploader.util.Resource;
+
+/**
+ *
+ * @author Jim
+ */
+public class HttpPartUploadJob implements Runnable {
+
+    int partNo;
+    long size;
+    String signedUrl;
+    Resource file;
+    Map eTags;
+
+    static private long partSize = -1;
+    static private CloseableHttpClient httpClient = null;
+    static private HttpClientContext localContext = null;
+
+    public static void setHttpClient(CloseableHttpClient client) {
+        httpClient = client;
+    }
+
+    public static void setHttpClientContext(HttpClientContext context) {
+        localContext = context;
+    }
+
+    public static void setPartSize(long ps) {
+        partSize=ps;
+    }
+
+    public HttpPartUploadJob(int partNo, String url, Resource file, long size, Map eTags) throws IllegalStateException {
+        if ((size == -1) || (httpClient == null) || (localContext == null)) {
+            throw new IllegalStateException("partSize not set");
+        }
+        this.partNo = partNo;
+        this.signedUrl = url;
+        this.file = file;
+        this.size=size;
+        this.eTags = eTags;
+    }
+
+    /*
+	 * (non-Javadoc)
+	 * 
+	 * @see java.lang.Runnable#run()
+     */
+    public void run() {
+        int retries = 3;
+        //println("Starting upload of part: " + partNo);
+        while (retries > 0) {
+            try (InputStream is = file.getInputStream((partNo - 1) * partSize, size)) {
+
+                HttpPut httpput = new HttpPut(signedUrl);
+                httpput.setEntity(new InputStreamEntity(is, size));
+                CloseableHttpResponse putResponse = httpClient.execute(httpput);
+                int putStatus = putResponse.getStatusLine().getStatusCode();
+                String putRes = null;
+                HttpEntity putEntity = putResponse.getEntity();
+                if (putEntity != null) {
+                    putRes = EntityUtils.toString(putEntity);
+                }
+                if (putStatus == 200) {
+                    //Part successfully stored - parse the eTag from the response and it it to the Map
+                    String eTag = putResponse.getFirstHeader("ETag").getValue();
+                    eTag= eTag.replace("\"","");
+                    eTags.put(Integer.toString(partNo), eTag);
+                    retries = 0;
+                    //println("Completed upload of part: " + partNo);
+                } else {
+                    if (putStatus >= 500) {
+                        println("Upload of part: " + partNo + " failed with status: " + putStatus + " (skipping)");
+                        println("Error response: " + putResponse.getStatusLine() + " : " + putRes);
+                        retries--;
+                    } else {
+                        println("Upload of part: " + partNo + " failed with status: " + putStatus + " (retrying)");
+                        println("Error response: " + putResponse.getStatusLine() + " : " + putRes);
+
+                        retries--;
+                    }
+                }
+
+            } catch (IOException e) {
+                e.printStackTrace(System.out);
+                println("Error uploading part: " + partNo + " : " + e.getMessage());
+                retries = 0;
+            }
+        }
+    }
+}

src/main/java/org/sead/uploader/dataverse/MD5Job.java

@@ -0,0 +1,62 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.sead.uploader.dataverse;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Map;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.http.HttpEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.entity.InputStreamEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
+import static org.sead.uploader.AbstractUploader.println;
+import org.sead.uploader.util.Resource;
+
+/**
+ *
+ * @author Jim
+ */
+public class MD5Job implements Runnable {
+
+    Resource file;
+    Map infoMap;
+
+    public MD5Job(Resource file, Map infoMap) throws IllegalStateException {
+        this.file = file;
+        this.infoMap = infoMap;
+    }
+
+    /*
+	 * (non-Javadoc)
+	 * 
+	 * @see java.lang.Runnable#run()
+     */
+    public void run() {
+        try {
+            MessageDigest messageDigest = MessageDigest.getInstance("MD5");
+
+            try (InputStream inStream = file.getInputStream(); DigestInputStream digestInputStream = new DigestInputStream(inStream, messageDigest)) {
+                byte[] bytes = new byte[64*1024];
+                while(digestInputStream.read(bytes) >= 0) {
+                }
+                String checksum = Hex.encodeHexString(digestInputStream.getMessageDigest().digest());
+                infoMap.put("md5", checksum);
+            } catch (IOException e) {
+                e.printStackTrace(System.out);
+                println("Error calculating digest for: " + file.getAbsolutePath() + " : " + e.getMessage());
+            }
+        } catch (NoSuchAlgorithmException nsae) {
+            println("MD5 algorithm not found: " + nsae.getMessage());
+        }
+    }
+}

src/main/java/org/sead/uploader/util/FileResource.java

@@ -31,12 +31,13 @@
 
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.input.BoundedInputStream;
 import org.apache.http.entity.ContentType;
 import org.apache.http.entity.mime.content.ContentBody;
 import org.apache.http.entity.mime.content.FileBody;
 import org.json.JSONObject;
 
-public class FileResource implements Resource {
+public class FileResource extends Resource {
 
     private File f;
 
@@ -163,5 +164,4 @@ public JSONObject getMetadata() {
         // No extra metadata by default for files
         return new JSONObject();
     }
-
 }

src/main/java/org/sead/uploader/util/PublishedFolderProxyResource.java

@@ -40,7 +40,7 @@
  * @author Jim
  *
  */
-public class PublishedFolderProxyResource extends PublishedResource implements Resource {
+public class PublishedFolderProxyResource extends PublishedResource {
 
     private PublishedResource resource;
     private String message;

src/main/java/org/sead/uploader/util/PublishedResource.java

@@ -33,7 +33,7 @@
 import org.json.JSONException;
 import org.json.JSONObject;
 
-public class PublishedResource implements Resource {
+public class PublishedResource extends Resource {
 
     protected JSONObject resource;
     private String path;

src/main/java/org/sead/uploader/util/Resource.java

@@ -15,33 +15,55 @@
  ***************************************************************************** */
 package org.sead.uploader.util;
 
+import java.io.FileNotFoundException;
+import java.io.IOException;
 import java.io.InputStream;
+import org.apache.commons.io.input.BoundedInputStream;
 
 import org.apache.http.entity.mime.content.ContentBody;
 import org.json.JSONObject;
 
-public interface Resource extends Iterable<Resource> {
+public abstract class Resource implements Iterable<Resource> {
 
-    String getName();
+    public abstract String getName();
 
-    boolean isDirectory();
+    public abstract boolean isDirectory();
 
-    String getPath();
+    public abstract String getPath();
 
-    long length();
+    public abstract long length();
 
-    String getAbsolutePath();
+    public abstract String getAbsolutePath();
 
-    ContentBody getContentBody();
+    public abstract ContentBody getContentBody();
 
-    InputStream getInputStream();
+    public abstract InputStream getInputStream();
 
-    Iterable<Resource> listResources();
+    public abstract Iterable<Resource> listResources();
 
-    String getHash(String algorithm);
+    public abstract String getHash(String algorithm);
 
-    JSONObject getMetadata();
+    public abstract JSONObject getMetadata();
 
-    String getMimeType();
+    public abstract String getMimeType();
+
+    /**
+     *
+     * @param l
+     * @param partSize
+     * @return
+     */
+    public InputStream getInputStream(long l, long partSize) {
+        try {
+            InputStream is = getInputStream();
+            is.skip(l);
+            return new BoundedInputStream(is, partSize);
+        } catch (FileNotFoundException e) {
+            e.printStackTrace();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
 
 }