cadc-vos-server vault-related tweaking

cadc-test-vos/build.gradle

@@ -16,7 +16,7 @@ sourceCompatibility = 1.8
 
 group = 'org.opencadc'
 
-version = '2.1.2'
+version = '2.1.3'
 
 description = 'OpenCADC VOSpace test library'
 def git_url = 'https://github.com/opencadc/vos'

cadc-test-vos/src/main/java/org/opencadc/conformance/vos/TransferTest.java

@@ -73,6 +73,7 @@
 import ca.nrc.cadc.net.FileContent;
 import ca.nrc.cadc.net.HttpGet;
 import ca.nrc.cadc.net.HttpPost;
+import ca.nrc.cadc.net.HttpUpload;
 import ca.nrc.cadc.reg.Standards;
 import ca.nrc.cadc.uws.ExecutionPhase;
 import ca.nrc.cadc.uws.Job;
@@ -85,9 +86,12 @@
 import java.net.MalformedURLException;
 import java.net.URI;
 import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import javax.security.auth.Subject;
 import org.apache.log4j.Logger;
 import org.junit.Assert;
@@ -108,6 +112,8 @@
 public class TransferTest extends VOSTest {
     private static final Logger log = Logger.getLogger(TransferTest.class);
 
+    private static final List<Integer> PUT_OK = Arrays.asList(new Integer[] { 200, 201});
+
     protected TransferTest(URI resourceID, File testCert) {
         super(resourceID, testCert);
     }
@@ -127,49 +133,86 @@ public void syncPushPullTest() {
             // Create a push-to-vospace Transfer for the node
             Transfer pushTransfer = new Transfer(nodeURI.getURI(), Direction.pushToVoSpace);
             pushTransfer.version = VOS.VOSPACE_21;
-            Protocol protocol = new Protocol(VOS.PROTOCOL_HTTPS_PUT);
-            protocol.setSecurityMethod(Standards.SECURITY_METHOD_CERT);
-            pushTransfer.getProtocols().add(protocol);
+            pushTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_PUT)); // anon, preauth
+            Protocol putWithCert = new Protocol(VOS.PROTOCOL_HTTPS_PUT);
+            putWithCert.setSecurityMethod(Standards.SECURITY_METHOD_CERT);
+            pushTransfer.getProtocols().add(putWithCert);
 
-            // Do the transfer
+            // negotiate the transfer
             Transfer details = doTransfer(pushTransfer);
             Assert.assertEquals("expected transfer direction = " + Direction.pushToVoSpace,
                     Direction.pushToVoSpace, details.getDirection());
-            Assert.assertNotNull("expected > 0 protocols", details.getProtocols());
-            String endpoint = null;
+            Assert.assertNotNull(details.getProtocols());
+            log.info(pushTransfer.getDirection() + " results: " + details.getProtocols().size());
+            URL putURL = null;
             for (Protocol p : details.getProtocols()) {
+                String endpoint = p.getEndpoint();
+                log.info("PUT endpoint: " + endpoint);
                 try {
-                    endpoint = p.getEndpoint();
-                    log.debug("endpoint: " + endpoint);
-                    new URL(endpoint);
+
+                    URL u = new URL(endpoint);
+                    if (putURL == null) {
+                        putURL = u; // first
+                    }
                 } catch (MalformedURLException e) {
                     Assert.fail(String.format("invalid protocol endpoint: %s because %s", endpoint, e.getMessage()));
                 }
             }
+            Assert.assertNotNull(putURL);
+
+            // put the bytes
+            Random rnd = new Random();
+            byte[] data = new byte[1024];
+            rnd.nextBytes(data);
+            FileContent content = new FileContent(data, "application/octet-stream");
+            HttpUpload put = new HttpUpload(content, putURL);
+            put.run();
+            log.info("put: " + put.getResponseCode() + " " + put.getThrowable());
+            Assert.assertTrue(PUT_OK.contains(put.getResponseCode()));
+            Assert.assertNull(put.getThrowable());
 
             // Create a pull-from-vospace Transfer for the node
             Transfer pullTransfer = new Transfer(nodeURI.getURI(), Direction.pullFromVoSpace);
             pullTransfer.version = VOS.VOSPACE_21;
-            protocol = new Protocol(VOS.PROTOCOL_HTTPS_PUT);
-            protocol.setSecurityMethod(Standards.SECURITY_METHOD_CERT);
-            pullTransfer.getProtocols().add(protocol);
+            pullTransfer.getProtocols().add(new Protocol(VOS.PROTOCOL_HTTPS_GET)); // anon, preauth
+            Protocol getWithCert = new Protocol(VOS.PROTOCOL_HTTPS_GET);
+            getWithCert.setSecurityMethod(Standards.SECURITY_METHOD_CERT);
+            pullTransfer.getProtocols().add(getWithCert);
+
 
             // Do the transfer
             details = doTransfer(pullTransfer);
             Assert.assertEquals("expected transfer direction = " + Direction.pullFromVoSpace,
                     Direction.pullFromVoSpace, details.getDirection());
-            Assert.assertNotNull("expected > 0 protocols", details.getProtocols());
-            endpoint = null;
+            Assert.assertNotNull(details.getProtocols());
+            log.info(pullTransfer.getDirection() + " results: " + details.getProtocols().size());
+            URL getURL = null;
             for (Protocol p : details.getProtocols()) {
+                String endpoint = p.getEndpoint();
+                log.info("GET endpoint: " + endpoint);
                 try {
-                    endpoint = p.getEndpoint();
-                    log.debug("endpoint: " + endpoint);
-                    new URL(endpoint);
+                    URL u = new URL(endpoint);
+                    if (getURL == null) {
+                        getURL = u; // first
+                    }
                 } catch (MalformedURLException e) {
                     Assert.fail(String.format("invalid protocol endpoint: %s because %s", endpoint, e.getMessage()));
                 }
             }
-
+            Assert.assertNotNull(getURL);
+
+            // get the bytes
+            ByteArrayOutputStream bos = new ByteArrayOutputStream();
+            HttpGet get = new HttpGet(getURL, bos);
+            get.run();
+            log.info("get: " + get.getResponseCode() + " " + get.getContentType() + " " + get.getThrowable());
+            Assert.assertEquals(200, get.getResponseCode());
+            Assert.assertNull(get.getThrowable());
+            Assert.assertEquals(content.getBytes().length, get.getContentLength());
+            Assert.assertEquals(content.getContentType(), get.getContentType());
+            byte[] actual = bos.toByteArray();
+            Assert.assertArrayEquals(content.getBytes(), actual);
+
             // Delete the node
             delete(nodeURL, false);
 

cadc-vos-server/build.gradle

@@ -16,7 +16,7 @@ sourceCompatibility = 1.8
 
 group = 'org.opencadc'
 
-version = '2.0.4'
+version = '2.0.5'
 
 description = 'OpenCADC VOSpace server'
 def git_url = 'https://github.com/opencadc/vos'

cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/TransferDetailsServlet.java

@@ -105,6 +105,15 @@
 import org.opencadc.vospace.transfer.TransferReader;
 import org.opencadc.vospace.transfer.TransferWriter;
 
+/**
+ * Servlet to support output of a transfer result from 
+ * /transfers/{jobID}/results/transferDetails. Implementation
+ * detail/requirement: This servlet MUST be deployed as a 
+ * sibling of the /nodes endpoint and MUST be named /xfer 
+ * in the servlet mapping.
+ * 
+ * @author pdowler
+ */
 public class TransferDetailsServlet extends HttpServlet {
 
     private static final long serialVersionUID = 2022026164700L;

cadc-vos-server/src/main/java/org/opencadc/vospace/server/transfers/VOSpaceTransfer.java

@@ -165,9 +165,10 @@ protected void updateTransferJob(Node target, URI resolvedPath, ExecutionPhase e
             Subject s = AuthenticationUtil.getCurrentSubject();
             AuthMethod authMethod = AuthenticationUtil.getAuthMethod(s);
             // HACK: self-lookup
-            URL serviceURL = regClient.getServiceURL(locService.getURI(), Standards.VOSPACE_XFER_20, authMethod);
-            String path = "/" + job.getID();
-            URL url = new URL(serviceURL.toExternalForm() + path);
+            URL nodesURL = regClient.getServiceURL(locService.getURI(), Standards.VOSPACE_NODES_20, authMethod);
+            String xferURL = nodesURL.toExternalForm().replace("/nodes", "/xfer"); // hard coded ugh
+            String surl = xferURL + "/" + job.getID();
+            URL url = new URL(surl);
             log.debug("transfer URL: " + url);
 
             uri = url.toURI();

cavern/VERSION

@@ -1,6 +1,6 @@
 ## deployable containers have a semantic and build tag
 # semantic version tag: major.minor
 # build version tag: timestamp
-VER=0.6.1
+VER=0.6.2
 TAGS="${VER} ${VER}-$(date -u +"%Y%m%dT%H%M%S")"
 unset VER

cavern/src/main/java/org/opencadc/cavern/files/GetAction.java

@@ -67,9 +67,9 @@
 
 package org.opencadc.cavern.files;
 
+import ca.nrc.cadc.io.ByteCountOutputStream;
 import ca.nrc.cadc.net.ResourceNotFoundException;
 import java.io.FileNotFoundException;
-import java.io.OutputStream;
 import java.nio.file.AccessDeniedException;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
@@ -94,10 +94,11 @@ public GetAction() {
 
     @Override
     public void doAction()  throws Exception {
+        ByteCountOutputStream out = null;
         try {
             Path source = resolveAndSetMetadata();
 
-            OutputStream out = syncOutput.getOutputStream();
+            out = new ByteCountOutputStream(syncOutput.getOutputStream());
             log.debug("Starting copy of file " + source);
             Files.copy(source, out);
             log.debug("Completed copy of file " + source);
@@ -112,6 +113,10 @@ public void doAction()  throws Exception {
         } catch (AccessControlException | AccessDeniedException e) {
             log.debug(e);
             throw new AccessControlException(e.getMessage());
+        } finally {
+            if (out != null && out.getByteCount() > 0L) {
+                logInfo.setBytes(out.getByteCount());
+            }
         }
     }
 }

cavern/src/main/java/org/opencadc/cavern/files/PutAction.java

@@ -69,6 +69,7 @@
 
 import ca.nrc.cadc.auth.AuthenticationUtil;
 import ca.nrc.cadc.auth.HttpPrincipal;
+import ca.nrc.cadc.io.ByteCountOutputStream;
 import ca.nrc.cadc.io.ByteLimitExceededException;
 import ca.nrc.cadc.io.MultiBufferIO;
 import ca.nrc.cadc.net.ResourceNotFoundException;
@@ -97,7 +98,7 @@
 import org.opencadc.vospace.VOSURI;
 
 /**
- *
+ * @author pdowler
  * @author majorb
  * @author jeevesh
  */
@@ -132,6 +133,7 @@ public void doAction() throws Exception {
         boolean putStarted = false;
         boolean successful = false;
 
+        long bytesWritten = 0L;
         try {
             log.debug("put: start " + nodeURI.getURI().toASCIIString());
 
@@ -172,6 +174,13 @@ public void doAction() throws Exception {
                 throw new IllegalArgumentException("not a data node");
             }
             node = (DataNode) n;
+            if (node == null) {
+                log.warn("target node: " + node + ": creating");
+                node = new DataNode(nodeURI.getName());
+                node.owner = caller;
+                node.parent = cn;
+                nodePersistence.put(node);
+            }
 
             // check write permission
             if (!preauthGranted) {
@@ -195,12 +204,15 @@ public void doAction() throws Exception {
             //Files.copy(vis, target, StandardCopyOption.REPLACE_EXISTING);
 
             // truncate: do not recreate file with wrong owner
+            StandardOpenOption openOption = StandardOpenOption.TRUNCATE_EXISTING;
             DigestOutputStream out = new DigestOutputStream(
-                    Files.newOutputStream(target, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING), md);
+                    Files.newOutputStream(target, StandardOpenOption.WRITE, openOption), md);
+            ByteCountOutputStream bcos = new ByteCountOutputStream(out);
             MultiBufferIO io = new MultiBufferIO();
-            io.copy(in, out);
-            out.flush();
+            io.copy(in, bcos);
+            bcos.flush();
             log.debug("copy: done " + target);
+            bytesWritten = bcos.getByteCount();
 
             URI expectedMD5 = syncInput.getDigest();
             byte[] md5 = md.digest();
@@ -212,16 +224,14 @@ public void doAction() throws Exception {
                 OutputStream trunc = Files.newOutputStream(target, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
                 trunc.close();
                 actualMD5 = null;
-                //PosixFileAttributes attrs = Files.readAttributes(target, PosixFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
-                //log.warn("after truncate, size: " + attrs.size());
             }
 
             // re-read node from filesystem
             node = (DataNode) nodePersistence.get(cn, nodeURI.getName());
 
             // update Node
             node.owner = caller;
-            node.ownerID = identityManager.toPosixPrincipal(caller);
+            node.ownerID = null; // just in case
 
             log.debug(nodeURI + " MD5: " + propValue);
             NodeProperty csp = node.getProperty(VOS.PROPERTY_URI_CONTENTMD5);
@@ -240,6 +250,17 @@ public void doAction() throws Exception {
                     csp.setValue(actualMD5.toASCIIString());
                 }
             }
+            // set/update content-type attr
+            String contentType = syncInput.getHeader("content-type");
+            if (contentType != null) {
+                NodeProperty ctp = node.getProperty(VOS.PROPERTY_URI_TYPE);
+                if (ctp == null) {
+                    ctp = new NodeProperty(VOS.PROPERTY_URI_TYPE, contentType);
+                    node.getProperties().add(ctp);
+                } else {
+                    ctp.setValue(contentType);
+                }
+            }
 
             nodePersistence.put(node);
             successful = true;
@@ -279,6 +300,9 @@ public void doAction() throws Exception {
 
             throw ex;
         } finally {
+            if (bytesWritten > 0L) {
+                logInfo.setBytes(bytesWritten);
+            }
             if (successful) {
                 log.debug("put: done " + nodeURI.getURI().toASCIIString());
             } else if (putStarted) {