From d703aa33a82553678b88783d5dae0d868ffa4098 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Thu, 26 Oct 2023 20:34:05 -0700 Subject: [PATCH 01/21] CADC-12563 new PackageJobManager and PackageTest classes --- .../java/org/opencadc/cavern/PackageTest.java | 92 ++++++++++++++++ .../cavern/uws/PackageJobManager.java | 100 ++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 cavern/src/intTest/java/org/opencadc/cavern/PackageTest.java create mode 100644 cavern/src/main/java/org/opencadc/cavern/uws/PackageJobManager.java diff --git a/cavern/src/intTest/java/org/opencadc/cavern/PackageTest.java b/cavern/src/intTest/java/org/opencadc/cavern/PackageTest.java new file mode 100644 index 00000000..d42b9781 --- /dev/null +++ b/cavern/src/intTest/java/org/opencadc/cavern/PackageTest.java @@ -0,0 +1,92 @@ +/* + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2023. (c) 2023. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + * : 5 $ + * + ************************************************************************ + */ + +package org.opencadc.cavern; + +import ca.nrc.cadc.util.Log4jInit; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +public class PackageTest extends org.opencadc.conformance.vos.PackageTest { + + private static final Logger log = Logger.getLogger(PackageTest.class); + + static { + Log4jInit.setLevel("org.opencadc.conformance.vos", Level.DEBUG); + Log4jInit.setLevel("org.opencadc.vospace", Level.DEBUG); + Log4jInit.setLevel("org.opencadc.vospace.server.pkg", Level.DEBUG); + Log4jInit.setLevel("org.opencadc.cavern", Level.DEBUG); + Log4jInit.setLevel("ca.nrc.cadc.uws", Level.DEBUG); + } + + public PackageTest() { + super(Constants.RESOURCE_ID, Constants.TEST_CERT); + } + +} diff --git a/cavern/src/main/java/org/opencadc/cavern/uws/PackageJobManager.java b/cavern/src/main/java/org/opencadc/cavern/uws/PackageJobManager.java new file mode 100644 index 00000000..c3aeefb8 --- /dev/null +++ b/cavern/src/main/java/org/opencadc/cavern/uws/PackageJobManager.java @@ -0,0 +1,100 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2022. (c) 2022. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +* $Revision: 5 $ +* +************************************************************************ +*/ + +package org.opencadc.cavern.uws; + +import ca.nrc.cadc.uws.server.JobExecutor; +import ca.nrc.cadc.uws.server.JobPersistence; +import ca.nrc.cadc.uws.server.SyncJobExecutor; +import org.apache.log4j.Logger; +import org.opencadc.cavern.uws.CavernJobManager; +import org.opencadc.vospace.server.pkg.VospacePackageRunner; + + +public class PackageJobManager extends CavernJobManager { + private static final Logger log = Logger.getLogger(PackageJobManager.class); + + private static final Long MAX_EXEC_DURATION = 3600L; // proxied download + private static final Long MAX_DESTRUCTION = 7200L; + private static final Long MAX_QUOTE = 3600L; + + public PackageJobManager() { + super(); + + JobPersistence jobPersistence = createJobPersistence(); + super.setJobPersistence(jobPersistence); + + JobExecutor jobExecutor = new SyncJobExecutor(jobPersistence, VospacePackageRunner.class); + super.setJobExecutor(jobExecutor); + + super.setMaxExecDuration(MAX_EXEC_DURATION); + super.setMaxDestruction(MAX_DESTRUCTION); + super.setMaxQuote(MAX_QUOTE); + } +} From 1b0a11f0ea504cca2b10ce89ddefa0d72c5d6576 Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Thu, 26 Oct 2023 20:34:40 -0700 Subject: [PATCH 02/21] CADC-12563 update cavern capabilities and web.xml for pkg endpoint --- cavern/src/main/webapp/WEB-INF/web.xml | 51 +++++++++++++------------ cavern/src/main/webapp/capabilities.xml | 3 +- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/cavern/src/main/webapp/WEB-INF/web.xml b/cavern/src/main/webapp/WEB-INF/web.xml index 799f671f..f373f614 100644 --- a/cavern/src/main/webapp/WEB-INF/web.xml +++ b/cavern/src/main/webapp/WEB-INF/web.xml @@ -211,6 +211,29 @@ 3 + + + PackageServlet + ca.nrc.cadc.uws.server.JobServlet + + get + ca.nrc.cadc.uws.web.SyncGetAction + + + post + ca.nrc.cadc.uws.web.SyncPostAction + + + ca.nrc.cadc.uws.server.JobManager + org.opencadc.cavern.uws.PackageJobManager + + + ca.nrc.cadc.rest.InlineContentHandler + org.opencadc.vospace.server.transfers.InlineTransferHandler + + 3 + + ViewsServlet @@ -305,30 +328,10 @@ /xfer/* - + + PackageServlet + /pkg/* + diff --git a/cavern/src/main/webapp/capabilities.xml b/cavern/src/main/webapp/capabilities.xml index f28fd93d..fd1a4198 100644 --- a/cavern/src/main/webapp/capabilities.xml +++ b/cavern/src/main/webapp/capabilities.xml @@ -60,7 +60,6 @@ - + https://replace.com/cavern/nodeprops From 6e09d7e79a4786b5a8f0b559571e32df16dc028e Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Thu, 26 Oct 2023 20:35:31 -0700 Subject: [PATCH 03/21] CADC-12563 add VospacePackageRunner to cadc-vos-server-alt --- cadc-vos-server-alt/build.gradle | 1 + .../server/pkg/VospacePackageRunner.java | 327 ++++++++++++++++++ 2 files changed, 328 insertions(+) create mode 100644 cadc-vos-server-alt/src/main/java/org/opencadc/vospace/server/pkg/VospacePackageRunner.java diff --git a/cadc-vos-server-alt/build.gradle b/cadc-vos-server-alt/build.gradle index fa4d2680..0e37e043 100644 --- a/cadc-vos-server-alt/build.gradle +++ b/cadc-vos-server-alt/build.gradle @@ -34,6 +34,7 @@ dependencies { compile 'org.opencadc:cadc-cdp:[1.2.3,)' compile 'org.opencadc:cadc-registry:[1.7,2.0)' compile 'org.opencadc:cadc-gms:[1.0.5,2.0)' + compile 'org.opencadc:cadc-pkg-server:[1.0,)' testCompile 'junit:junit:4.13' } diff --git a/cadc-vos-server-alt/src/main/java/org/opencadc/vospace/server/pkg/VospacePackageRunner.java b/cadc-vos-server-alt/src/main/java/org/opencadc/vospace/server/pkg/VospacePackageRunner.java new file mode 100644 index 00000000..105282ae --- /dev/null +++ b/cadc-vos-server-alt/src/main/java/org/opencadc/vospace/server/pkg/VospacePackageRunner.java @@ -0,0 +1,327 @@ +/* + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2023. (c) 2023. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + * : 5 $ + * + ************************************************************************ + */ + +package org.opencadc.vospace.server.pkg; + +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.io.ResourceIterator; +import ca.nrc.cadc.net.TransientException; +import ca.nrc.cadc.uws.Job; +import ca.nrc.cadc.uws.JobInfo; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.security.AccessControlException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import javax.naming.Context; +import javax.naming.InitialContext; +import javax.naming.NamingException; +import javax.security.auth.Subject; +import org.apache.log4j.Logger; +import org.opencadc.pkg.server.PackageItem; +import org.opencadc.pkg.server.PackageRunner; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.LinkingException; +import org.opencadc.vospace.Node; +import org.opencadc.vospace.NodeNotFoundException; +import org.opencadc.vospace.VOSURI; +import org.opencadc.vospace.server.NodeFault; +import org.opencadc.vospace.server.NodePersistence; +import org.opencadc.vospace.server.PathResolver; +import org.opencadc.vospace.server.Utils; +import org.opencadc.vospace.server.auth.VOSpaceAuthorizer; +import org.opencadc.vospace.server.transfers.TransferGenerator; +import org.opencadc.vospace.transfer.Protocol; +import org.opencadc.vospace.transfer.Transfer; +import org.opencadc.vospace.transfer.TransferParsingException; +import org.opencadc.vospace.transfer.TransferReader; + +public class VospacePackageRunner extends PackageRunner { + + private static final Logger log = Logger.getLogger(VospacePackageRunner.class); + private static final String CONTINUING_PROCESSING = ", continuing package processing..."; + private URI resourceID; + private Transfer packageTransfer; + private NodePersistence nodePersistence; + private VOSpaceAuthorizer vospaceAuthorizer; + private PathResolver pathResolver; + + public VospacePackageRunner() {} + + /** + * + * @param appName + */ + @Override + public void setAppName(String appName) { + String jndiKey = appName + "-" + NodePersistence.class.getName(); + try { + Context ctx = new InitialContext(); + this.nodePersistence = (NodePersistence) ctx.lookup(jndiKey); + this.vospaceAuthorizer = new VOSpaceAuthorizer(nodePersistence); + this.pathResolver = new PathResolver(nodePersistence, vospaceAuthorizer, true); + this.resourceID = nodePersistence.getResourceID(); + } catch (NamingException e) { + throw new RuntimeException("BUG: NodePersistence implementation not found with JNDI key " + jndiKey, e); + } + } + + /** + * + * @throws IllegalArgumentException + */ + @Override + protected void initPackage() throws IllegalArgumentException { + log.debug("initPackage start"); + try { + // check job is valid + log.debug("job id passed in: " + job.getID()); + JobInfo jobInfo = job.getJobInfo(); + if (jobInfo != null) { + throw new RuntimeException("null jobInfo: " + job.toString()); + } + + // Get the target list from the job + TransferReader tr = new TransferReader(); + this.packageTransfer = tr.read(jobInfo.getContent(), VOSURI.SCHEME); + List targetList = packageTransfer.getTargets(); + + if (targetList.size() > 1) { + this.packageName = "cadc-download-" + job.getID(); + } else { + this.packageName = getFilenameFromURI(targetList.get(0)); + } + log.debug("package name: " + this.packageName); + } catch (IOException | TransferParsingException e) { + throw new RuntimeException("ERROR reading jobInfo: ", e); + } + log.debug("initPackage done"); + } + + /** + * + * @return + * @throws IOException + */ + @Override + protected Iterator getItems() throws IOException { + // packageTransfer is populated in initPackage + List targetList = packageTransfer.getTargets(); + log.debug("target list: " + targetList.toString()); + + Subject caller = AuthenticationUtil.getCurrentSubject(); + log.debug("subject: " + caller); + + List packageItems = new ArrayList<>(); + for (URI targetURI : targetList) { + VOSURI nodeURI = new VOSURI(targetURI); + log.debug("node: " + nodeURI); + + addPackageItem(nodeURI, packageItems, caller); + } + return packageItems.iterator(); + } + + /** + * + * @param nodeURI + * @param packageItems + * @param caller + */ + private void addPackageItem(VOSURI nodeURI, List packageItems, Subject caller) { + String nodePath = nodeURI.getPath(); + log.debug("node path: " + nodePath); + + try { + // PathResolver checks read permission for the node path + Node node = pathResolver.getNode(nodePath); + if (node == null) { + throw NodeFault.NodeNotFound.getStatus(nodePath); + } + + if (node instanceof DataNode) { + addDataNode((DataNode) node, packageItems); + } else if (node instanceof ContainerNode) { + addContainerNode((ContainerNode) node, packageItems, caller); + } else { + log.info("unrecognized or unsupported node type " + nodeURI + CONTINUING_PROCESSING); + } + } catch (AccessControlException e) { + log.info("permission denied: " + nodeURI + CONTINUING_PROCESSING); + } catch (NodeNotFoundException e) { + log.info("node not found: " + nodeURI + CONTINUING_PROCESSING); + } catch (FileNotFoundException e) { + log.info("couldn't generate endpoints: " + nodeURI + CONTINUING_PROCESSING); + } catch (LinkingException e) { + log.info("linking exception: " + nodeURI + CONTINUING_PROCESSING); + } catch (MalformedURLException e) { + log.info("malformed URL: " + nodeURI + CONTINUING_PROCESSING); + } catch (TransientException e) { + log.info("transientException: " + nodeURI + CONTINUING_PROCESSING); + } catch (Exception e) { + log.info(String.format("%s: %s%s", e.getClass().getName(), nodeURI, CONTINUING_PROCESSING)); + } + } + + private void addDataNode(DataNode node, List packageItems) + throws Exception { + String nodePath = Utils.getPath(node); + log.debug("DataNode path: " + nodePath); + + VOSURI nodeURI = new VOSURI(this.resourceID, nodePath); + log.debug("DataNode uri: " + nodeURI); + + // Use a temporary Job with just the remote IP to avoid the original Job in + // a transfer URL which may close the Job. + Job pkgJob = new Job(); + pkgJob.setRemoteIP(this.job.getRemoteIP()); + TransferGenerator transferGenerator = nodePersistence.getTransferGenerator(); + List protocols = transferGenerator.getEndpoints(nodeURI, packageTransfer, pkgJob, null); + + // Get the node endpoint from the first protocol + Protocol protocol = protocols.get(0); + String endpoint = protocol.getEndpoint(); + log.debug("DataNode endpoint:" + endpoint); + + packageItems.add(new PackageItem(new URL(endpoint), nodePath)); + } + + private void addContainerNode(ContainerNode node, List packageItems, Subject caller) + throws Exception { + + List childContainers = new ArrayList<>(); + + try (ResourceIterator iterator = nodePersistence.iterator(node, null, null)) { + while (iterator.hasNext()) { + Node child = iterator.next(); + boolean canRead = vospaceAuthorizer.hasSingleNodeReadPermission(child, caller); + if (child instanceof ContainerNode) { + if (canRead) { + childContainers.add((ContainerNode) child); + } else { + // add empty container node to the package + packageItems.add(getPackageItem(node)); + } + } else if (child instanceof DataNode) { + if (canRead) { + addDataNode((DataNode) child, packageItems); + } + } else { + log.info("unrecognized or unsupported node type " + Utils.getPath(node) + CONTINUING_PROCESSING); + } + } + } + + if (!childContainers.isEmpty()) { + for (ContainerNode child : childContainers) { + addContainerNode(child, packageItems, caller); + } + } + } + + /** + * Build a filename from the URI provided + * @return - name of last element in URI path. + */ + private static String getFilenameFromURI(URI uri) { + String path = uri.getPath(); + int i = path.lastIndexOf("/"); + if (i >= 0) { + path = path.substring(i + 1); + } + return path; + } + + private PackageItem getPackageItem(Node node) throws Exception { + String nodePath = Utils.getPath(node); + VOSURI nodeURI = new VOSURI(this.resourceID, nodePath); + log.debug("Node uri: " + nodeURI); + + // Use a temporary Job with just the remote IP to avoid the original Job in + // a transfer URL which may close the Job. + // Use a temporary Job with just the remote IP to avoid the original Job in + // a transfer URL which may close the Job. + Job pkgJob = new Job(); + pkgJob.setRemoteIP(this.job.getRemoteIP()); + TransferGenerator transferGenerator = nodePersistence.getTransferGenerator(); + List protocols = transferGenerator.getEndpoints(nodeURI, packageTransfer, pkgJob, null); + + // Get the node endpoint from the first protocol + Protocol protocol = protocols.get(0); + URL endpoint = new URL(protocol.getEndpoint()); + log.debug("Node endpoint:" + endpoint); + return new PackageItem(endpoint, nodePath); + } + +} From 477da4f73924d0eb37d081b8e4dc588b09c7fc3e Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Thu, 26 Oct 2023 20:36:04 -0700 Subject: [PATCH 04/21] Initial PackageRunner int tests --- cadc-test-vos/build.gradle | 4 +- .../opencadc/conformance/vos/PackageTest.java | 505 ++++++++++++++++++ .../org/opencadc/conformance/vos/VOSTest.java | 4 + 3 files changed, 512 insertions(+), 1 deletion(-) create mode 100644 cadc-test-vos/src/main/java/org/opencadc/conformance/vos/PackageTest.java diff --git a/cadc-test-vos/build.gradle b/cadc-test-vos/build.gradle index 8d8f950a..dde5c468 100644 --- a/cadc-test-vos/build.gradle +++ b/cadc-test-vos/build.gradle @@ -12,7 +12,7 @@ repositories { apply from: '../opencadc.gradle' -sourceCompatibility = 1.8 +sourceCompatibility = 11 group = 'org.opencadc' @@ -25,8 +25,10 @@ dependencies { implementation 'org.opencadc:cadc-util:[1.6,2.0)' implementation 'org.opencadc:cadc-gms:[1.0.5,)' implementation 'org.opencadc:cadc-vos:[2.0,)' + implementation 'org.opencadc:cadc-vos-client:[2.0,)' implementation 'org.opencadc:cadc-uws:[1.0,2.0)' implementation 'org.opencadc:cadc-registry:[1.7.4,2.0)' implementation 'junit:junit:[4.0,)' + implementation 'org.apache.commons:commons-compress:[1.12,)' } diff --git a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/PackageTest.java b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/PackageTest.java new file mode 100644 index 00000000..c79ecf06 --- /dev/null +++ b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/PackageTest.java @@ -0,0 +1,505 @@ +/* + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2023. (c) 2023. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + * : 5 $ + * + ************************************************************************ + */ + +package org.opencadc.conformance.vos; + +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.RunnableAction; +import ca.nrc.cadc.net.FileContent; +import ca.nrc.cadc.net.HttpGet; +import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.net.OutputStreamWrapper; +import ca.nrc.cadc.reg.Standards; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringWriter; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.AccessControlException; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import javax.security.auth.Subject; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveException; +import org.apache.commons.compress.archivers.ArchiveInputStream; +import org.apache.commons.compress.archivers.ArchiveStreamFactory; +import org.apache.commons.compress.utils.IOUtils; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; +import org.opencadc.vospace.ContainerNode; +import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.VOS; +import org.opencadc.vospace.VOSURI; +import org.opencadc.vospace.View; +import org.opencadc.vospace.client.ClientTransfer; +import org.opencadc.vospace.client.VOSpaceClient; +import org.opencadc.vospace.transfer.Direction; +import org.opencadc.vospace.transfer.Protocol; +import org.opencadc.vospace.transfer.Transfer; +import org.opencadc.vospace.transfer.TransferParsingException; +import org.opencadc.vospace.transfer.TransferReader; +import org.opencadc.vospace.transfer.TransferWriter; + +public class PackageTest extends VOSTest { + private static final Logger log = Logger.getLogger(PackageTest.class); + + private static final String ZIP_CONTENT_TYPE = "application/zip"; + private static final String TAR_CONTENT_TYPE = "application/x-tar"; +// private static Subject s; + +// private static VOSURI nodeURI; + + protected PackageTest(URI resourceID, File testCert) { + super(resourceID, testCert); + } + + @Test + public void tarPermissionDeniedTest () { + try { + permissionDeniedTest(TAR_CONTENT_TYPE); + } catch (Exception e) { + log.error("Unexpected error", e); + Assert.fail("Unexpected error: " + e); + } + } + + @Test + public void zipPermissionDeniedTest () { + try { + permissionDeniedTest(ZIP_CONTENT_TYPE); + } catch (Exception e) { + log.error("Unexpected error", e); + Assert.fail("Unexpected error: " + e); + } + } + + @Test + public void tarTargetNotFoundTest () { + try { + targetNotFoundTest(TAR_CONTENT_TYPE); + } catch (Exception e) { + log.error("Unexpected error", e); + Assert.fail("Unexpected error: " + e); + } + } + + @Test + public void zipTargetNotFoundTest () { + try { + targetNotFoundTest(ZIP_CONTENT_TYPE); + } catch (Exception e) { + log.error("Unexpected error", e); + Assert.fail("Unexpected error: " + e); + } + } + + @Test + public void tarEmptyTargetTest() { + try { + emptyTargetTest(TAR_CONTENT_TYPE); + } catch (Exception e) { + log.error("Unexpected error", e); + Assert.fail("Unexpected error: " + e); + } + } + + @Test + public void zipEmptyTargetTest() { + try { + emptyTargetTest(ZIP_CONTENT_TYPE); + } catch (Exception e) { + log.error("Unexpected error", e); + Assert.fail("Unexpected error: " + e); + } + } + + @Test + public void singleTargetTest() { + try { + + } catch (Exception e) { + log.error("Unexpected error", e); + Assert.fail("Unexpected error: " + e); + } + } + + @Test + public void multipleTargetTest() { + try { + + } catch (Exception e) { + log.error("Unexpected error", e); + Assert.fail("Unexpected error: " + e); + } + } + + + public void permissionDeniedTest(String contentType) throws Exception { + // Root container node created by auth subject + String name = "permission-denied-node"; + URL nodeURL = getNodeURL(nodesServiceURL, name); + + // cleanup + delete(nodeURL, false); + + VOSURI nodeURI = getVOSURI(name); + ContainerNode node = new ContainerNode(name); + put(nodeURL, nodeURI, node); + log.debug(String.format("PUT %s URL: %s", name, nodeURL)); + + List targets = new ArrayList<>(); + targets.add(nodeURI.getURI()); + + // Package request using anon subject + Subject anonSubject = AuthenticationUtil.getAnonSubject(); + + // download and extract the package + try { + downloadPackage(targets, name, null, anonSubject, contentType); + Assert.fail("should have thrown exception for permission denied"); + } catch (AccessControlException e) { + log.debug("expected exception: " + e.getMessage()); + } + + // cleanup + delete(nodeURL, false); + } + + public void targetNotFoundTest(String contentType) throws Exception { + String name = "target-not-found-node"; + VOSURI nodeURI = getVOSURI(name); + + List targets = new ArrayList<>(); + targets.add(nodeURI.getURI()); + + // download and extract the package + try { + downloadPackage(targets, name, null, authSubject, contentType); + Assert.fail("should have thrown exception for target not found"); + } catch (AccessControlException e) { + log.debug("expected exception: " + e.getMessage()); + } + } + + public void emptyTargetTest(String contentType) throws Exception { + + String name = "empty-target-node"; + URL nodeURL = getNodeURL(nodesServiceURL, name); + + // cleanup + delete(nodeURL, false); + + VOSURI nodeURI = getVOSURI(name); + ContainerNode node = new ContainerNode(name); + put(nodeURL, nodeURI, node); + log.debug(String.format("PUT %s URL: %s", name, nodeURL)); + + List targets = new ArrayList<>(); + targets.add(nodeURI.getURI()); + + // download and extract the package + Path packageRoot = downloadPackage(targets, name, null, authSubject, contentType); + + File[] files = packageRoot.toFile().listFiles(); + Assert.assertNotNull(files); + Assert.assertEquals("expected single file", 1, files.length); + Assert.assertTrue("expected directory", files[0].isDirectory()); + Assert.assertEquals("", name, files[0].getName()); + + // cleanup + delete(nodeURL, false); + } + + public void singleTargetTest(String contentType) throws Exception { + + String name = "single-target-node"; + URL nodeURL = getNodeURL(nodesServiceURL, name); + + // cleanup + delete(nodeURL, false); + + // PUT the node + VOSURI nodeURI = getVOSURI(name); + ContainerNode node = new ContainerNode(name); + put(nodeURL, nodeURI, node); + log.debug(String.format("PUT %s URL: %s", name, nodeURL)); + + List targets = new ArrayList<>(); + targets.add(nodeURI.getURI()); + + // Upload + + // download and extract the package + Path packageRoot = downloadPackage(targets, name, null, authSubject, contentType); + + File[] files = packageRoot.toFile().listFiles(); + Assert.assertNotNull(files); + Assert.assertEquals("expected single file", 1, files.length); + Assert.assertTrue("expected directory", files[0].isDirectory()); + Assert.assertEquals("", name, files[0].getName()); + + // cleanup + delete(nodeURL, false); + } + + private void uploadFile(String filename, String content) + throws MalformedURLException, IOException, TransferParsingException { + // Put a file (DataNode) + URL nodeURL = getNodeURL(nodesServiceURL, filename); + VOSURI nodeURI = getVOSURI(filename); + log.debug("upload node URL: " + nodeURL); + + // Create a Transfer + Transfer transfer = new Transfer(nodeURI.getURI(), Direction.pushToVoSpace); + transfer.version = VOS.VOSPACE_21; + Protocol protocol = new Protocol(VOS.PROTOCOL_HTTPS_PUT); + protocol.setSecurityMethod(Standards.SECURITY_METHOD_CERT); + transfer.getProtocols().add(protocol); + + // Get the transfer document + TransferWriter writer = new TransferWriter(); + StringWriter sw = new StringWriter(); + writer.write(transfer, sw); + log.debug("uploadFile transfer XML: " + sw); + + // POST the transfer document + FileContent fileContent = new FileContent(sw.toString().getBytes(), VOSTest.XML_CONTENT_TYPE); + URL transferURL = getNodeURL(synctransServiceURL, filename); + log.debug("transfer URL: " + transferURL); + HttpPost post = new HttpPost(synctransServiceURL, fileContent, false); + Subject.doAs(authSubject, new RunnableAction(post)); + Assert.assertEquals("expected POST response code = 303", 303, post.getResponseCode()); + Assert.assertNull("expected PUT throwable == null", post.getThrowable()); + + // Get the transfer details + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpGet get = new HttpGet(post.getRedirectURL(), out); + log.debug("GET: " + post.getRedirectURL()); + Subject.doAs(authSubject, new RunnableAction(get)); + log.debug("GET responseCode: " + get.getResponseCode()); + Assert.assertEquals("expected GET response code = 200", 200, get.getResponseCode()); + Assert.assertNull("expected GET throwable == null", get.getThrowable()); + Assert.assertTrue("expected GET Content-Type starting with " + VOSTest.XML_CONTENT_TYPE, + get.getContentType().startsWith(VOSTest.XML_CONTENT_TYPE)); + + // Get the endpoint from the transfer details + log.debug("transfer details XML: " + out); + TransferReader transferReader = new TransferReader(); + Transfer details = transferReader.read(out.toString(), "vos"); + Assert.assertEquals("expected transfer direction = " + Direction.pushToVoSpace, + Direction.pushToVoSpace, details.getDirection()); + Assert.assertFalse("expected > 0 endpoints", details.getProtocols().isEmpty()); + URL endpoint = new URL(details.getProtocols().get(0).getEndpoint()); + + // PUT a file to the endpoint + log.info("PUT: " + endpoint); + ByteArrayInputStream is = new ByteArrayInputStream(content.getBytes()); + put(endpoint, is, VOSTest.TEXT_CONTENT_TYPE); + } + + private void uploadFile(VOSURI target, File uploadFile) + throws PrivilegedActionException { + List protocols = new ArrayList(); + Protocol basicTLS = new Protocol(VOS.PROTOCOL_HTTPS_PUT); + basicTLS.setSecurityMethod(Standards.SECURITY_METHOD_CERT); + protocols.add(basicTLS); +// DataNode targetNode = new DataNode(target.getName()); + log.debug("uploading: " + target.getURI().toASCIIString()); + Transfer transfer = new Transfer(target.getURI(), Direction.pushToVoSpace); + transfer.getProtocols().addAll(protocols); + transfer.version = VOS.VOSPACE_21; + + final VOSpaceClient voSpaceClient = new VOSpaceClient(resourceID); + final ClientTransfer clientTransfer = Subject.doAs(authSubject, new CreateTransferAction(voSpaceClient, transfer, false)); + clientTransfer.setOutputStreamWrapper(out -> { + InputStream in = new FileInputStream(uploadFile); + try { + in.transferTo(out); + } finally { + in.close(); + } + }); + Subject.doAs(authSubject, (PrivilegedExceptionAction) () -> { + clientTransfer.runTransfer(); + return null; + }); + + } + + private Path downloadPackage(List targets, String expectedFilename, List expectedFilenames, + Subject testSubject, String contentType) + throws Exception { + + // Create a Transfer + Transfer transfer = new Transfer(Direction.pullFromVoSpace); + transfer.getTargets().addAll(targets); + View packageView = new View(Standards.PKG_10); + packageView.getParameters().add(new View.Parameter(VOS.PROPERTY_URI_FORMAT, contentType)); + transfer.setView(packageView); + Protocol protocol = new Protocol(VOS.PROTOCOL_HTTPS_GET); + protocol.setSecurityMethod(Standards.SECURITY_METHOD_CERT); + transfer.getProtocols().add(protocol); + + // Write a transfer document + TransferWriter transferWriter = new TransferWriter(); + StringWriter sw = new StringWriter(); + transferWriter.write(transfer, sw); + log.debug("transfer XML: " + sw); + + // POST the transfer + FileContent fileContent = new FileContent(sw.toString().getBytes(), VOSTest.XML_CONTENT_TYPE); + HttpPost post = new HttpPost(pkgServiceURL, fileContent, false); + Subject.doAs(testSubject, new RunnableAction(post)); + Assert.assertEquals("expected POST response code = 303",303, post.getResponseCode()); + Assert.assertNull("expected POST throwable == null", post.getThrowable()); + URL redirctURL = post.getRedirectURL(); + + // Download the package + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpGet get = new HttpGet(redirctURL, out); + log.debug("GET: " + redirctURL); + Subject.doAs(authSubject, new RunnableAction(get)); + log.debug("GET responseCode: " + get.getResponseCode()); + Assert.assertEquals("expected GET response code = 200", 200, get.getResponseCode()); + Assert.assertNull("expected GET throwable == null", get.getThrowable()); + Assert.assertTrue(String.format("expected GET Content-Type %s, found %s ", contentType, get.getContentType()), + get.getContentType().startsWith(contentType)); + + // Extract the package into a tmp directory + InputStream inputStream = new ByteArrayInputStream(out.toByteArray()); + String extractDirectory = String.format("%s%s", System.getProperty("java.io.tmpdir"), UUID.randomUUID()); + String archiveType = contentType.equals(ZIP_CONTENT_TYPE) ? ArchiveStreamFactory.ZIP : ArchiveStreamFactory.TAR; + extractArchive(inputStream, extractDirectory, archiveType); + log.debug(String.format("extract package: %s -> %s", redirctURL, extractDirectory)); + + return Paths.get(extractDirectory); + } + + private void extractArchive(InputStream inputStream, String extractDirectory, String archiveType) + throws ArchiveException, IOException { + ArchiveStreamFactory archiveStreamFactory = new ArchiveStreamFactory(); + ArchiveInputStream archiveInputStream = archiveStreamFactory.createArchiveInputStream( + archiveType, inputStream); + ArchiveEntry archiveEntry; + while((archiveEntry = archiveInputStream.getNextEntry()) != null) { + Path path = Paths.get(extractDirectory, archiveEntry.getName()); + File file = path.toFile(); + if(archiveEntry.isDirectory()) { + if(!file.isDirectory()) { + file.mkdirs(); + } + } else { + File parent = file.getParentFile(); + if(!parent.isDirectory()) { + parent.mkdirs(); + } + try (OutputStream outputStream = Files.newOutputStream(path)) { + IOUtils.copy(archiveInputStream, outputStream); + } + } + } + } + + static class CreateTransferAction implements PrivilegedExceptionAction { + + VOSpaceClient voSpaceClient; + Transfer transfer; + boolean run; + + CreateTransferAction(VOSpaceClient voSpaceClient, Transfer transfer, boolean run) { + this.voSpaceClient = voSpaceClient; + this.transfer = transfer; + this.run = run; + } + + @Override + public ClientTransfer run() throws Exception { + ClientTransfer clientTransfer = voSpaceClient.createTransfer(transfer); + if (run) { + clientTransfer.run(); + } + return clientTransfer; + } + + } + +} diff --git a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/VOSTest.java b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/VOSTest.java index c78eeecd..c0497241 100644 --- a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/VOSTest.java +++ b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/VOSTest.java @@ -127,6 +127,7 @@ public abstract class VOSTest { public final URL synctransServiceURL; public final URL transferServiceURL; public final URL recursiveDeleteServiceURL; + public final URL pkgServiceURL; public final Subject authSubject; protected VOSTest(URI resourceID, File testCert) { @@ -149,6 +150,9 @@ protected VOSTest(URI resourceID, File testCert) { this.recursiveDeleteServiceURL = regClient.getServiceURL(resourceID, Standards.VOSPACE_RECURSIVE_DELETE, AuthMethod.CERT); log.info(String.format("%s: %s", Standards.VOSPACE_RECURSIVE_DELETE, recursiveDeleteServiceURL)); + + this.pkgServiceURL = regClient.getServiceURL(resourceID, Standards.PKG_10, AuthMethod.CERT); + log.info(String.format("%s: %s", Standards.PKG_10, pkgServiceURL)); } @Before From 7af9bb5d6721f98cfeecc4c05d86d351a268562c Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Fri, 3 Nov 2023 12:22:28 -0700 Subject: [PATCH 05/21] CADC-12563 better int-test logging --- cadc-test-vos/build.gradle | 2 +- .../opencadc/conformance/vos/PackageTest.java | 334 ++++++++++++------ .../server/pkg/VospacePackageRunner.java | 124 ++++--- cavern/build.gradle | 2 +- .../cavern/files/CavernURLGenerator.java | 1 + cavern/src/main/webapp/WEB-INF/web.xml | 7 +- 6 files changed, 308 insertions(+), 162 deletions(-) diff --git a/cadc-test-vos/build.gradle b/cadc-test-vos/build.gradle index dde5c468..3bc57c51 100644 --- a/cadc-test-vos/build.gradle +++ b/cadc-test-vos/build.gradle @@ -16,7 +16,7 @@ sourceCompatibility = 11 group = 'org.opencadc' -version = '2.0' +version = '2.1' description = 'OpenCADC VOSpace test library' def git_url = 'https://github.com/opencadc/vos' diff --git a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/PackageTest.java b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/PackageTest.java index c79ecf06..020d1f2d 100644 --- a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/PackageTest.java +++ b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/PackageTest.java @@ -71,7 +71,9 @@ import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.RunnableAction; +import ca.nrc.cadc.io.ByteCountOutputStream; import ca.nrc.cadc.net.FileContent; +import ca.nrc.cadc.net.HttpDownload; import ca.nrc.cadc.net.HttpGet; import ca.nrc.cadc.net.HttpPost; import ca.nrc.cadc.net.OutputStreamWrapper; @@ -80,6 +82,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -94,8 +97,11 @@ import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.security.auth.Subject; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveException; @@ -153,7 +159,7 @@ public void zipPermissionDeniedTest () { } @Test - public void tarTargetNotFoundTest () { + public void tarNodeNotFoundTest () { try { targetNotFoundTest(TAR_CONTENT_TYPE); } catch (Exception e) { @@ -163,7 +169,7 @@ public void tarTargetNotFoundTest () { } @Test - public void zipTargetNotFoundTest () { + public void zipNodeNotFoundTest () { try { targetNotFoundTest(ZIP_CONTENT_TYPE); } catch (Exception e) { @@ -173,7 +179,7 @@ public void zipTargetNotFoundTest () { } @Test - public void tarEmptyTargetTest() { + public void tarEmptyNodeTest() { try { emptyTargetTest(TAR_CONTENT_TYPE); } catch (Exception e) { @@ -183,7 +189,7 @@ public void tarEmptyTargetTest() { } @Test - public void zipEmptyTargetTest() { + public void zipEmptyNodeTest() { try { emptyTargetTest(ZIP_CONTENT_TYPE); } catch (Exception e) { @@ -193,9 +199,29 @@ public void zipEmptyTargetTest() { } @Test - public void singleTargetTest() { + public void tarSingleNodeTest() { try { + singleTargetTest(TAR_CONTENT_TYPE); + } catch (Exception e) { + log.error("Unexpected error", e); + Assert.fail("Unexpected error: " + e); + } + } + @Test + public void zipSingleNodeTest() { + try { + singleTargetTest(ZIP_CONTENT_TYPE); + } catch (Exception e) { + log.error("Unexpected error", e); + Assert.fail("Unexpected error: " + e); + } + } + + @Test + public void tarParentChildTest() { + try { + parentChildTest(TAR_CONTENT_TYPE); } catch (Exception e) { log.error("Unexpected error", e); Assert.fail("Unexpected error: " + e); @@ -234,7 +260,7 @@ public void permissionDeniedTest(String contentType) throws Exception { // download and extract the package try { - downloadPackage(targets, name, null, anonSubject, contentType); + File archive = downloadPackage(targets, contentType, authSubject); Assert.fail("should have thrown exception for permission denied"); } catch (AccessControlException e) { log.debug("expected exception: " + e.getMessage()); @@ -253,7 +279,7 @@ public void targetNotFoundTest(String contentType) throws Exception { // download and extract the package try { - downloadPackage(targets, name, null, authSubject, contentType); + File archive = downloadPackage(targets, contentType, authSubject); Assert.fail("should have thrown exception for target not found"); } catch (AccessControlException e) { log.debug("expected exception: " + e.getMessage()); @@ -277,13 +303,13 @@ public void emptyTargetTest(String contentType) throws Exception { targets.add(nodeURI.getURI()); // download and extract the package - Path packageRoot = downloadPackage(targets, name, null, authSubject, contentType); + File archive = downloadPackage(targets, contentType, authSubject); - File[] files = packageRoot.toFile().listFiles(); - Assert.assertNotNull(files); - Assert.assertEquals("expected single file", 1, files.length); - Assert.assertTrue("expected directory", files[0].isDirectory()); - Assert.assertEquals("", name, files[0].getName()); +// File[] files = packageRoot.toFile().listFiles(); +// Assert.assertNotNull(files); +// Assert.assertEquals("expected single file", 1, files.length); +// Assert.assertTrue("expected directory", files[0].isDirectory()); +// Assert.assertEquals("", name, files[0].getName()); // cleanup delete(nodeURL, false); @@ -291,37 +317,82 @@ public void emptyTargetTest(String contentType) throws Exception { public void singleTargetTest(String contentType) throws Exception { - String name = "single-target-node"; + String name = "single-target-node.txt"; + String content = "single-target-node.txt content"; URL nodeURL = getNodeURL(nodesServiceURL, name); + VOSURI nodeURI = getVOSURI(name); // cleanup - delete(nodeURL, false); +// delete(nodeURL, false); - // PUT the node - VOSURI nodeURI = getVOSURI(name); - ContainerNode node = new ContainerNode(name); - put(nodeURL, nodeURI, node); - log.debug(String.format("PUT %s URL: %s", name, nodeURL)); + // Upload +// uploadFile(name, content, authSubject); List targets = new ArrayList<>(); targets.add(nodeURI.getURI()); - // Upload - // download and extract the package - Path packageRoot = downloadPackage(targets, name, null, authSubject, contentType); + File archive = downloadPackage(targets, contentType, authSubject); - File[] files = packageRoot.toFile().listFiles(); - Assert.assertNotNull(files); - Assert.assertEquals("expected single file", 1, files.length); - Assert.assertTrue("expected directory", files[0].isDirectory()); - Assert.assertEquals("", name, files[0].getName()); +// File[] files = packageRoot.toFile().listFiles(); +// Assert.assertNotNull(files); +// Assert.assertEquals("expected single file", 1, files.length); +// Assert.assertTrue("expected directory", files[0].isDirectory()); +// Assert.assertEquals("", name, files[0].getName()); // cleanup - delete(nodeURL, false); +// delete(nodeURL, false); + } + + public void parentChildTest(String contentType) throws Exception { + + // parent ContainerNode + String parent = "foo-parent-node"; + URL parentNodeURL = getNodeURL(nodesServiceURL, parent); + + // child DataNode + String child = "foo-child-node.txt"; + URL childNodeURL = getNodeURL(nodesServiceURL, parent + "/" + child); + + // cleanup +// delete(childNodeURL, false); +// delete(parentNodeURL, false); + + // PUT parent node + VOSURI parentNodeURI = getVOSURI(parent); + ContainerNode node = new ContainerNode(parent); +// put(parentNodeURL, parentNodeURI, node); + log.debug(String.format("PUT %s URL: %s", parent, parentNodeURL)); + + // Upload child node + String content = "foo-child-node content"; +// uploadFile(parent + "/" + child, content, authSubject); + + // download the package + List targets = new ArrayList<>(); + targets.add(parentNodeURI.getURI()); + File pkg = downloadPackage(targets, contentType, authSubject); + Assert.assertNotNull(pkg); + log.debug("archive file: " + pkg.getAbsolutePath()); + + // extract the archive + File extracted = extractPackage(pkg, contentType); + Assert.assertNotNull(extracted); + log.debug("extracted file: " + extracted.getAbsolutePath()); + + // verify package files + List expected = new ArrayList<>(); + expected.add(child); + verifyPackage(expected, extracted.getAbsolutePath()); + + // cleanup + delete(childNodeURL, false); + delete(parentNodeURL, false); + deleteFile(pkg); + deleteFile(extracted); } - private void uploadFile(String filename, String content) + private void uploadFile(String filename, String content, Subject caller) throws MalformedURLException, IOException, TransferParsingException { // Put a file (DataNode) URL nodeURL = getNodeURL(nodesServiceURL, filename); @@ -346,7 +417,7 @@ private void uploadFile(String filename, String content) URL transferURL = getNodeURL(synctransServiceURL, filename); log.debug("transfer URL: " + transferURL); HttpPost post = new HttpPost(synctransServiceURL, fileContent, false); - Subject.doAs(authSubject, new RunnableAction(post)); + Subject.doAs(caller, new RunnableAction(post)); Assert.assertEquals("expected POST response code = 303", 303, post.getResponseCode()); Assert.assertNull("expected PUT throwable == null", post.getThrowable()); @@ -354,7 +425,7 @@ private void uploadFile(String filename, String content) ByteArrayOutputStream out = new ByteArrayOutputStream(); HttpGet get = new HttpGet(post.getRedirectURL(), out); log.debug("GET: " + post.getRedirectURL()); - Subject.doAs(authSubject, new RunnableAction(get)); + Subject.doAs(caller, new RunnableAction(get)); log.debug("GET responseCode: " + get.getResponseCode()); Assert.assertEquals("expected GET response code = 200", 200, get.getResponseCode()); Assert.assertNull("expected GET throwable == null", get.getThrowable()); @@ -376,37 +447,36 @@ private void uploadFile(String filename, String content) put(endpoint, is, VOSTest.TEXT_CONTENT_TYPE); } - private void uploadFile(VOSURI target, File uploadFile) - throws PrivilegedActionException { - List protocols = new ArrayList(); - Protocol basicTLS = new Protocol(VOS.PROTOCOL_HTTPS_PUT); - basicTLS.setSecurityMethod(Standards.SECURITY_METHOD_CERT); - protocols.add(basicTLS); -// DataNode targetNode = new DataNode(target.getName()); - log.debug("uploading: " + target.getURI().toASCIIString()); - Transfer transfer = new Transfer(target.getURI(), Direction.pushToVoSpace); - transfer.getProtocols().addAll(protocols); - transfer.version = VOS.VOSPACE_21; - - final VOSpaceClient voSpaceClient = new VOSpaceClient(resourceID); - final ClientTransfer clientTransfer = Subject.doAs(authSubject, new CreateTransferAction(voSpaceClient, transfer, false)); - clientTransfer.setOutputStreamWrapper(out -> { - InputStream in = new FileInputStream(uploadFile); - try { - in.transferTo(out); - } finally { - in.close(); - } - }); - Subject.doAs(authSubject, (PrivilegedExceptionAction) () -> { - clientTransfer.runTransfer(); - return null; - }); - - } - - private Path downloadPackage(List targets, String expectedFilename, List expectedFilenames, - Subject testSubject, String contentType) +// private void uploadFile(VOSURI target, File uploadFile) +// throws PrivilegedActionException { +// List protocols = new ArrayList(); +// Protocol basicTLS = new Protocol(VOS.PROTOCOL_HTTPS_PUT); +// basicTLS.setSecurityMethod(Standards.SECURITY_METHOD_CERT); +// protocols.add(basicTLS); +//// DataNode targetNode = new DataNode(target.getName()); +// log.debug("uploading: " + target.getURI().toASCIIString()); +// Transfer transfer = new Transfer(target.getURI(), Direction.pushToVoSpace); +// transfer.getProtocols().addAll(protocols); +// transfer.version = VOS.VOSPACE_21; +// +// final VOSpaceClient voSpaceClient = new VOSpaceClient(resourceID); +// final ClientTransfer clientTransfer = Subject.doAs(authSubject, new CreateTransferAction(voSpaceClient, transfer, false)); +// clientTransfer.setOutputStreamWrapper(out -> { +// InputStream in = new FileInputStream(uploadFile); +// try { +// in.transferTo(out); +// } finally { +// in.close(); +// } +// }); +// Subject.doAs(authSubject, (PrivilegedExceptionAction) () -> { +// clientTransfer.runTransfer(); +// return null; +// }); +// +// } + + private File downloadPackage(List targets, String contentType, Subject testSubject) throws Exception { // Create a Transfer @@ -434,72 +504,124 @@ private Path downloadPackage(List targets, String expectedFilename, List %s", redirctURL, extractDirectory)); - - return Paths.get(extractDirectory); + Assert.assertTrue("package file has wrong extension", pkg.getName().endsWith(archiveType)); + log.debug("package file: " + pkg.getAbsolutePath()); + return pkg; } - private void extractArchive(InputStream inputStream, String extractDirectory, String archiveType) + private File extractPackage(File packageFile, String contentType) throws ArchiveException, IOException { + + FileInputStream inputStream = new FileInputStream(packageFile); + String archiveType = contentType.equals(ZIP_CONTENT_TYPE) ? ArchiveStreamFactory.ZIP : ArchiveStreamFactory.TAR; + log.debug("archive type: " + archiveType); + + String packageDir = packageFile.getName().replace("." + archiveType, ""); + File extractDir = new File(packageFile.getParent(), packageDir); + ArchiveStreamFactory archiveStreamFactory = new ArchiveStreamFactory(); ArchiveInputStream archiveInputStream = archiveStreamFactory.createArchiveInputStream( archiveType, inputStream); - ArchiveEntry archiveEntry; - while((archiveEntry = archiveInputStream.getNextEntry()) != null) { - Path path = Paths.get(extractDirectory, archiveEntry.getName()); - File file = path.toFile(); - if(archiveEntry.isDirectory()) { - if(!file.isDirectory()) { - file.mkdirs(); + ArchiveEntry entry; + while((entry = archiveInputStream.getNextEntry()) != null) { + if (!archiveInputStream.canReadEntryData(entry)) { + log.info("unable to read archive entry: " + entry.getName()); + continue; + } + + File file = new File(extractDir, entry.getName()); + log.debug("archive entry path:" + file.getAbsolutePath()); + + if (entry.isDirectory()) { + if (!file.isDirectory() && !file.mkdirs()) { + throw new IOException("failed to create entry directory " + file.getAbsolutePath()); } } else { File parent = file.getParentFile(); - if(!parent.isDirectory()) { - parent.mkdirs(); + if (!parent.isDirectory() && !parent.mkdirs()) { + log.debug("archive entry parent is not directory"); + throw new IOException("failed to create entry parent directory " + parent.getAbsolutePath()); } - try (OutputStream outputStream = Files.newOutputStream(path)) { + try (OutputStream outputStream = Files.newOutputStream(file.toPath())) { IOUtils.copy(archiveInputStream, outputStream); } } } - } - static class CreateTransferAction implements PrivilegedExceptionAction { - - VOSpaceClient voSpaceClient; - Transfer transfer; - boolean run; +// int index = archiveFile.getName().indexOf("."); +// String filename = archiveFile.getName().substring(0, index); +// return new File(archiveFile.getParent(), filename); + return extractDir; + } - CreateTransferAction(VOSpaceClient voSpaceClient, Transfer transfer, boolean run) { - this.voSpaceClient = voSpaceClient; - this.transfer = transfer; - this.run = run; + private void verifyPackage(List expectedFiles, String packageDir) throws IOException { + + // List of extracted files with path + List extractedFiles = listFiles(packageDir); + + // compare expected to extracted + int count = 0; + for (String expected : expectedFiles) { + log.debug("expected file: " + expected); + for (Path extracted : extractedFiles) { + log.debug("extracted file: " + extracted.getFileName()); + if (expected.equals(extracted.getFileName().toString())) { + log.debug("matched: " + expected); + count++; + break; + } + } } + Assert.assertEquals("", expectedFiles.size(), count); + } - @Override - public ClientTransfer run() throws Exception { - ClientTransfer clientTransfer = voSpaceClient.createTransfer(transfer); - if (run) { - clientTransfer.run(); - } - return clientTransfer; + private List listFiles(String rootDir) throws IOException { + try (Stream paths = Files.walk(Paths.get(rootDir))) { + return paths.filter(Files::isRegularFile).collect(Collectors.toList()); } + } + private void deleteFile(File file) { + boolean deleted = file.delete(); + log.debug(String.format("%s deleted: %s", file.getName(), deleted)); } +// static class CreateTransferAction implements PrivilegedExceptionAction { +// +// VOSpaceClient voSpaceClient; +// Transfer transfer; +// boolean run; +// +// CreateTransferAction(VOSpaceClient voSpaceClient, Transfer transfer, boolean run) { +// this.voSpaceClient = voSpaceClient; +// this.transfer = transfer; +// this.run = run; +// } +// +// @Override +// public ClientTransfer run() throws Exception { +// ClientTransfer clientTransfer = voSpaceClient.createTransfer(transfer); +// if (run) { +// clientTransfer.run(); +// } +// return clientTransfer; +// } +// +// } + } diff --git a/cadc-vos-server-alt/src/main/java/org/opencadc/vospace/server/pkg/VospacePackageRunner.java b/cadc-vos-server-alt/src/main/java/org/opencadc/vospace/server/pkg/VospacePackageRunner.java index 105282ae..103112fc 100644 --- a/cadc-vos-server-alt/src/main/java/org/opencadc/vospace/server/pkg/VospacePackageRunner.java +++ b/cadc-vos-server-alt/src/main/java/org/opencadc/vospace/server/pkg/VospacePackageRunner.java @@ -80,7 +80,9 @@ import java.net.URI; import java.net.URL; import java.security.AccessControlException; +import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import javax.naming.Context; @@ -148,14 +150,15 @@ protected void initPackage() throws IllegalArgumentException { // check job is valid log.debug("job id passed in: " + job.getID()); JobInfo jobInfo = job.getJobInfo(); - if (jobInfo != null) { - throw new RuntimeException("null jobInfo: " + job.toString()); - } // Get the target list from the job TransferReader tr = new TransferReader(); this.packageTransfer = tr.read(jobInfo.getContent(), VOSURI.SCHEME); List targetList = packageTransfer.getTargets(); + log.debug("packageTransfer protocols: " + packageTransfer.getProtocols().size()); + for (Protocol p : packageTransfer.getProtocols()) { + log.debug("packageTransfer protocols: " + p.getUri()); + } if (targetList.size() > 1) { this.packageName = "cadc-download-" + job.getID(); @@ -180,15 +183,12 @@ protected Iterator getItems() throws IOException { List targetList = packageTransfer.getTargets(); log.debug("target list: " + targetList.toString()); - Subject caller = AuthenticationUtil.getCurrentSubject(); - log.debug("subject: " + caller); - List packageItems = new ArrayList<>(); for (URI targetURI : targetList) { VOSURI nodeURI = new VOSURI(targetURI); log.debug("node: " + nodeURI); - addPackageItem(nodeURI, packageItems, caller); + addPackageItem(nodeURI, packageItems); } return packageItems.iterator(); } @@ -197,23 +197,37 @@ protected Iterator getItems() throws IOException { * * @param nodeURI * @param packageItems - * @param caller */ - private void addPackageItem(VOSURI nodeURI, List packageItems, Subject caller) { + private void addPackageItem(VOSURI nodeURI, List packageItems) { String nodePath = nodeURI.getPath(); log.debug("node path: " + nodePath); +// Subject caller = AuthenticationUtil.getCurrentSubject(); + log.debug("calling subject: " + AuthenticationUtil.getCurrentSubject()); + try { - // PathResolver checks read permission for the node path - Node node = pathResolver.getNode(nodePath); + // PathResolver checks read permission for the root node path + Node node; + try { + node = pathResolver.getNode(nodePath); + } catch (AccessControlException e) { + // add empty container to package + log.debug("read permission denied to root node: " + nodePath); +// packageItems.add(new PackageItem(endpoint, nodePath)); + log.debug("??? add empty root node to packages: " + nodePath); + return; + } + if (node == null) { throw NodeFault.NodeNotFound.getStatus(nodePath); } if (node instanceof DataNode) { - addDataNode((DataNode) node, packageItems); + log.debug("data node, add to packages: " + nodePath); + packageItems.add(getPackageItem(node)); } else if (node instanceof ContainerNode) { - addContainerNode((ContainerNode) node, packageItems, caller); + log.debug("container node, add to packages: " + nodePath); + addContainerNode((ContainerNode) node, packageItems); } else { log.info("unrecognized or unsupported node type " + nodeURI + CONTINUING_PROCESSING); } @@ -227,65 +241,58 @@ private void addPackageItem(VOSURI nodeURI, List packageItems, Subj log.info("linking exception: " + nodeURI + CONTINUING_PROCESSING); } catch (MalformedURLException e) { log.info("malformed URL: " + nodeURI + CONTINUING_PROCESSING); + e.printStackTrace(); } catch (TransientException e) { log.info("transientException: " + nodeURI + CONTINUING_PROCESSING); } catch (Exception e) { log.info(String.format("%s: %s%s", e.getClass().getName(), nodeURI, CONTINUING_PROCESSING)); + e.printStackTrace(); } } - private void addDataNode(DataNode node, List packageItems) + private void addContainerNode(ContainerNode node, List packageItems) throws Exception { - String nodePath = Utils.getPath(node); - log.debug("DataNode path: " + nodePath); - - VOSURI nodeURI = new VOSURI(this.resourceID, nodePath); - log.debug("DataNode uri: " + nodeURI); - - // Use a temporary Job with just the remote IP to avoid the original Job in - // a transfer URL which may close the Job. - Job pkgJob = new Job(); - pkgJob.setRemoteIP(this.job.getRemoteIP()); - TransferGenerator transferGenerator = nodePersistence.getTransferGenerator(); - List protocols = transferGenerator.getEndpoints(nodeURI, packageTransfer, pkgJob, null); - - // Get the node endpoint from the first protocol - Protocol protocol = protocols.get(0); - String endpoint = protocol.getEndpoint(); - log.debug("DataNode endpoint:" + endpoint); - packageItems.add(new PackageItem(new URL(endpoint), nodePath)); - } - - private void addContainerNode(ContainerNode node, List packageItems, Subject caller) - throws Exception { + // check the job phase for abort or illegal state +// checkJobPhase(); + log.debug("process container node: " + node); List childContainers = new ArrayList<>(); + Subject caller = AuthenticationUtil.getCurrentSubject(); try (ResourceIterator iterator = nodePersistence.iterator(node, null, null)) { while (iterator.hasNext()) { Node child = iterator.next(); + log.debug("process child node: " + child); boolean canRead = vospaceAuthorizer.hasSingleNodeReadPermission(child, caller); + if (child instanceof ContainerNode) { if (canRead) { + log.debug("add child container node to queue"); childContainers.add((ContainerNode) child); } else { // add empty container node to the package - packageItems.add(getPackageItem(node)); +// PackageItem packageItem = getPackageItem(node); +// log.debug("add child container node packageItem: " + packageItem.getRelativePath()); +// packageItems.add(packageItem); } } else if (child instanceof DataNode) { if (canRead) { - addDataNode((DataNode) child, packageItems); + PackageItem packageItem = getPackageItem(child); + log.debug("add child data node packageItem: " + packageItem.getRelativePath()); + packageItems.add(packageItem); } } else { - log.info("unrecognized or unsupported node type " + Utils.getPath(node) + CONTINUING_PROCESSING); + log.info("unrecognized or unsupported node type " + Utils.getPath(child) + CONTINUING_PROCESSING); } } } if (!childContainers.isEmpty()) { + log.debug("process container nodes"); for (ContainerNode child : childContainers) { - addContainerNode(child, packageItems, caller); + log.debug("process: " + child); + addContainerNode(child, packageItems); } } } @@ -303,23 +310,34 @@ private static String getFilenameFromURI(URI uri) { return path; } - private PackageItem getPackageItem(Node node) throws Exception { + private URL getEndpointURL(VOSURI nodeURI) + throws Exception { + + String remoteIP = this.job.getRemoteIP(); + return Subject.doAs(AuthenticationUtil.getCurrentSubject(), (PrivilegedExceptionAction) () -> { + // Use a temporary Job with just the remote IP to avoid the original Job in + // a transfer URL which may close the Job. + Job pkgJob = new Job(); + pkgJob.setRemoteIP(remoteIP); + TransferGenerator transferGenerator = nodePersistence.getTransferGenerator(); + List protocols = transferGenerator.getEndpoints(nodeURI, packageTransfer, pkgJob, null); + log.debug("num transfer protocols: " + protocols.size()); + + // Get the node endpoint from the first protocol + Protocol protocol = protocols.get(0); + String endpoint = protocol.getEndpoint(); + return new URL(endpoint); + }); + } + + private PackageItem getPackageItem(Node node) + throws Exception { + log.debug("getPackageItem node type is container?: " + (node instanceof ContainerNode)); String nodePath = Utils.getPath(node); VOSURI nodeURI = new VOSURI(this.resourceID, nodePath); log.debug("Node uri: " + nodeURI); - // Use a temporary Job with just the remote IP to avoid the original Job in - // a transfer URL which may close the Job. - // Use a temporary Job with just the remote IP to avoid the original Job in - // a transfer URL which may close the Job. - Job pkgJob = new Job(); - pkgJob.setRemoteIP(this.job.getRemoteIP()); - TransferGenerator transferGenerator = nodePersistence.getTransferGenerator(); - List protocols = transferGenerator.getEndpoints(nodeURI, packageTransfer, pkgJob, null); - - // Get the node endpoint from the first protocol - Protocol protocol = protocols.get(0); - URL endpoint = new URL(protocol.getEndpoint()); + URL endpoint = getEndpointURL(nodeURI); log.debug("Node endpoint:" + endpoint); return new PackageItem(endpoint, nodePath); } diff --git a/cavern/build.gradle b/cavern/build.gradle index 168fc417..8373d71a 100644 --- a/cavern/build.gradle +++ b/cavern/build.gradle @@ -52,6 +52,6 @@ dependencies { testImplementation 'org.opencadc:cadc-test-uws:[1.0,)' testImplementation 'org.opencadc:cadc-access-control-identity:[1.1.0,)' - intTestImplementation 'org.opencadc:cadc-test-vos:[2.0,)' + intTestImplementation 'org.opencadc:cadc-test-vos:[2.1,)' intTestImplementation 'org.opencadc:cadc-test-vosi:[1.0.11,)' } diff --git a/cavern/src/main/java/org/opencadc/cavern/files/CavernURLGenerator.java b/cavern/src/main/java/org/opencadc/cavern/files/CavernURLGenerator.java index f089f1e2..8d6e80d5 100644 --- a/cavern/src/main/java/org/opencadc/cavern/files/CavernURLGenerator.java +++ b/cavern/src/main/java/org/opencadc/cavern/files/CavernURLGenerator.java @@ -267,6 +267,7 @@ private List handleContainerMount(String path, Transfer trans, Subject PosixPrincipal pp = nodePersistence.getIdentityManager().toPosixPrincipal(caller); List ret = new ArrayList<>(); for (Protocol p : trans.getProtocols()) { + log.debug("transfer protocol: " + p.getUri()); if (VOS.PROTOCOL_SSHFS.equals(p.getUri())) { if (sshServerBase == null) { throw new UnsupportedOperationException("sshfs mount not configured"); diff --git a/cavern/src/main/webapp/WEB-INF/web.xml b/cavern/src/main/webapp/WEB-INF/web.xml index f373f614..d2acb1a5 100644 --- a/cavern/src/main/webapp/WEB-INF/web.xml +++ b/cavern/src/main/webapp/WEB-INF/web.xml @@ -20,7 +20,7 @@ ca.nrc.cadc.log.LogControlServlet logLevel - info + debug logLevelPackages @@ -30,6 +30,11 @@ ca.nrc.cadc.rest ca.nrc.cadc.vosi ca.nrc.cadc.auth + + org.opencadc.cavern.files + org.opencadc.pkg.server + org.opencadc.vospace.server.pkg + org.opencadc.conformance.vos From aa8212e74f882cc1a957e79ad86cb6b4ef78cadb Mon Sep 17 00:00:00 2001 From: Jeff Burke Date: Tue, 7 Nov 2023 17:03:38 -0800 Subject: [PATCH 06/21] CADC-12563 pkg int-test updates --- cadc-test-vos/build.gradle | 3 +- .../opencadc/conformance/vos/PackageTest.java | 471 ++++++++++-------- cadc-vos-server-alt/build.gradle | 40 -- cadc-vos-server/build.gradle | 3 +- .../server/pkg/VospacePackageRunner.java | 0 cavern/build.gradle | 2 +- cavern/src/main/webapp/capabilities.xml | 2 - 7 files changed, 265 insertions(+), 256 deletions(-) delete mode 100644 cadc-vos-server-alt/build.gradle rename {cadc-vos-server-alt => cadc-vos-server}/src/main/java/org/opencadc/vospace/server/pkg/VospacePackageRunner.java (100%) diff --git a/cadc-test-vos/build.gradle b/cadc-test-vos/build.gradle index 3d319cb1..65a022d1 100644 --- a/cadc-test-vos/build.gradle +++ b/cadc-test-vos/build.gradle @@ -16,7 +16,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '2.0.0' +version = '2.0.1' description = 'OpenCADC VOSpace test library' def git_url = 'https://github.com/opencadc/vos' @@ -29,4 +29,5 @@ dependencies { implementation 'org.opencadc:cadc-registry:[1.7.4,2.0)' implementation 'junit:junit:[4.0,)' + implementation 'org.apache.commons:commons-compress:[1.12,)' } diff --git a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/PackageTest.java b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/PackageTest.java index 020d1f2d..dcec4e17 100644 --- a/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/PackageTest.java +++ b/cadc-test-vos/src/main/java/org/opencadc/conformance/vos/PackageTest.java @@ -71,33 +71,25 @@ import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.RunnableAction; -import ca.nrc.cadc.io.ByteCountOutputStream; import ca.nrc.cadc.net.FileContent; import ca.nrc.cadc.net.HttpDownload; import ca.nrc.cadc.net.HttpGet; import ca.nrc.cadc.net.HttpPost; -import ca.nrc.cadc.net.OutputStreamWrapper; import ca.nrc.cadc.reg.Standards; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.io.StringWriter; -import java.net.MalformedURLException; import java.net.URI; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.security.AccessControlException; -import java.security.PrivilegedActionException; -import java.security.PrivilegedExceptionAction; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -112,12 +104,10 @@ import org.junit.Assert; import org.junit.Test; import org.opencadc.vospace.ContainerNode; -import org.opencadc.vospace.DataNode; +import org.opencadc.vospace.LinkNode; import org.opencadc.vospace.VOS; import org.opencadc.vospace.VOSURI; import org.opencadc.vospace.View; -import org.opencadc.vospace.client.ClientTransfer; -import org.opencadc.vospace.client.VOSpaceClient; import org.opencadc.vospace.transfer.Direction; import org.opencadc.vospace.transfer.Protocol; import org.opencadc.vospace.transfer.Transfer; @@ -130,68 +120,76 @@ public class PackageTest extends VOSTest { private static final String ZIP_CONTENT_TYPE = "application/zip"; private static final String TAR_CONTENT_TYPE = "application/x-tar"; -// private static Subject s; - -// private static VOSURI nodeURI; protected PackageTest(URI resourceID, File testCert) { super(resourceID, testCert); } - @Test - public void tarPermissionDeniedTest () { - try { - permissionDeniedTest(TAR_CONTENT_TYPE); - } catch (Exception e) { - log.error("Unexpected error", e); - Assert.fail("Unexpected error: " + e); - } - } + /** + * returns empty archive file, should return archive with empty directory? + */ @Test - public void zipPermissionDeniedTest () { + public void permissionDeniedTest() { try { - permissionDeniedTest(ZIP_CONTENT_TYPE); - } catch (Exception e) { - log.error("Unexpected error", e); - Assert.fail("Unexpected error: " + e); - } - } + // Root container node + String root = "permission-denied-root"; + String file = "permission-denied-file.txt"; - @Test - public void tarNodeNotFoundTest () { - try { - targetNotFoundTest(TAR_CONTENT_TYPE); - } catch (Exception e) { - log.error("Unexpected error", e); - Assert.fail("Unexpected error: " + e); - } - } + URL rootURL = getNodeURL(nodesServiceURL, root); + URL fileURL = getNodeURL(nodesServiceURL, file); - @Test - public void zipNodeNotFoundTest () { - try { - targetNotFoundTest(ZIP_CONTENT_TYPE); - } catch (Exception e) { - log.error("Unexpected error", e); - Assert.fail("Unexpected error: " + e); - } - } + // cleanup + delete(fileURL, false); + delete(rootURL, false); + + // upload the folders and files as auth subject + String content = "permission-denied-file-content"; + VOSURI nodeURI = putContainerNode(root, rootURL); + VOSURI fileURI = putDataNode(file, content); + + // package targets to download + List targets = new ArrayList<>(); + targets.add(nodeURI.getURI()); + + List expected = new ArrayList<>(); + + Subject anonSubject = AuthenticationUtil.getAnonSubject(); + File archive = downloadPackage(targets, TAR_CONTENT_TYPE, anonSubject); + File extracted = extractPackage(archive, TAR_CONTENT_TYPE); + verifyPackage(expected, extracted); + + // cleanup + delete(fileURL, false); + delete(rootURL, false); - @Test - public void tarEmptyNodeTest() { - try { - emptyTargetTest(TAR_CONTENT_TYPE); } catch (Exception e) { log.error("Unexpected error", e); Assert.fail("Unexpected error: " + e); } } + /** + * returns empty archive file, should throw 404 instead? + */ @Test - public void zipEmptyNodeTest() { + public void targetNotFoundTest() { try { - emptyTargetTest(ZIP_CONTENT_TYPE); + String name = "target-not-found-node"; + VOSURI nodeURI = getVOSURI(name); + + // package targets to download + List targets = new ArrayList<>(); + targets.add(nodeURI.getURI()); + + // download and extract the package + try { + File archive = downloadPackage(targets, TAR_CONTENT_TYPE, authSubject); + Assert.fail("should have thrown exception for target not found"); + } catch (AccessControlException e) { + log.debug("expected exception: " + e.getMessage()); + } + } catch (Exception e) { log.error("Unexpected error", e); Assert.fail("Unexpected error: " + e); @@ -199,9 +197,30 @@ public void zipEmptyNodeTest() { } @Test - public void tarSingleNodeTest() { + public void emptyTargetTest() { try { - singleTargetTest(TAR_CONTENT_TYPE); + String name = "empty-target-node"; + URL nodeURL = getNodeURL(nodesServiceURL, name); + + // cleanup + delete(nodeURL, false); + + // upload the folders and files + VOSURI nodeURI = putContainerNode(name, nodeURL); + + // package targets to download + List targets = new ArrayList<>(); + targets.add(nodeURI.getURI()); + + // expected files in download + List expected = new ArrayList<>(); + + doTest(targets, expected, TAR_CONTENT_TYPE); + doTest(targets, expected, ZIP_CONTENT_TYPE); + + // cleanup + delete(nodeURL, false); + } catch (Exception e) { log.error("Unexpected error", e); Assert.fail("Unexpected error: " + e); @@ -209,9 +228,32 @@ public void tarSingleNodeTest() { } @Test - public void zipSingleNodeTest() { + public void singleFileTargetTest() { try { - singleTargetTest(ZIP_CONTENT_TYPE); + String name = "single-target-node.txt"; + String content = "single-target-node-content"; + URL nodeURL = getNodeURL(nodesServiceURL, name); + + // cleanup + delete(nodeURL, false); + + // upload the folders and files + VOSURI nodeURI = putDataNode(name, content); + + // package targets to download + List targets = new ArrayList<>(); + targets.add(nodeURI.getURI()); + + // expected files in download + List expected = new ArrayList<>(); + expected.add(name); + + doTest(targets, expected, TAR_CONTENT_TYPE); + doTest(targets, expected, ZIP_CONTENT_TYPE); + + // cleanup + delete(nodeURL, false); + } catch (Exception e) { log.error("Unexpected error", e); Assert.fail("Unexpected error: " + e); @@ -219,9 +261,57 @@ public void zipSingleNodeTest() { } @Test - public void tarParentChildTest() { + public void multipleTargetTest() { try { - parentChildTest(TAR_CONTENT_TYPE); + // /root/ + // /root/target-1/ + // /root/target-1/file1 + // /root/target-2/ + // /root/target-2/file2 + + String root = "multi-target-root-node"; + String target1 = root + "/target-1"; + String file1 = target1 + "/target-1-file.txt"; + String target2 = root + "/target-2"; + String file2 = target2 + "/target-2-file.txt"; + + URL rootURL = getNodeURL(nodesServiceURL, root); + URL target1URL = getNodeURL(nodesServiceURL, target1); + URL file1URL = getNodeURL(nodesServiceURL, file1); + URL target2URL = getNodeURL(nodesServiceURL, target2); + URL file2URL = getNodeURL(nodesServiceURL, file2); + + URL[] nodes = new URL[] {file2URL, target2URL, file1URL, target1URL, rootURL}; + + // cleanup + delete(nodes); + + // upload the folders and files + String content1 = "target-1-file-content"; + String content2 = "target-2-file-content"; + + VOSURI rootURI = putContainerNode(root, rootURL); + VOSURI target1URI = putContainerNode(target1, target1URL); + VOSURI file1URI = putDataNode(file1, content1, authSubject); + VOSURI target2URI = putContainerNode(target2, target2URL); + VOSURI file2URI = putDataNode(file2, content2, authSubject); + + // package targets to download + List targets = new ArrayList<>(); + targets.add(target1URI.getURI()); + targets.add(target2URI.getURI()); + + // expected files in download + List expected = new ArrayList<>(); + expected.add(file1); + expected.add(file2); + + doTest(targets, expected, TAR_CONTENT_TYPE); + doTest(targets, expected, ZIP_CONTENT_TYPE); + + // cleanup + delete(nodes); + } catch (Exception e) { log.error("Unexpected error", e); Assert.fail("Unexpected error: " + e); @@ -229,8 +319,72 @@ public void tarParentChildTest() { } @Test - public void multipleTargetTest() { + public void fullTest() { try { + // /root-folder/ + // /root-folder/file-1.txt + // /root-folder/folder-1/ + // /root-folder/folder-2/ + // /root-folder/folder-2/file-2.txt + // /root-folder/folder-2/file-3.txt + // /root-folder/folder-2/folder-3/ + // /root-folder/folder-2/folder-3/link-1.txt + + // nodes paths + String root = "full-root-folder"; + String file1 = root + "/file-1.txt"; + String folder1 = root + "/folder-1"; + String folder2 = root + "/folder-2"; + String file2 = folder2 + "/file-2.txt"; + String file3 = folder2 + "/file-3.txt"; + String folder3 = folder2 + "/folder-3"; + String link1 = folder3 + "/link-1.txt"; + + // node URL's + URL rootURL = getNodeURL(nodesServiceURL, root); + URL file1URL = getNodeURL(nodesServiceURL, file1); + URL folder1URL = getNodeURL(nodesServiceURL, folder1); + URL folder2URL = getNodeURL(nodesServiceURL, folder2); + URL file2URL = getNodeURL(nodesServiceURL, file2); + URL file3URL = getNodeURL(nodesServiceURL, file3); + URL folder3URL = getNodeURL(nodesServiceURL, folder3); + URL link1URL = getNodeURL(nodesServiceURL, link1); + + URL[] nodes = new URL[] {link1URL, folder3URL, file3URL, file2URL, folder2URL, folder1URL, file1URL, rootURL}; + + // cleanup + delete(nodes); + + // upload the folders and files + String file1Content = "file-1-content"; + String file2Content = "file-2-content"; + String file3Content = "file-3-content"; + URI linkTarget = URI.create("vos://opencadc.org~cavern/link-node"); + + VOSURI rootURI = putContainerNode(root, rootURL); + VOSURI file1URI = putDataNode(file1, file1Content); + VOSURI folder1URI = putContainerNode(folder1, folder1URL); + VOSURI folder2URI = putContainerNode(folder2, folder2URL); + VOSURI file2URI = putDataNode(file2, file2Content); + VOSURI file3URI = putDataNode(file3, file3Content); + VOSURI folder3URI = putContainerNode(folder3, folder3URL); + VOSURI link1URI = putLinkNode(link1, link1URL, linkTarget); + + // package targets to download + List targets = new ArrayList<>(); + targets.add(rootURI.getURI()); + + // expected files in download + List expected = new ArrayList<>(); + expected.add(file1); + expected.add(file2); + expected.add(file3); + + doTest(targets, expected, TAR_CONTENT_TYPE); + doTest(targets, expected, ZIP_CONTENT_TYPE); + + // cleanup + delete(nodes); } catch (Exception e) { log.error("Unexpected error", e); @@ -238,168 +392,56 @@ public void multipleTargetTest() { } } - - public void permissionDeniedTest(String contentType) throws Exception { - // Root container node created by auth subject - String name = "permission-denied-node"; - URL nodeURL = getNodeURL(nodesServiceURL, name); - - // cleanup - delete(nodeURL, false); - - VOSURI nodeURI = getVOSURI(name); - ContainerNode node = new ContainerNode(name); - put(nodeURL, nodeURI, node); - log.debug(String.format("PUT %s URL: %s", name, nodeURL)); - - List targets = new ArrayList<>(); - targets.add(nodeURI.getURI()); - - // Package request using anon subject - Subject anonSubject = AuthenticationUtil.getAnonSubject(); - - // download and extract the package - try { - File archive = downloadPackage(targets, contentType, authSubject); - Assert.fail("should have thrown exception for permission denied"); - } catch (AccessControlException e) { - log.debug("expected exception: " + e.getMessage()); - } - - // cleanup - delete(nodeURL, false); + private VOSURI putContainerNode(String path, URL url) throws IOException { + ContainerNode node = new ContainerNode(path); + VOSURI uri = getVOSURI(path); + put(url, uri, node); + return uri; } - public void targetNotFoundTest(String contentType) throws Exception { - String name = "target-not-found-node"; - VOSURI nodeURI = getVOSURI(name); - - List targets = new ArrayList<>(); - targets.add(nodeURI.getURI()); - - // download and extract the package - try { - File archive = downloadPackage(targets, contentType, authSubject); - Assert.fail("should have thrown exception for target not found"); - } catch (AccessControlException e) { - log.debug("expected exception: " + e.getMessage()); - } + private VOSURI putDataNode(String path, String content) + throws IOException, TransferParsingException { + return uploadFile(path, content, authSubject); } - - public void emptyTargetTest(String contentType) throws Exception { - - String name = "empty-target-node"; - URL nodeURL = getNodeURL(nodesServiceURL, name); - - // cleanup - delete(nodeURL, false); - - VOSURI nodeURI = getVOSURI(name); - ContainerNode node = new ContainerNode(name); - put(nodeURL, nodeURI, node); - log.debug(String.format("PUT %s URL: %s", name, nodeURL)); - - List targets = new ArrayList<>(); - targets.add(nodeURI.getURI()); - - // download and extract the package - File archive = downloadPackage(targets, contentType, authSubject); - -// File[] files = packageRoot.toFile().listFiles(); -// Assert.assertNotNull(files); -// Assert.assertEquals("expected single file", 1, files.length); -// Assert.assertTrue("expected directory", files[0].isDirectory()); -// Assert.assertEquals("", name, files[0].getName()); - - // cleanup - delete(nodeURL, false); + private VOSURI putDataNode(String path, String content, Subject testSubject) + throws IOException, TransferParsingException { + return uploadFile(path, content, testSubject); } - public void singleTargetTest(String contentType) throws Exception { - - String name = "single-target-node.txt"; - String content = "single-target-node.txt content"; - URL nodeURL = getNodeURL(nodesServiceURL, name); - VOSURI nodeURI = getVOSURI(name); - - // cleanup -// delete(nodeURL, false); - - // Upload -// uploadFile(name, content, authSubject); - - List targets = new ArrayList<>(); - targets.add(nodeURI.getURI()); - - // download and extract the package - File archive = downloadPackage(targets, contentType, authSubject); - -// File[] files = packageRoot.toFile().listFiles(); -// Assert.assertNotNull(files); -// Assert.assertEquals("expected single file", 1, files.length); -// Assert.assertTrue("expected directory", files[0].isDirectory()); -// Assert.assertEquals("", name, files[0].getName()); - - // cleanup -// delete(nodeURL, false); + private VOSURI putLinkNode(String path, URL url, URI target) + throws IOException { + LinkNode node = new LinkNode(path, target); + VOSURI uri = getVOSURI(path); + put(url, uri, node); + return uri; } - public void parentChildTest(String contentType) throws Exception { - - // parent ContainerNode - String parent = "foo-parent-node"; - URL parentNodeURL = getNodeURL(nodesServiceURL, parent); - - // child DataNode - String child = "foo-child-node.txt"; - URL childNodeURL = getNodeURL(nodesServiceURL, parent + "/" + child); - - // cleanup -// delete(childNodeURL, false); -// delete(parentNodeURL, false); - - // PUT parent node - VOSURI parentNodeURI = getVOSURI(parent); - ContainerNode node = new ContainerNode(parent); -// put(parentNodeURL, parentNodeURI, node); - log.debug(String.format("PUT %s URL: %s", parent, parentNodeURL)); - - // Upload child node - String content = "foo-child-node content"; -// uploadFile(parent + "/" + child, content, authSubject); + private void doTest(List targets, List expected, String contentType) + throws Exception { // download the package - List targets = new ArrayList<>(); - targets.add(parentNodeURI.getURI()); File pkg = downloadPackage(targets, contentType, authSubject); Assert.assertNotNull(pkg); log.debug("archive file: " + pkg.getAbsolutePath()); - // extract the archive + // extract the package File extracted = extractPackage(pkg, contentType); Assert.assertNotNull(extracted); log.debug("extracted file: " + extracted.getAbsolutePath()); // verify package files - List expected = new ArrayList<>(); - expected.add(child); - verifyPackage(expected, extracted.getAbsolutePath()); - - // cleanup - delete(childNodeURL, false); - delete(parentNodeURL, false); - deleteFile(pkg); - deleteFile(extracted); + verifyPackage(expected, extracted); + + // file cleanup +// deleteFile(pkg); +// deleteFile(extracted); } - private void uploadFile(String filename, String content, Subject caller) - throws MalformedURLException, IOException, TransferParsingException { - // Put a file (DataNode) - URL nodeURL = getNodeURL(nodesServiceURL, filename); - VOSURI nodeURI = getVOSURI(filename); - log.debug("upload node URL: " + nodeURL); + private VOSURI uploadFile(String filename, String content, Subject testSubject) + throws IOException, TransferParsingException { // Create a Transfer + VOSURI nodeURI = getVOSURI(filename); Transfer transfer = new Transfer(nodeURI.getURI(), Direction.pushToVoSpace); transfer.version = VOS.VOSPACE_21; Protocol protocol = new Protocol(VOS.PROTOCOL_HTTPS_PUT); @@ -417,7 +459,7 @@ private void uploadFile(String filename, String content, Subject caller) URL transferURL = getNodeURL(synctransServiceURL, filename); log.debug("transfer URL: " + transferURL); HttpPost post = new HttpPost(synctransServiceURL, fileContent, false); - Subject.doAs(caller, new RunnableAction(post)); + Subject.doAs(testSubject, new RunnableAction(post)); Assert.assertEquals("expected POST response code = 303", 303, post.getResponseCode()); Assert.assertNull("expected PUT throwable == null", post.getThrowable()); @@ -425,7 +467,7 @@ private void uploadFile(String filename, String content, Subject caller) ByteArrayOutputStream out = new ByteArrayOutputStream(); HttpGet get = new HttpGet(post.getRedirectURL(), out); log.debug("GET: " + post.getRedirectURL()); - Subject.doAs(caller, new RunnableAction(get)); + Subject.doAs(testSubject, new RunnableAction(get)); log.debug("GET responseCode: " + get.getResponseCode()); Assert.assertEquals("expected GET response code = 200", 200, get.getResponseCode()); Assert.assertNull("expected GET throwable == null", get.getThrowable()); @@ -445,6 +487,8 @@ private void uploadFile(String filename, String content, Subject caller) log.info("PUT: " + endpoint); ByteArrayInputStream is = new ByteArrayInputStream(content.getBytes()); put(endpoint, is, VOSTest.TEXT_CONTENT_TYPE); + + return nodeURI; } // private void uploadFile(VOSURI target, File uploadFile) @@ -504,7 +548,10 @@ private File downloadPackage(List targets, String contentType, Subject test URL redirctURL = post.getRedirectURL(); // Download the package - File tmp = new File(System.getProperty("java.io.tmpdir")); + File tmp = new File(System.getProperty("java.io.tmpdir"), "package-test-" + UUID.randomUUID()); + if (!tmp.mkdirs()) { + throw new IOException("unable to create tmp directory: " + tmp.getAbsolutePath()); + } HttpDownload download = new HttpDownload(redirctURL, tmp); download.setOverwrite(true); log.debug("GET: " + redirctURL); @@ -531,7 +578,7 @@ private File extractPackage(File packageFile, String contentType) String archiveType = contentType.equals(ZIP_CONTENT_TYPE) ? ArchiveStreamFactory.ZIP : ArchiveStreamFactory.TAR; log.debug("archive type: " + archiveType); - String packageDir = packageFile.getName().replace("." + archiveType, ""); + String packageDir = packageFile.getName().replace(".", "-"); File extractDir = new File(packageFile.getParent(), packageDir); ArchiveStreamFactory archiveStreamFactory = new ArchiveStreamFactory(); @@ -562,17 +609,13 @@ private File extractPackage(File packageFile, String contentType) } } } - -// int index = archiveFile.getName().indexOf("."); -// String filename = archiveFile.getName().substring(0, index); -// return new File(archiveFile.getParent(), filename); return extractDir; } - private void verifyPackage(List expectedFiles, String packageDir) throws IOException { + private void verifyPackage(List expectedFiles, File packageDir) throws IOException { // List of extracted files with path - List extractedFiles = listFiles(packageDir); + List extractedFiles = listFiles(packageDir.getAbsolutePath()); // compare expected to extracted int count = 0; @@ -596,6 +639,12 @@ private List listFiles(String rootDir) throws IOException { } } + private void delete(URL[] nodes) { + for (URL node : nodes) { + delete(node, false); + } + } + private void deleteFile(File file) { boolean deleted = file.delete(); log.debug(String.format("%s deleted: %s", file.getName(), deleted)); diff --git a/cadc-vos-server-alt/build.gradle b/cadc-vos-server-alt/build.gradle deleted file mode 100644 index 0e37e043..00000000 --- a/cadc-vos-server-alt/build.gradle +++ /dev/null @@ -1,40 +0,0 @@ -plugins { - id 'java' - id 'maven' - id 'maven-publish' - id 'checkstyle' -} - -repositories { - mavenCentral() - mavenLocal() -} - -apply from: '../opencadc.gradle' - -sourceCompatibility = 1.8 - -group = 'org.opencadc' - -version = '2.0' - -description = 'OpenCADC VOSpace server' -def git_url = 'https://github.com/opencadc/vos' - -dependencies { - compile 'javax.servlet:javax.servlet-api:[3.1,4.0)' - - compile 'org.opencadc:cadc-util:[1.9.5,2.0)' - compile 'org.opencadc:cadc-gms:[1.0.0,)' - compile 'org.opencadc:cadc-rest:[1.3.16,)' - compile 'org.opencadc:cadc-vos:[2.0,)' - compile 'org.opencadc:cadc-vosi:[1.3.2,)' - compile 'org.opencadc:cadc-uws:[1.0,)' - compile 'org.opencadc:cadc-uws-server:[1.2.19,)' - compile 'org.opencadc:cadc-cdp:[1.2.3,)' - compile 'org.opencadc:cadc-registry:[1.7,2.0)' - compile 'org.opencadc:cadc-gms:[1.0.5,2.0)' - compile 'org.opencadc:cadc-pkg-server:[1.0,)' - - testCompile 'junit:junit:4.13' -} diff --git a/cadc-vos-server/build.gradle b/cadc-vos-server/build.gradle index 37ce9ab3..0564d899 100644 --- a/cadc-vos-server/build.gradle +++ b/cadc-vos-server/build.gradle @@ -16,7 +16,7 @@ sourceCompatibility = 1.8 group = 'org.opencadc' -version = '2.0.0' +version = '2.0.1' description = 'OpenCADC VOSpace server' def git_url = 'https://github.com/opencadc/vos' @@ -34,6 +34,7 @@ dependencies { compile 'org.opencadc:cadc-cdp:[1.2.3,)' compile 'org.opencadc:cadc-registry:[1.7,2.0)' compile 'org.opencadc:cadc-gms:[1.0.5,2.0)' + compile 'org.opencadc:cadc-pkg-server:[1.0,)' testCompile 'junit:junit:4.13' } diff --git a/cadc-vos-server-alt/src/main/java/org/opencadc/vospace/server/pkg/VospacePackageRunner.java b/cadc-vos-server/src/main/java/org/opencadc/vospace/server/pkg/VospacePackageRunner.java similarity index 100% rename from cadc-vos-server-alt/src/main/java/org/opencadc/vospace/server/pkg/VospacePackageRunner.java rename to cadc-vos-server/src/main/java/org/opencadc/vospace/server/pkg/VospacePackageRunner.java diff --git a/cavern/build.gradle b/cavern/build.gradle index a354617b..bd5f0a54 100644 --- a/cavern/build.gradle +++ b/cavern/build.gradle @@ -52,6 +52,6 @@ dependencies { testImplementation 'org.opencadc:cadc-test-uws:[1.0,)' testImplementation 'org.opencadc:cadc-access-control-identity:[1.1.0,)' - intTestImplementation 'org.opencadc:cadc-test-vos:[2.1,)' + intTestImplementation 'org.opencadc:cadc-test-vos:[2.0,)' intTestImplementation 'org.opencadc:cadc-test-vosi:[1.0.11,)' } diff --git a/cavern/src/main/webapp/capabilities.xml b/cavern/src/main/webapp/capabilities.xml index 88a83ea3..e591d6b3 100644 --- a/cavern/src/main/webapp/capabilities.xml +++ b/cavern/src/main/webapp/capabilities.xml @@ -60,7 +60,6 @@ -