diff --git a/doc/tpc.md b/doc/tpc.md index d93e0274..ce37378d 100644 --- a/doc/tpc.md +++ b/doc/tpc.md @@ -56,3 +56,16 @@ STORM_WEBDAV_AUTHZ_SERVER_MAX_TOKEN_LIFETIME_SEC="43200" STORM_WEBDAV_REQUIRE_CLIENT_CERT="false" ``` For other configuration options, see the /etc/sysconfig/storm-webdav file. + +## SciTags + +StoRM WebDAV supports the `SciTag` header. +To correctly mark the network packets and/or network flows, you need to install [flowd](https://github.com/scitags/flowd) and configure it to use the `np_api` plugin. + +Example flowd configuration (`/etc/flowd/flowd-tags.cfg`): + +``` +PLUGIN='np_api' +BACKEND='udp_firefly' +FLOW_MAP_API='https://www.scitags.org/api.json' +``` diff --git a/src/main/java/org/italiangrid/storm/webdav/scitag/SciTag.java b/src/main/java/org/italiangrid/storm/webdav/scitag/SciTag.java new file mode 100644 index 00000000..1d39d2d1 --- /dev/null +++ b/src/main/java/org/italiangrid/storm/webdav/scitag/SciTag.java @@ -0,0 +1,49 @@ +/** + * Copyright (c) Istituto Nazionale di Fisica Nucleare, 2014-2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.italiangrid.storm.webdav.scitag; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SciTag { + + public static final Logger LOG = LoggerFactory.getLogger(SciTag.class); + public static final String SCITAG_HEADER = "SciTag"; + public static final String SCITAG_ATTRIBUTE = "scitag"; + + private final int experimentId; + private final int activityId; + private final boolean remoteAddressIsSource; + + public SciTag(int experimentId, int activityId, boolean remoteAddressIsSource) { + this.experimentId = experimentId; + this.activityId = activityId; + this.remoteAddressIsSource = remoteAddressIsSource; + } + + public int experimentId() { + return experimentId; + } + + public int activityId() { + return activityId; + } + + public boolean remoteAddressIsSource() { + return remoteAddressIsSource; + } + +} diff --git a/src/main/java/org/italiangrid/storm/webdav/scitag/SciTagTransfer.java b/src/main/java/org/italiangrid/storm/webdav/scitag/SciTagTransfer.java new file mode 100644 index 00000000..5a99d53a --- /dev/null +++ b/src/main/java/org/italiangrid/storm/webdav/scitag/SciTagTransfer.java @@ -0,0 +1,80 @@ +/** + * Copyright (c) Istituto Nazionale di Fisica Nucleare, 2014-2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.italiangrid.storm.webdav.scitag; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SciTagTransfer { + + public static final Logger LOG = LoggerFactory.getLogger(SciTagTransfer.class); + public static final String SCITAG_TRANSFER_ATTRIBUTE = "scitagTransfer"; + + private static final String FLOWD_PIPE_NAME = "/var/run/flowd"; + private SciTag scitag; + private String sourceAddress; + private int sourcePort; + private String destinationAddress; + private int destinationPort; + private File flowdPipeFile; + + public SciTagTransfer(SciTag scitag, String localAddress, int localPort, String remoteAddress, + int remotePort) { + this(scitag, localAddress, localPort, remoteAddress, remotePort, new File(FLOWD_PIPE_NAME)); + } + + public SciTagTransfer(SciTag scitag, String localAddress, int localPort, String remoteAddress, + int remotePort, File flowdPipeFile) { + this.scitag = scitag; + if (scitag.remoteAddressIsSource()) { + this.sourceAddress = remoteAddress; + this.sourcePort = remotePort; + this.destinationAddress = localAddress; + this.destinationPort = localPort; + } else { + this.sourceAddress = localAddress; + this.sourcePort = localPort; + this.destinationAddress = remoteAddress; + this.destinationPort = remotePort; + } + this.flowdPipeFile = flowdPipeFile; + } + + private String flowdEntry() { + return " tcp " + sourceAddress + " " + sourcePort + " " + destinationAddress + " " + + destinationPort + " " + scitag.experimentId() + " " + scitag.activityId() + "\n"; + } + + public void writeStart() { + try (RandomAccessFile flowdPipe = new RandomAccessFile(flowdPipeFile, "rw")) { + flowdPipe.writeBytes("start" + this.flowdEntry()); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + } + } + + public void writeEnd() { + try (RandomAccessFile flowdPipe = new RandomAccessFile(flowdPipeFile, "rw")) { + flowdPipe.writeBytes("end" + this.flowdEntry()); + } catch (IOException e) { + LOG.warn(e.getMessage(), e); + } + } + +} diff --git a/src/main/java/org/italiangrid/storm/webdav/server/servlet/MiltonFilter.java b/src/main/java/org/italiangrid/storm/webdav/server/servlet/MiltonFilter.java index 666ea2a4..a413980b 100644 --- a/src/main/java/org/italiangrid/storm/webdav/server/servlet/MiltonFilter.java +++ b/src/main/java/org/italiangrid/storm/webdav/server/servlet/MiltonFilter.java @@ -38,6 +38,8 @@ import org.italiangrid.storm.webdav.milton.StoRMResourceFactory; import org.italiangrid.storm.webdav.milton.util.ReplaceContentStrategy; import org.italiangrid.storm.webdav.server.PathResolver; +import org.italiangrid.storm.webdav.scitag.SciTag; +import org.italiangrid.storm.webdav.scitag.SciTagTransfer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -133,11 +135,23 @@ public void doMilton(HttpServletRequest request, HttpServletResponse response) { Request miltonReq = new StoRMMiltonRequest(request, servletContext); Response miltonRes = new io.milton.servlet.ServletResponse(response); + SciTag scitag = (SciTag) request.getAttribute(SciTag.SCITAG_ATTRIBUTE); + if (scitag != null) { + SciTagTransfer scitagTransfer = new SciTagTransfer(scitag, request.getLocalAddr(), + request.getLocalPort(), request.getRemoteAddr(), request.getRemotePort()); + scitagTransfer.writeStart(); + request.setAttribute(SciTagTransfer.SCITAG_TRANSFER_ATTRIBUTE, scitagTransfer); + } miltonHTTPManager.process(miltonReq, miltonRes); } finally { MiltonServlet.clearThreadlocals(); + SciTagTransfer scitagTransfer = + (SciTagTransfer) request.getAttribute(SciTagTransfer.SCITAG_TRANSFER_ATTRIBUTE); + if (scitagTransfer != null) { + scitagTransfer.writeEnd(); + } try { diff --git a/src/main/java/org/italiangrid/storm/webdav/server/servlet/SciTagFilter.java b/src/main/java/org/italiangrid/storm/webdav/server/servlet/SciTagFilter.java new file mode 100644 index 00000000..546c562a --- /dev/null +++ b/src/main/java/org/italiangrid/storm/webdav/server/servlet/SciTagFilter.java @@ -0,0 +1,65 @@ +/** + * Copyright (c) Istituto Nazionale di Fisica Nucleare, 2014-2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.italiangrid.storm.webdav.server.servlet; + +import java.io.IOException; +import java.util.Optional; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; + +import org.italiangrid.storm.webdav.scitag.SciTag; +import org.italiangrid.storm.webdav.tpc.TransferConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SciTagFilter implements Filter { + + public static final Logger logger = LoggerFactory.getLogger(SciTagFilter.class); + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + HttpServletRequest req = (HttpServletRequest) request; + if (req.getHeader(SciTag.SCITAG_HEADER) != null) { + Optional source = Optional.ofNullable(req.getHeader(TransferConstants.SOURCE_HEADER)); + boolean remoteAddressIsSource = + req.getMethod().equals("PUT") || (req.getMethod().equals("COPY") && source.isPresent()); + // state prot src_ip src_port dst_ip dst_port exp act + // If the active party receives an HTTP-TPC COPY request with a SciTag request header with + // a valid value then the server SHOULD mark the resulting network traffic with the + // experiment ID and activity ID encoded in the value. + int scitagValue = Integer.parseInt(req.getHeader(SciTag.SCITAG_HEADER)); + // Valid value is a single positive integer > 64 and <65536 (16bit). Any other value is + // considered invalid. + if (scitagValue > 64 && scitagValue < 65536) { + request.setAttribute(SciTag.SCITAG_ATTRIBUTE, + new SciTag(scitagValue >> 6, scitagValue & ((1 << 6) - 1), remoteAddressIsSource)); + } else { + // If the active party receives an HTTP-TPC COPY request with a SciTag request header + // with an invalid value then the server SHOULD mark the resulting network traffic with + // the 0 as the experiment ID and the activity ID. + request.setAttribute(SciTag.SCITAG_ATTRIBUTE, new SciTag(0, 0, remoteAddressIsSource)); + } + } + chain.doFilter(request, response); + } +} diff --git a/src/main/java/org/italiangrid/storm/webdav/server/servlet/StoRMServlet.java b/src/main/java/org/italiangrid/storm/webdav/server/servlet/StoRMServlet.java index 03f83598..68145aa7 100644 --- a/src/main/java/org/italiangrid/storm/webdav/server/servlet/StoRMServlet.java +++ b/src/main/java/org/italiangrid/storm/webdav/server/servlet/StoRMServlet.java @@ -34,7 +34,7 @@ public class StoRMServlet extends DefaultServlet { /** - * + * */ private static final long serialVersionUID = 4204673943980786498L; @@ -74,6 +74,12 @@ public Resource getResource(String pathInContext) { } + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + resourceService.doGet(request, response); + } + @Override protected void doHead(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { diff --git a/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResourceService.java b/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResourceService.java index 64684415..aa606090 100644 --- a/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResourceService.java +++ b/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResourceService.java @@ -18,6 +18,7 @@ import java.io.FileNotFoundException; import java.io.IOException; +import javax.servlet.ServletException; import javax.servlet.RequestDispatcher; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -25,6 +26,8 @@ import org.eclipse.jetty.http.HttpContent; import org.eclipse.jetty.server.ResourceService; import org.eclipse.jetty.util.URIUtil; +import org.italiangrid.storm.webdav.scitag.SciTag; +import org.italiangrid.storm.webdav.scitag.SciTagTransfer; public class StormResourceService extends ResourceService { @@ -50,6 +53,23 @@ private String pathInContext(HttpServletRequest request) { return URIUtil.addPaths(servletPath, pathInfo); } + @Override + public boolean doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + SciTag scitag = (SciTag) request.getAttribute(SciTag.SCITAG_ATTRIBUTE); + SciTagTransfer scitagTransfer = null; + if (scitag != null) { + scitagTransfer = new SciTagTransfer(scitag, request.getLocalAddr(), request.getLocalPort(), + request.getRemoteAddr(), request.getRemotePort()); + scitagTransfer.writeStart(); + } + boolean result = super.doGet(request, response); + if (scitagTransfer != null) { + scitagTransfer.writeEnd(); + } + return result; + } + public boolean doHead(HttpServletRequest request, HttpServletResponse response) throws IOException { diff --git a/src/main/java/org/italiangrid/storm/webdav/spring/AppConfig.java b/src/main/java/org/italiangrid/storm/webdav/spring/AppConfig.java index 22d43a01..5c502f86 100644 --- a/src/main/java/org/italiangrid/storm/webdav/spring/AppConfig.java +++ b/src/main/java/org/italiangrid/storm/webdav/spring/AppConfig.java @@ -45,8 +45,6 @@ import org.apache.http.conn.HttpClientConnectionManager; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.LayeredConnectionSocketFactory; -import org.apache.http.conn.socket.PlainConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; @@ -92,6 +90,8 @@ import org.italiangrid.storm.webdav.server.util.CANLListener; import org.italiangrid.storm.webdav.tpc.LocalURLService; import org.italiangrid.storm.webdav.tpc.StaticHostListLocalURLService; +import org.italiangrid.storm.webdav.tpc.TpcPlainConnectionSocketFactory; +import org.italiangrid.storm.webdav.tpc.TpcSSLConnectionSocketFactory; import org.italiangrid.storm.webdav.tpc.TransferConstants; import org.italiangrid.storm.webdav.tpc.http.SuperLaxRedirectStrategy; import org.italiangrid.voms.util.CertificateValidatorBuilder; @@ -263,8 +263,8 @@ HttpClientConnectionManager tpcClientConnectionManager(ThirdPartyCopyProperties ctx.init(null, new TrustManager[] {tm}, null); } - ConnectionSocketFactory sf = PlainConnectionSocketFactory.getSocketFactory(); - LayeredConnectionSocketFactory tlsSf = new SSLConnectionSocketFactory(ctx); + ConnectionSocketFactory sf = TpcPlainConnectionSocketFactory.getSocketFactory(); + LayeredConnectionSocketFactory tlsSf = new TpcSSLConnectionSocketFactory(ctx); Registry r = RegistryBuilder.create() .register(HTTP, sf) diff --git a/src/main/java/org/italiangrid/storm/webdav/spring/web/ServletConfiguration.java b/src/main/java/org/italiangrid/storm/webdav/spring/web/ServletConfiguration.java index b6813811..261fac93 100644 --- a/src/main/java/org/italiangrid/storm/webdav/spring/web/ServletConfiguration.java +++ b/src/main/java/org/italiangrid/storm/webdav/spring/web/ServletConfiguration.java @@ -39,6 +39,7 @@ import org.italiangrid.storm.webdav.server.servlet.MiltonFilter; import org.italiangrid.storm.webdav.server.servlet.MoveRequestSanityChecksFilter; import org.italiangrid.storm.webdav.server.servlet.SAIndexServlet; +import org.italiangrid.storm.webdav.server.servlet.SciTagFilter; import org.italiangrid.storm.webdav.server.servlet.StoRMServlet; import org.italiangrid.storm.webdav.server.servlet.resource.StormResourceService; import org.italiangrid.storm.webdav.server.tracing.LogbackAccessAuthnInfoFilter; @@ -69,11 +70,12 @@ public class ServletConfiguration { public static final Logger LOG = LoggerFactory.getLogger(ServletConfiguration.class); static final int REQUEST_ID_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1000; - static final int LOGBACK_ACCESS_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1002; - static final int LOG_REQ_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1003; - static final int REDIRECT_REQ_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1004; - static final int CHECKSUM_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1005; - static final int MACAROON_REQ_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1006; + static final int LOGBACK_ACCESS_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1001; + static final int LOG_REQ_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1002; + static final int REDIRECT_REQ_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1003; + static final int CHECKSUM_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1004; + static final int MACAROON_REQ_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1005; + static final int SCITAG_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1006; static final int TPC_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1007; static final int MOVE_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1008; static final int DELETE_FILTER_ORDER = DEFAULT_FILTER_ORDER + 1009; @@ -151,6 +153,15 @@ FilterRegistrationBean macaroonRequestFilter(ObjectMapper return filter; } + @Bean + @ConditionalOnProperty(name = "storm.scitag.enabled", havingValue = "true") + FilterRegistrationBean scitagFilter() { + LOG.info("SciTag filter enabled"); + FilterRegistrationBean filter = new FilterRegistrationBean<>(new SciTagFilter()); + filter.setOrder(SCITAG_FILTER_ORDER); + return filter; + } + @Bean FilterRegistrationBean miltonFilter(FilesystemAccess fsAccess, ExtendedAttributesHelper attrsHelper, PathResolver resolver, ReplaceContentStrategy rcs) { diff --git a/src/main/java/org/italiangrid/storm/webdav/tpc/TpcPlainConnectionSocketFactory.java b/src/main/java/org/italiangrid/storm/webdav/tpc/TpcPlainConnectionSocketFactory.java new file mode 100644 index 00000000..04289a9d --- /dev/null +++ b/src/main/java/org/italiangrid/storm/webdav/tpc/TpcPlainConnectionSocketFactory.java @@ -0,0 +1,56 @@ +/** + * Copyright (c) Istituto Nazionale di Fisica Nucleare, 2014-2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.italiangrid.storm.webdav.tpc; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import org.apache.http.HttpHost; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; +import org.apache.http.protocol.HttpContext; +import org.italiangrid.storm.webdav.scitag.SciTag; +import org.italiangrid.storm.webdav.scitag.SciTagTransfer; + +public class TpcPlainConnectionSocketFactory extends PlainConnectionSocketFactory { + + public static final TpcPlainConnectionSocketFactory INSTANCE = + new TpcPlainConnectionSocketFactory(); + + public static TpcPlainConnectionSocketFactory getSocketFactory() { + return INSTANCE; + } + + public TpcPlainConnectionSocketFactory() { + super(); + } + + @Override + public Socket connectSocket(int connectTimeout, Socket socket, HttpHost host, + InetSocketAddress remoteAddress, InetSocketAddress localAddress, HttpContext context) + throws IOException { + Socket s = + super.connectSocket(connectTimeout, socket, host, remoteAddress, localAddress, context); + SciTag scitag = (SciTag) context.getAttribute(SciTag.SCITAG_ATTRIBUTE); + if (scitag != null) { + SciTagTransfer scitagTransfer = + new SciTagTransfer(scitag, s.getLocalAddress().getHostAddress(), s.getLocalPort(), + s.getInetAddress().getHostAddress(), s.getPort()); + scitagTransfer.writeStart(); + context.setAttribute(SciTagTransfer.SCITAG_TRANSFER_ATTRIBUTE, scitagTransfer); + } + return s; + } +} diff --git a/src/main/java/org/italiangrid/storm/webdav/tpc/TpcSSLConnectionSocketFactory.java b/src/main/java/org/italiangrid/storm/webdav/tpc/TpcSSLConnectionSocketFactory.java new file mode 100644 index 00000000..d7660cb6 --- /dev/null +++ b/src/main/java/org/italiangrid/storm/webdav/tpc/TpcSSLConnectionSocketFactory.java @@ -0,0 +1,46 @@ +/** + * Copyright (c) Istituto Nazionale di Fisica Nucleare, 2014-2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.italiangrid.storm.webdav.tpc; + +import java.io.IOException; +import java.net.Socket; +import javax.net.ssl.SSLContext; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.protocol.HttpContext; +import org.italiangrid.storm.webdav.scitag.SciTag; +import org.italiangrid.storm.webdav.scitag.SciTagTransfer; + +public class TpcSSLConnectionSocketFactory extends SSLConnectionSocketFactory { + + public TpcSSLConnectionSocketFactory(SSLContext sslContext) { + super(sslContext); + } + + @Override + public Socket createLayeredSocket(Socket socket, String target, int port, HttpContext context) + throws IOException { + Socket s = super.createLayeredSocket(socket, target, port, context); + SciTag scitag = (SciTag) context.getAttribute(SciTag.SCITAG_ATTRIBUTE); + if (scitag != null) { + SciTagTransfer scitagTransfer = + new SciTagTransfer(scitag, s.getLocalAddress().getHostAddress(), s.getLocalPort(), + s.getInetAddress().getHostAddress(), s.getPort()); + scitagTransfer.writeStart(); + context.setAttribute(SciTagTransfer.SCITAG_TRANSFER_ATTRIBUTE, scitagTransfer); + } + return s; + } +} diff --git a/src/main/java/org/italiangrid/storm/webdav/tpc/TransferFilter.java b/src/main/java/org/italiangrid/storm/webdav/tpc/TransferFilter.java index d2dec61d..7788865a 100644 --- a/src/main/java/org/italiangrid/storm/webdav/tpc/TransferFilter.java +++ b/src/main/java/org/italiangrid/storm/webdav/tpc/TransferFilter.java @@ -36,6 +36,7 @@ import org.apache.http.client.HttpResponseException; import org.italiangrid.storm.webdav.error.BadRequest; import org.italiangrid.storm.webdav.error.ResourceNotFound; +import org.italiangrid.storm.webdav.scitag.SciTag; import org.italiangrid.storm.webdav.server.PathResolver; import org.italiangrid.storm.webdav.server.tracing.RequestIdHolder; import org.italiangrid.storm.webdav.tpc.transfer.GetTransferRequest; @@ -108,10 +109,11 @@ protected void handleTpc(HttpServletRequest request, HttpServletResponse respons if (validRequest(request, response)) { Optional source = Optional.ofNullable(request.getHeader(SOURCE_HEADER)); + SciTag scitag = (SciTag) request.getAttribute(SciTag.SCITAG_ATTRIBUTE); if (source.isPresent()) { - handlePullCopy(request, response); + handlePullCopy(request, response, scitag); } else { - handlePushCopy(request, response); + handlePushCopy(request, response, scitag); } } @@ -210,8 +212,8 @@ protected void logTransferException(TransferRequest request, Exception e) { } } - protected void handlePullCopy(HttpServletRequest request, HttpServletResponse response) - throws IOException { + protected void handlePullCopy(HttpServletRequest request, HttpServletResponse response, + SciTag scitag) throws IOException { URI uri = URI.create(request.getHeader(SOURCE_HEADER)); String path = getScopedPathInfo(request); @@ -221,6 +223,7 @@ protected void handlePullCopy(HttpServletRequest request, HttpServletResponse re .uri(uri) .path(path) .headers(getTransferHeaders(request, response)) + .scitag(scitag) .verifyChecksum(verifyChecksum && verifyChecksumRequested(request)) .overwrite(overwriteRequested(request)) .build(); @@ -249,8 +252,8 @@ protected void handlePullCopy(HttpServletRequest request, HttpServletResponse re } } - protected void handlePushCopy(HttpServletRequest request, HttpServletResponse response) - throws IOException { + protected void handlePushCopy(HttpServletRequest request, HttpServletResponse response, + SciTag scitag) throws IOException { URI uri = URI.create(request.getHeader(DESTINATION_HEADER)); String path = getScopedPathInfo(request); @@ -259,6 +262,7 @@ protected void handlePushCopy(HttpServletRequest request, HttpServletResponse re .uri(uri) .path(path) .headers(getTransferHeaders(request, response)) + .scitag(scitag) .verifyChecksum(verifyChecksum && verifyChecksumRequested(request)) .overwrite(overwriteRequested(request)) .build(); diff --git a/src/main/java/org/italiangrid/storm/webdav/tpc/TransferFilterSupport.java b/src/main/java/org/italiangrid/storm/webdav/tpc/TransferFilterSupport.java index 5a087e9e..54046d8e 100644 --- a/src/main/java/org/italiangrid/storm/webdav/tpc/TransferFilterSupport.java +++ b/src/main/java/org/italiangrid/storm/webdav/tpc/TransferFilterSupport.java @@ -34,6 +34,7 @@ import org.apache.http.client.ClientProtocolException; import org.apache.http.client.HttpResponseException; +import org.italiangrid.storm.webdav.scitag.SciTag; import org.italiangrid.storm.webdav.server.PathResolver; import org.italiangrid.storm.webdav.tpc.transfer.TransferRequest; import org.italiangrid.storm.webdav.tpc.transfer.TransferStatus; @@ -87,6 +88,14 @@ protected Multimap getTransferHeaders(HttpServletRequest request LOG.warn("Ignoring invalid transfer header {}", headerName); continue; } + if (xferHeaderName.trim().equalsIgnoreCase(SciTag.SCITAG_HEADER) + && request.getHeader(SciTag.SCITAG_HEADER) != null) { + // If the active party receives an HTTP-TPC COPY request with both a SciTag request header + // and a TransferHeaderSciTag request header then it SHOULD ignore the + // TransferHeaderSciTag and continue to process the request. + LOG.warn("Ignoring TransferHeaderSciTag header because SciTag header is present"); + continue; + } xferHeaders.put(xferHeaderName.trim(), request.getHeader(headerName)); } } diff --git a/src/main/java/org/italiangrid/storm/webdav/tpc/http/HttpTransferClient.java b/src/main/java/org/italiangrid/storm/webdav/tpc/http/HttpTransferClient.java index f89f0154..87cad35d 100644 --- a/src/main/java/org/italiangrid/storm/webdav/tpc/http/HttpTransferClient.java +++ b/src/main/java/org/italiangrid/storm/webdav/tpc/http/HttpTransferClient.java @@ -39,11 +39,14 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpPut; +import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.impl.client.CloseableHttpClient; import org.italiangrid.storm.webdav.config.ServiceConfigurationProperties; import org.italiangrid.storm.webdav.config.ThirdPartyCopyProperties; import org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributesHelper; import org.italiangrid.storm.webdav.server.PathResolver; +import org.italiangrid.storm.webdav.scitag.SciTag; +import org.italiangrid.storm.webdav.scitag.SciTagTransfer; import org.italiangrid.storm.webdav.tpc.transfer.GetTransferRequest; import org.italiangrid.storm.webdav.tpc.transfer.PutTransferRequest; import org.italiangrid.storm.webdav.tpc.transfer.TransferClient; @@ -165,6 +168,7 @@ public void handle(GetTransferRequest request, TransferStatusCallback cb) { StormCountingOutputStream os = prepareOutputStream(resolver.resolvePath(request.path())); HttpGet get = prepareRequest(request); + HttpClientContext context = HttpClientContext.create(); ScheduledFuture reportTask = executorService.scheduleAtFixedRate(() -> { reportStatus(cb, request, statusBuilder.inProgress(os.getCount())); @@ -172,8 +176,9 @@ public void handle(GetTransferRequest request, TransferStatusCallback cb) { try { + context.setAttribute(SciTag.SCITAG_ATTRIBUTE, request.scitag()); httpClient.execute(get, new GetResponseHandler(request, os, attributesHelper, - MDC.getCopyOfContextMap(), socketBufferSize, true)); + MDC.getCopyOfContextMap(), socketBufferSize, true), context); reportTask.cancel(true); reportStatus(cb, request, statusBuilder.done(os.getCount())); @@ -196,6 +201,11 @@ public void handle(GetTransferRequest request, TransferStatusCallback cb) { if (!reportTask.isCancelled()) { reportTask.cancel(true); } + SciTagTransfer scitagTransfer = + (SciTagTransfer) context.getAttribute(SciTagTransfer.SCITAG_TRANSFER_ATTRIBUTE); + if (scitagTransfer != null) { + scitagTransfer.writeEnd(); + } } } @@ -221,6 +231,7 @@ public void handle(PutTransferRequest request, TransferStatusCallback cb) { CountingFileEntity cfe = prepareFileEntity(resolver.resolvePath(request.path())); HttpPut put = null; + HttpClientContext context = HttpClientContext.create(); try { put = prepareRequest(request, cfe); @@ -236,7 +247,8 @@ public void handle(PutTransferRequest request, TransferStatusCallback cb) { try { checkOverwrite(request); - httpClient.execute(put, new PutResponseHandler(MDC.getCopyOfContextMap())); + context.setAttribute(SciTag.SCITAG_ATTRIBUTE, request.scitag()); + httpClient.execute(put, new PutResponseHandler(MDC.getCopyOfContextMap()), context); reportTask.cancel(true); reportStatus(cb, request, statusBuilder.done(cfe.getCount())); } catch (HttpResponseException e) { @@ -255,6 +267,11 @@ public void handle(PutTransferRequest request, TransferStatusCallback cb) { if (!reportTask.isCancelled()) { reportTask.cancel(true); } + SciTagTransfer scitagTransfer = + (SciTagTransfer) context.getAttribute(SciTagTransfer.SCITAG_TRANSFER_ATTRIBUTE); + if (scitagTransfer != null) { + scitagTransfer.writeEnd(); + } } } diff --git a/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/GetTransferRequestBuilder.java b/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/GetTransferRequestBuilder.java index a993ff9e..ea9107ec 100644 --- a/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/GetTransferRequestBuilder.java +++ b/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/GetTransferRequestBuilder.java @@ -21,7 +21,7 @@ public class GetTransferRequestBuilder extends RequestBuilder { String uuid; @@ -28,6 +30,8 @@ public abstract class RequestBuilder { URI uri; + SciTag scitag; + boolean verifyChecksum = true; boolean overwrite = true; @@ -63,6 +67,11 @@ public RequestBuilder addHeader(String header, String value) { return this; } + public RequestBuilder scitag(SciTag scitag) { + this.scitag = scitag; + return this; + } + public RequestBuilder overwrite(boolean o) { overwrite = o; return this; diff --git a/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/TransferRequest.java b/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/TransferRequest.java index 5b149147..70e480e4 100644 --- a/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/TransferRequest.java +++ b/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/TransferRequest.java @@ -22,6 +22,8 @@ import com.google.common.collect.Multimap; +import org.italiangrid.storm.webdav.scitag.SciTag; + public interface TransferRequest { String uuid(); @@ -32,6 +34,8 @@ public interface TransferRequest { Multimap transferHeaders(); + SciTag scitag(); + boolean verifyChecksum(); boolean overwrite(); diff --git a/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/impl/GetTransferRequestImpl.java b/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/impl/GetTransferRequestImpl.java index 7cb4e2e5..f0368061 100644 --- a/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/impl/GetTransferRequestImpl.java +++ b/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/impl/GetTransferRequestImpl.java @@ -19,6 +19,7 @@ import java.net.URI; +import org.italiangrid.storm.webdav.scitag.SciTag; import org.italiangrid.storm.webdav.tpc.transfer.GetTransferRequest; import com.google.common.collect.Multimap; @@ -26,9 +27,9 @@ public class GetTransferRequestImpl extends TransferRequestImpl implements GetTransferRequest { public GetTransferRequestImpl(String uuid, String path, URI uri, - Multimap xferHeaders, - boolean verifyChecksum, boolean overwrite) { - super(uuid, path, uri, xferHeaders, verifyChecksum, overwrite); + Multimap xferHeaders, SciTag scitag, boolean verifyChecksum, + boolean overwrite) { + super(uuid, path, uri, xferHeaders, scitag, verifyChecksum, overwrite); } @Override diff --git a/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/impl/PutTransferRequestImpl.java b/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/impl/PutTransferRequestImpl.java index b2f4d37c..60f273eb 100644 --- a/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/impl/PutTransferRequestImpl.java +++ b/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/impl/PutTransferRequestImpl.java @@ -19,6 +19,7 @@ import java.net.URI; +import org.italiangrid.storm.webdav.scitag.SciTag; import org.italiangrid.storm.webdav.tpc.transfer.PutTransferRequest; import com.google.common.collect.Multimap; @@ -26,8 +27,9 @@ public class PutTransferRequestImpl extends TransferRequestImpl implements PutTransferRequest { public PutTransferRequestImpl(String uuid, String path, URI uri, - Multimap xferHeaders, boolean verifyChecksum, boolean overwrite) { - super(uuid, path, uri, xferHeaders, verifyChecksum, overwrite); + Multimap xferHeaders, SciTag scitag, boolean verifyChecksum, + boolean overwrite) { + super(uuid, path, uri, xferHeaders, scitag, verifyChecksum, overwrite); } @Override diff --git a/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/impl/TransferRequestImpl.java b/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/impl/TransferRequestImpl.java index a1d29eb4..737ac992 100644 --- a/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/impl/TransferRequestImpl.java +++ b/src/main/java/org/italiangrid/storm/webdav/tpc/transfer/impl/TransferRequestImpl.java @@ -21,6 +21,7 @@ import java.util.Objects; import java.util.Optional; +import org.italiangrid.storm.webdav.scitag.SciTag; import org.italiangrid.storm.webdav.tpc.transfer.TransferRequest; import org.italiangrid.storm.webdav.tpc.transfer.TransferStatus; import org.italiangrid.storm.webdav.tpc.transfer.TransferStatus.Status; @@ -40,6 +41,8 @@ public abstract class TransferRequestImpl implements TransferRequest { final Multimap xferHeaders; + final SciTag scitag; + final boolean verifyChecksum; final boolean overwrite; @@ -51,12 +54,13 @@ public abstract class TransferRequestImpl implements TransferRequest { private Optional lastTransferStatus = Optional.empty(); TransferRequestImpl(String uuid, String path, URI uri, Multimap xferHeaders, - boolean verifyChecksum, boolean overwrite) { + SciTag scitag, boolean verifyChecksum, boolean overwrite) { this.uuid = uuid; this.path = path; this.uri = uri; this.xferHeaders = xferHeaders; + this.scitag = scitag; this.verifyChecksum = verifyChecksum; this.overwrite = overwrite; } @@ -76,6 +80,11 @@ public Multimap transferHeaders() { return xferHeaders; } + @Override + public SciTag scitag() { + return scitag; + } + @Override public boolean verifyChecksum() { return verifyChecksum; diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 4db620d9..e18c7e51 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -85,6 +85,9 @@ storm: macaroon-filter: enabled: ${STORM_WEBDAV_MACAROON_FILTER_ENABLED:true} + scitag: + enabled: ${STORM_WEBDAV_SCITAG_ENABLED:false} + server: # Jetty Thread-Pool maximum idle time (in milliseconds) max-idle-time-msec: ${STORM_WEBDAV_SERVER_MAX_IDLE_TIME:3600000} diff --git a/src/test/java/org/italiangrid/storm/webdav/test/tpc/PullTransferTest.java b/src/test/java/org/italiangrid/storm/webdav/test/tpc/PullTransferTest.java index 3c7e84a6..1eaee85c 100644 --- a/src/test/java/org/italiangrid/storm/webdav/test/tpc/PullTransferTest.java +++ b/src/test/java/org/italiangrid/storm/webdav/test/tpc/PullTransferTest.java @@ -104,9 +104,11 @@ void checkTransferHeaderPassing() throws IOException, ServletException { .thenReturn(TRANSFER_HEADER_AUTHORIZATION_VALUE); when(request.getHeader(TRANSFER_HEADER_WHATEVER_KEY)) .thenReturn(TRANSFER_HEADER_WHATEVER_VALUE); + when(request.getHeader(TRANSFER_HEADER_SCITAG)).thenReturn(SCITAG_HEADER_VALUE); + when(request.getHeader(SCITAG_HEADER)).thenReturn(null); - when(request.getHeaderNames()).thenReturn( - enumeration(asList(TRANSFER_HEADER_AUTHORIZATION_KEY, TRANSFER_HEADER_WHATEVER_KEY))); + when(request.getHeaderNames()).thenReturn(enumeration( + asList(TRANSFER_HEADER_AUTHORIZATION_KEY, TRANSFER_HEADER_WHATEVER_KEY, TRANSFER_HEADER_SCITAG))); filter.doFilter(request, response, chain); verify(client).handle(getXferRequest.capture(), Mockito.any()); @@ -118,18 +120,20 @@ void checkTransferHeaderPassing() throws IOException, ServletException { Multimap xferHeaders = getXferRequest.getValue().transferHeaders(); - assertThat(xferHeaders.size(), is(2)); + assertThat(xferHeaders.size(), is(3)); assertThat(xferHeaders.containsKey("Authorization"), is(true)); assertThat(xferHeaders.get("Authorization").iterator().next(), is(TRANSFER_HEADER_AUTHORIZATION_VALUE)); assertThat(xferHeaders.containsKey("Whatever"), is(true)); assertThat(xferHeaders.get("Whatever").iterator().next(), is(TRANSFER_HEADER_WHATEVER_VALUE)); + assertThat(xferHeaders.containsKey("SciTag"), is(true)); + assertThat(xferHeaders.get("SciTag").iterator().next(), is(SCITAG_HEADER_VALUE)); } @Test void emptyTransferHeaderAreIgnored() throws IOException, ServletException { - when(request.getHeaderNames()) - .thenReturn(enumeration(asList(TRANSFER_HEADER, TRANSFER_HEADER_WHATEVER_KEY))); + when(request.getHeaderNames()).thenReturn( + enumeration(asList(TRANSFER_HEADER, TRANSFER_HEADER_WHATEVER_KEY))); when(request.getHeader(TRANSFER_HEADER_WHATEVER_KEY)) .thenReturn(TRANSFER_HEADER_WHATEVER_VALUE); @@ -150,4 +154,26 @@ void emptyTransferHeaderAreIgnored() throws IOException, ServletException { assertThat(xferHeaders.get("Whatever").iterator().next(), is(TRANSFER_HEADER_WHATEVER_VALUE)); } + @Test + void bothSciTagAndTransferHeaderSciTag() throws IOException, ServletException { + when(request.getHeaderNames()) + .thenReturn(enumeration(asList(SCITAG_HEADER, TRANSFER_HEADER_SCITAG))); + + when(request.getHeader(SCITAG_HEADER)).thenReturn(SCITAG_HEADER_VALUE); + + filter.doFilter(request, response, chain); + verify(client).handle(getXferRequest.capture(), Mockito.any()); + + assertThat(getXferRequest.getValue().path(), is(FULL_LOCAL_PATH)); + assertThat(getXferRequest.getValue().remoteURI(), is(HTTP_URL_URI)); + assertThat(getXferRequest.getValue().overwrite(), is(true)); + assertThat(getXferRequest.getValue().verifyChecksum(), is(true)); + + + Multimap xferHeaders = getXferRequest.getValue().transferHeaders(); + assertThat(xferHeaders.size(), is(0)); + + assertThat(xferHeaders.containsKey("SciTag"), is(false)); + } + } diff --git a/src/test/java/org/italiangrid/storm/webdav/test/tpc/PushTransferTest.java b/src/test/java/org/italiangrid/storm/webdav/test/tpc/PushTransferTest.java index e70feeb0..6569e33a 100644 --- a/src/test/java/org/italiangrid/storm/webdav/test/tpc/PushTransferTest.java +++ b/src/test/java/org/italiangrid/storm/webdav/test/tpc/PushTransferTest.java @@ -113,8 +113,8 @@ void checkTransferHeaderPassing() throws IOException, ServletException { when(request.getHeader(TRANSFER_HEADER_WHATEVER_KEY)) .thenReturn(TRANSFER_HEADER_WHATEVER_VALUE); - when(request.getHeaderNames()).thenReturn( - enumeration(asList(TRANSFER_HEADER_AUTHORIZATION_KEY, TRANSFER_HEADER_WHATEVER_KEY))); + when(request.getHeaderNames()).thenReturn(enumeration( + asList(TRANSFER_HEADER_AUTHORIZATION_KEY, TRANSFER_HEADER_WHATEVER_KEY, SCITAG_HEADER))); filter.doFilter(request, response, chain); verify(client).handle(putXferRequest.capture(), Mockito.any()); @@ -132,12 +132,13 @@ void checkTransferHeaderPassing() throws IOException, ServletException { is(TRANSFER_HEADER_AUTHORIZATION_VALUE)); assertThat(xferHeaders.containsKey("Whatever"), is(true)); assertThat(xferHeaders.get("Whatever").iterator().next(), is(TRANSFER_HEADER_WHATEVER_VALUE)); + assertThat(xferHeaders.containsKey("SciTag"), is(false)); } @Test void emptyTransferHeaderAreIgnored() throws IOException, ServletException { - when(request.getHeaderNames()) - .thenReturn(enumeration(asList(TRANSFER_HEADER, TRANSFER_HEADER_WHATEVER_KEY))); + when(request.getHeaderNames()).thenReturn( + enumeration(asList(TRANSFER_HEADER, TRANSFER_HEADER_WHATEVER_KEY))); when(request.getHeader(TRANSFER_HEADER_WHATEVER_KEY)) .thenReturn(TRANSFER_HEADER_WHATEVER_VALUE); @@ -207,4 +208,26 @@ void checkExpectContinueHeaderIsNotSet() throws IOException, ServletException { assertThat(xferHeaders.containsKey(EXPECTED_HEADER), is(false)); } + + @Test + void bothSciTagAndTransferHeaderSciTag() throws IOException, ServletException { + when(request.getHeaderNames()) + .thenReturn(enumeration(asList(SCITAG_HEADER, TRANSFER_HEADER_SCITAG))); + + when(request.getHeader(SCITAG_HEADER)).thenReturn(SCITAG_HEADER_VALUE); + + filter.doFilter(request, response, chain); + verify(client).handle(putXferRequest.capture(), Mockito.any()); + + assertThat(putXferRequest.getValue().path(), is(FULL_LOCAL_PATH)); + assertThat(putXferRequest.getValue().remoteURI(), is(HTTPS_URL_URI)); + assertThat(putXferRequest.getValue().overwrite(), is(true)); + assertThat(putXferRequest.getValue().verifyChecksum(), is(true)); + + + Multimap xferHeaders = putXferRequest.getValue().transferHeaders(); + assertThat(xferHeaders.size(), is(0)); + + assertThat(xferHeaders.containsKey("SciTag"), is(false)); + } } diff --git a/src/test/java/org/italiangrid/storm/webdav/test/tpc/SciTagFilterActivationTest.java b/src/test/java/org/italiangrid/storm/webdav/test/tpc/SciTagFilterActivationTest.java new file mode 100644 index 00000000..4a87ae05 --- /dev/null +++ b/src/test/java/org/italiangrid/storm/webdav/test/tpc/SciTagFilterActivationTest.java @@ -0,0 +1,100 @@ +/** + * Copyright (c) Istituto Nazionale di Fisica Nucleare, 2014-2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.italiangrid.storm.webdav.test.tpc; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; + +import javax.servlet.ServletException; + +import org.italiangrid.storm.webdav.config.StorageAreaInfo; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.italiangrid.storm.webdav.server.servlet.SciTagFilter; +import org.italiangrid.storm.webdav.scitag.SciTag; +import org.italiangrid.storm.webdav.scitag.SciTagTransfer; + +@ExtendWith(MockitoExtension.class) +class SciTagFilterActivationTest extends TransferFilterTestSupport { + @TempDir + File tempDir; + + SciTagFilter sciTagFilter; + + @Mock + StorageAreaInfo testSa; + + @Mock + StorageAreaInfo otherSa; + + @Override + @BeforeEach + public void setup() throws IOException { + super.setup(); + sciTagFilter = new SciTagFilter(); + lenient().when(request.getServletPath()).thenReturn(SERVLET_PATH); + lenient().when(request.getPathInfo()).thenReturn(LOCAL_PATH); + lenient().when(response.getWriter()).thenReturn(responseWriter); + lenient().when(resolver.resolveStorageArea(FULL_LOCAL_PATH)).thenReturn(testSa); + lenient().when(resolver.resolveStorageArea("/test/otherfile")).thenReturn(testSa); + lenient().when(resolver.resolveStorageArea("/other/file")).thenReturn(otherSa); + lenient().when(request.getHeader(SOURCE_HEADER)).thenReturn(null); + } + + @Test + void requestWithScitag() throws IOException, ServletException { + when(request.getMethod()).thenReturn("GET"); + when(request.getHeader(SCITAG_HEADER)).thenReturn("66"); + + sciTagFilter.doFilter(request, response, chain); + verify(chain).doFilter(request, response); + verify(request).setAttribute(eq(SciTag.SCITAG_ATTRIBUTE), any(SciTag.class)); + } + + @Test + void requestWithoutScitag() throws IOException, ServletException { + sciTagFilter.doFilter(request, response, chain); + verify(chain).doFilter(request, response); + verify(request, times(0)).setAttribute(eq(SciTag.SCITAG_ATTRIBUTE), any(SciTag.class)); + } + + @Test + void testSciTagWrite() { + SciTag scitag = new SciTag(1, 2, false); + assertThat(scitag.experimentId(), is(1)); + assertThat(scitag.activityId(), is(2)); + assertThat(scitag.remoteAddressIsSource(), is(false)); + File mockFile = new File(tempDir, "flowd"); + SciTagTransfer scitagTransfer = + new SciTagTransfer(scitag, "10.10.10.10", 8443, "10.10.10.11", 12345, mockFile); + scitagTransfer.writeStart(); + scitagTransfer.writeEnd(); + } + +} diff --git a/src/test/java/org/italiangrid/storm/webdav/test/tpc/TransferFilterTestSupport.java b/src/test/java/org/italiangrid/storm/webdav/test/tpc/TransferFilterTestSupport.java index 1702f996..6733e0e5 100644 --- a/src/test/java/org/italiangrid/storm/webdav/test/tpc/TransferFilterTestSupport.java +++ b/src/test/java/org/italiangrid/storm/webdav/test/tpc/TransferFilterTestSupport.java @@ -67,6 +67,10 @@ public class TransferFilterTestSupport implements TransferConstants { public static final String TRANSFER_HEADER_WHATEVER_KEY = "TransferHeaderWhatever"; public static final String TRANSFER_HEADER_WHATEVER_VALUE = "papisilviobelluscona"; + public static final String SCITAG_HEADER = "SciTag"; + public static final String SCITAG_HEADER_VALUE = "65"; + public static final String TRANSFER_HEADER_SCITAG = "TransferHeaderSciTag"; + public static final URI HTTP_URL_URI = URI.create(HTTP_URL); public static final URI HTTPS_URL_URI = URI.create(HTTPS_URL); diff --git a/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/ClientTest.java b/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/ClientTest.java index 4cad306d..353df6ba 100644 --- a/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/ClientTest.java +++ b/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/ClientTest.java @@ -30,6 +30,7 @@ import org.apache.http.client.ResponseHandler; import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.protocol.HttpClientContext; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -74,7 +75,8 @@ public void testClientCorrectlyBuildsHttpRequestNoHeaders() throws IOException { }); verify(httpClient).execute(getRequest.capture(), - ArgumentMatchers.>any()); + ArgumentMatchers.>any(), + ArgumentMatchers.any()); HttpGet httpGetReq = getRequest.getValue(); @@ -92,7 +94,8 @@ public void testClientCorrectlyBuildsHttpRequestWithHeaders() throws IOException }); verify(httpClient).execute(getRequest.capture(), - ArgumentMatchers.>any()); + ArgumentMatchers.>any(), + ArgumentMatchers.any()); HttpGet httpGetReq = getRequest.getValue(); diff --git a/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/integration/TpcClientRedirectionTest.java b/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/integration/TpcClientRedirectionTest.java index 73cebca4..498084a4 100644 --- a/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/integration/TpcClientRedirectionTest.java +++ b/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/integration/TpcClientRedirectionTest.java @@ -149,9 +149,8 @@ public void handleCrossProtocolRedirectionCorrectly() { headers.put("Authorization", "Bearer this-is-a-fake-token"); - GetTransferRequest getRequest = - new GetTransferRequestImpl(UUID.randomUUID().toString(), - "/test/example", URI.create(mockHttpsUrl("/test/example")), headers, false, false); + GetTransferRequest getRequest = new GetTransferRequestImpl(UUID.randomUUID().toString(), + "/test/example", URI.create(mockHttpsUrl("/test/example")), headers, null, false, false); mockServer .when(request().withMethod("GET").withPath("/test/example").withSecure(true), diff --git a/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/integration/TpcIntegrationTest.java b/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/integration/TpcIntegrationTest.java index 27b3d7e3..ca1fba9f 100644 --- a/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/integration/TpcIntegrationTest.java +++ b/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/integration/TpcIntegrationTest.java @@ -28,6 +28,7 @@ import java.net.URI; import java.util.UUID; +import org.italiangrid.storm.webdav.scitag.SciTag; import org.italiangrid.storm.webdav.tpc.http.HttpTransferClient; import org.italiangrid.storm.webdav.tpc.transfer.GetTransferRequest; import org.italiangrid.storm.webdav.tpc.transfer.PutTransferRequest; @@ -92,7 +93,8 @@ public void testPutRedirectHandled() { Multimap emptyHeaders = ArrayListMultimap.create(); PutTransferRequest putRequest = new PutTransferRequestImpl(UUID.randomUUID().toString(), - "/test/example", URI.create(mockUrl("/test/example")), emptyHeaders, false, true); + "/test/example", URI.create(mockUrl("/test/example")), emptyHeaders, new SciTag(1, 2, true), + false, true); mockServer.when(request().withMethod("PUT").withPath("/test/example"), Times.exactly(1)) .respond(HttpResponse.response() @@ -121,7 +123,7 @@ public void testAuthorizationHeaderIsDroppedOnRedirectForPut() { headers.put("Authorization", "Bearer this-is-a-fake-token"); PutTransferRequest putRequest = new PutTransferRequestImpl(UUID.randomUUID().toString(), - "/test/example", URI.create(mockUrl("/test/example")), headers, false, true); + "/test/example", URI.create(mockUrl("/test/example")), headers, null, false, true); mockServer.when(request().withMethod("PUT").withPath("/test/example"), Times.exactly(1)) .respond(HttpResponse.response() @@ -153,7 +155,7 @@ public void testAuthorizationHeaderIsDroppedOnRedirectForGet() { GetTransferRequest getRequest = new GetTransferRequestImpl(UUID.randomUUID().toString(), - "/test/example", URI.create(mockUrl("/test/example")), headers, false, false); + "/test/example", URI.create(mockUrl("/test/example")), headers, null, false, false); mockServer.when(request().withMethod("GET").withPath("/test/example"), Times.exactly(1)) diff --git a/src/test/resources/application-authz-test.yml b/src/test/resources/application-authz-test.yml index 3d9ceafd..df719d4f 100644 --- a/src/test/resources/application-authz-test.yml +++ b/src/test/resources/application-authz-test.yml @@ -20,6 +20,9 @@ storm: redirector: enabled: false + + scitag: + enabled: true authz: policies: @@ -52,4 +55,4 @@ storm: params: iss: https://issuer.example group: /example/admins - \ No newline at end of file +