diff --git a/modules/dcache-srm/src/main/java/diskCacheV111/srm/dcache/Storage.java b/modules/dcache-srm/src/main/java/diskCacheV111/srm/dcache/Storage.java index bb3132bd30e..05409d0487b 100644 --- a/modules/dcache-srm/src/main/java/diskCacheV111/srm/dcache/Storage.java +++ b/modules/dcache-srm/src/main/java/diskCacheV111/srm/dcache/Storage.java @@ -151,6 +151,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; @@ -1784,7 +1785,7 @@ private String performRemoteTransfer(SRMUser srmUser, remoteTURL.toString(), isVerifyRequired(extraInfo), httpHeaders(extraInfo), credential, - Optional.empty()); + Collections.emptyList()); break; case "http": @@ -1792,7 +1793,7 @@ private String performRemoteTransfer(SRMUser srmUser, 1, 1, remoteAddr, remoteTURL.toString(), isVerifyRequired(extraInfo), httpHeaders(extraInfo), - Optional.empty()); + Collections.emptyList()); break; default: diff --git a/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/HttpProtocolInfo.java b/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/HttpProtocolInfo.java index 8e1cd76d557..ae4aa13ab2f 100644 --- a/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/HttpProtocolInfo.java +++ b/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/HttpProtocolInfo.java @@ -1,8 +1,13 @@ package diskCacheV111.vehicles; +import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; -import java.util.Optional; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.dcache.util.ChecksumType; /** @@ -42,7 +47,10 @@ public enum Disposition { private final String httpDoorDomainName; private final String path; private final URI _location; + @Nullable private final ChecksumType _wantedChecksum; + @Nonnull + private Set _wantedChecksums; private final Disposition _disposition; @@ -55,7 +63,7 @@ public HttpProtocolInfo(String protocol, int major, int minor, String path, URI location) { this(protocol, major, minor, clientSocketAddress, httpDoorCellName, - httpDoorDomainName, path, location, null, null); + httpDoorDomainName, path, location, null, Collections.emptyList()); } public HttpProtocolInfo(String protocol, int major, int minor, @@ -66,9 +74,15 @@ public HttpProtocolInfo(String protocol, int major, int minor, URI location, Disposition disposition) { this(protocol, major, minor, clientSocketAddress, httpDoorCellName, - httpDoorDomainName, path, location, disposition, null); + httpDoorDomainName, path, location, disposition, + Collections.emptyList()); } + /** + * @param wantedChecksums Desired checksum to calculate for this transfer. + * The first checksum is used as a fall-back for old pools that only support + * a single checksum. + */ public HttpProtocolInfo(String protocol, int major, int minor, InetSocketAddress clientSocketAddress, String httpDoorCellName, @@ -76,7 +90,8 @@ public HttpProtocolInfo(String protocol, int major, int minor, String path, URI location, Disposition disposition, - ChecksumType wantedChecksum) { + @Nonnull + List wantedChecksums) { _name = protocol; _minor = minor; _major = major; @@ -86,7 +101,8 @@ public HttpProtocolInfo(String protocol, int major, int minor, this.path = path; _location = location; _disposition = disposition; - _wantedChecksum = wantedChecksum; + _wantedChecksum = wantedChecksums.isEmpty() ? null : wantedChecksums.get(0); + _wantedChecksums = Set.copyOf(wantedChecksums); } public String getHttpDoorCellName() { @@ -109,8 +125,8 @@ public void setSessionId(int sessionId) { _sessionId = sessionId; } - public Optional getWantedChecksum() { - return Optional.ofNullable(_wantedChecksum); + public Set getWantedChecksums() { + return _wantedChecksums; } // @@ -197,6 +213,18 @@ public URI getLocation() { public Disposition getDisposition() { return _disposition != null ? _disposition : Disposition.ATTACHMENT; } + + private void readObject(java.io.ObjectInputStream stream) + throws ClassNotFoundException, IOException { + stream.defaultReadObject(); + + // Handle objects sent from old doors. + if (_wantedChecksums == null) { + _wantedChecksums = _wantedChecksum == null + ? Collections.emptySet() + : Set.of(_wantedChecksum); + } + } } diff --git a/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpDataTransferProtocolInfo.java b/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpDataTransferProtocolInfo.java index 724cacc519a..faa20ca1905 100644 --- a/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpDataTransferProtocolInfo.java +++ b/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpDataTransferProtocolInfo.java @@ -3,9 +3,14 @@ import static java.util.Objects.requireNonNull; import com.google.common.collect.ImmutableMap; +import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; -import java.util.Optional; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.dcache.auth.OpenIdCredential; import org.dcache.util.ChecksumType; @@ -24,22 +29,36 @@ public class RemoteHttpDataTransferProtocolInfo implements IpProtocolInfo { private final boolean isVerificationRequired; private final ImmutableMap headers; private final OpenIdCredential openIdCredential; + @Nullable private final ChecksumType desiredChecksum; + @Nonnull + private Set desiredChecksums; private static final long serialVersionUID = 4482469147378465931L; + /** + * @param desiredChecksums Desired checksum to calculate for this transfer. + * The first checksum is used as a fall-back for old pools that only support + * a single checksum. + */ public RemoteHttpDataTransferProtocolInfo(String protocol, int major, int minor, InetSocketAddress addr, String url, boolean isVerificationRequired, ImmutableMap headers, - Optional desiredChecksum) { - this(protocol, minor, major, addr, url, isVerificationRequired, headers, desiredChecksum, - null); + List desiredChecksums) { + this(protocol, minor, major, addr, url, isVerificationRequired, headers, + desiredChecksums, null); } + /** + * @param desiredChecksums Desired checksum to calculate for this transfer. + * The first checksum is used as a fall-back for old pools that only support + * a single checksum. + */ public RemoteHttpDataTransferProtocolInfo(String protocol, int major, int minor, InetSocketAddress addr, String url, boolean isVerificationRequired, ImmutableMap headers, - Optional desiredChecksum, + @Nonnull + List desiredChecksums, OpenIdCredential openIdCredential) { this.name = protocol; this.minor = minor; @@ -49,7 +68,8 @@ public RemoteHttpDataTransferProtocolInfo(String protocol, int major, this.isVerificationRequired = isVerificationRequired; this.headers = requireNonNull(headers); this.openIdCredential = openIdCredential; - this.desiredChecksum = desiredChecksum.orElse(null); + this.desiredChecksum = desiredChecksums.isEmpty() ? null : desiredChecksums.get(0); + this.desiredChecksums = Set.copyOf(desiredChecksums); } public URI getUri() { @@ -102,7 +122,19 @@ public boolean hasTokenCredential() { return openIdCredential != null; } - public Optional getDesiredChecksum() { - return Optional.ofNullable(desiredChecksum); + public Set getDesiredChecksums() { + return desiredChecksums; + } + + private void readObject(java.io.ObjectInputStream stream) + throws ClassNotFoundException, IOException { + stream.defaultReadObject(); + + // Handle objects sent from old doors. + if (desiredChecksums == null) { + desiredChecksums = desiredChecksum == null + ? Collections.emptySet() + : Set.of(desiredChecksum); + } } } diff --git a/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpsDataTransferProtocolInfo.java b/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpsDataTransferProtocolInfo.java index b6795a73eda..e24ec389ee3 100644 --- a/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpsDataTransferProtocolInfo.java +++ b/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpsDataTransferProtocolInfo.java @@ -7,11 +7,12 @@ import java.security.KeyStoreException; import java.security.PrivateKey; import java.security.cert.X509Certificate; -import java.util.Optional; +import java.util.List; import org.dcache.auth.OpenIdCredential; import org.dcache.util.ChecksumType; import static java.util.Objects.requireNonNull; +import javax.annotation.Nonnull; /** * Provides information for HTTP transfer of a file using SSL/TLS encryption. @@ -23,34 +24,53 @@ public class RemoteHttpsDataTransferProtocolInfo extends RemoteHttpDataTransferP private final PrivateKey key; private final X509Certificate[] certChain; + /** + * @param desiredChecksums Desired checksum to calculate for this transfer. + * The first checksum is used as a fall-back for old pools that only support + * a single checksum. + */ public RemoteHttpsDataTransferProtocolInfo(String protocol, int major, int minor, InetSocketAddress addr, String url, boolean isVerificationRequired, ImmutableMap headers, X509Credential credential, - Optional desiredChecksum) { + @Nonnull + List desiredChecksums) { this(protocol, major, minor, addr, url, isVerificationRequired, headers, credential == null ? null : credential.getKey(), credential == null ? null : credential.getCertificateChain(), - desiredChecksum); + desiredChecksums); } + /** + * @param desiredChecksums Desired checksum to calculate for this transfer. + * The first checksum is used as a fall-back for old pools that only support + * a single checksum. + */ public RemoteHttpsDataTransferProtocolInfo(String protocol, int major, int minor, InetSocketAddress addr, String url, boolean isVerificationRequired, ImmutableMap headers, PrivateKey privateKey, X509Certificate[] certificateChain, - Optional desiredChecksum) { - super(protocol, major, minor, addr, url, isVerificationRequired, headers, desiredChecksum); + @Nonnull + List desiredChecksums) { + super(protocol, major, minor, addr, url, isVerificationRequired, headers, + desiredChecksums); key = privateKey; certChain = certificateChain; } + /** + * @param desiredChecksums Desired checksum to calculate for this transfer. + * The first checksum is used as a fall-back for old pools that only support + * a single checksum. + */ public RemoteHttpsDataTransferProtocolInfo(String protocol, int major, int minor, InetSocketAddress addr, String url, boolean isVerificationRequired, ImmutableMap headers, - Optional desiredChecksum, + @Nonnull + List desiredChecksums, OpenIdCredential token) { - super(protocol, major, minor, addr, url, isVerificationRequired, headers, desiredChecksum, - token); + super(protocol, major, minor, addr, url, isVerificationRequired, headers, + desiredChecksums, token); key = null; certChain = null; } diff --git a/modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java b/modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java index c9f966f38ad..ad2136fb2be 100644 --- a/modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java +++ b/modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java @@ -1696,6 +1696,9 @@ public HttpTransfer(PnfsHandler pnfs, Subject subject, } protected ProtocolInfo createProtocolInfo(InetSocketAddress address) { + List wantedChecksums = _wantedChecksum == null + ? Collections.emptyList() + : List.of(_wantedChecksum); HttpProtocolInfo protocolInfo = new HttpProtocolInfo( _isSSL ? PROTOCOL_INFO_SSL_NAME : PROTOCOL_INFO_NAME, @@ -1706,7 +1709,7 @@ protected ProtocolInfo createProtocolInfo(InetSocketAddress address) { _requestPath, _location, _disposition, - _wantedChecksum); + wantedChecksums); protocolInfo.setSessionId((int) getId()); return protocolInfo; } diff --git a/modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/RemoteTransferHandler.java b/modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/RemoteTransferHandler.java index fa5b1765433..3e75f221c2c 100644 --- a/modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/RemoteTransferHandler.java +++ b/modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/RemoteTransferHandler.java @@ -151,6 +151,7 @@ import static diskCacheV111.services.TransferManagerHandler.RECEIVED_FIRST_POOL_REPLY_STATE; import static dmg.util.CommandException.checkCommand; +import java.util.Collections; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.summingInt; @@ -1030,6 +1031,9 @@ private IpProtocolInfo buildProtocolInfo() throws ErrorResponseException { Optional desiredChecksum = _wantDigest.flatMap( Checksums::parseWantDigest); + var desiredChecksums = desiredChecksum + .map(List::of) + .orElseGet(Collections::emptyList); switch (_type) { case GSIFTP: @@ -1042,20 +1046,21 @@ private IpProtocolInfo buildProtocolInfo() throws ErrorResponseException { return new RemoteHttpDataTransferProtocolInfo("RemoteHttpDataTransfer", 1, 1, address, _destination.toASCIIString(), _flags.contains(TransferFlag.REQUIRE_VERIFICATION), - _transferHeaders, desiredChecksum); + _transferHeaders, desiredChecksums); case HTTPS: if (_source == CredentialSource.OIDC) { return new RemoteHttpsDataTransferProtocolInfo("RemoteHttpsDataTransfer", 1, 1, address, _destination.toASCIIString(), _flags.contains(TransferFlag.REQUIRE_VERIFICATION), - _transferHeaders, desiredChecksum, _oidCredential); + _transferHeaders, desiredChecksums, + _oidCredential); } else { return new RemoteHttpsDataTransferProtocolInfo("RemoteHttpsDataTransfer", 1, 1, address, _destination.toASCIIString(), _flags.contains(TransferFlag.REQUIRE_VERIFICATION), _transferHeaders, _privateKey, _certificateChain, - desiredChecksum); + desiredChecksums); } } diff --git a/modules/dcache/src/main/java/diskCacheV111/doors/CopyManager.java b/modules/dcache/src/main/java/diskCacheV111/doors/CopyManager.java index 129c7c25e1c..d3fb3635e1e 100644 --- a/modules/dcache/src/main/java/diskCacheV111/doors/CopyManager.java +++ b/modules/dcache/src/main/java/diskCacheV111/doors/CopyManager.java @@ -26,6 +26,7 @@ import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayDeque; +import java.util.Collections; import java.util.Map; import java.util.Optional; import java.util.Queue; @@ -460,7 +461,7 @@ private RemoteHttpDataTransferProtocolInfo createTargetProtocolInfo(String urlRe urlRemote, false, ImmutableMap.of(), - Optional.empty()); + Collections.emptyList()); } diff --git a/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java b/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java index 3208ee7d039..726d3c2ec2d 100644 --- a/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java +++ b/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java @@ -521,7 +521,7 @@ protected ChannelFuture doOnPut(ChannelHandlerContext context, HttpRequest reque file.truncate(file.getFileAttributes().getSize()); } - file.getProtocolInfo().getWantedChecksum().ifPresent(file::addChecksumType); + file.getProtocolInfo().getWantedChecksums().forEach(file::addChecksumType); _wantedDigest = wantDigest(request).flatMap(Checksums::parseWantDigest); _wantedDigest.ifPresent(file::addChecksumType); diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java b/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java index 8fe1b2949b3..47d5f7eda11 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java @@ -238,7 +238,7 @@ public void runIO(FileAttributes attributes, RepositoryChannel channel, _channel = new MoverChannel<>(access, attributes, info, channel); channel.optionallyAs(ChecksumChannel.class).ifPresent(c -> { - info.getDesiredChecksum().ifPresent(t -> { + info.getDesiredChecksums().forEach(t -> { try { c.addType(t); } catch (IOException e) {