From 9023ceaee4a9323c8d5b2b84901462a2c6a26f14 Mon Sep 17 00:00:00 2001 From: Paul Millar Date: Fri, 25 Oct 2024 10:55:14 +0200 Subject: [PATCH] vehicles: support door requesting multiple checksums Motivation: A client that is uploading a file or initiating an HTTP TPC pull transfer may wish to validate the transfer did not result in data corruption, using the new file's checksum / digest value. When receiving a file, the pool will calculate a set of checksums (from different checksum algorithms) based on the pool's configuration. This configured list of checksum algorithms might not include the client's desired checksum algorithm. One solution would be for the pool configuration to be updated, to include the client's desired checksum. Although technically possible, this is impractical, as it would require the person transferring the data to negotiate with the dCache admins, asking them to update the pool configuration for all pools their transfer might use. Such support operations are undesirable. As an alternative approach, the WebDAV door accepts a `Want-Digest` HTTP request header on PUT and COPY (HTTP-TPC) requests. The door uses this header to select a single desired checksum algorithm. This algorithm (if provided) is transferred to the pool, which then ensures that this algorithm is calculated during the file upload. A dCache instance may provide storage to multiple communities, with different conventions on which algorithm is use for data integrety. A client initiating a transfer between these communities (using dCache as an intermediate) would require two checksums: one to verify data integrety of the transfer to dCache and another to validate transferred within the second community. Modification: The ProtocolInfo subclasses are updated to carry a collection of algorithms, taking care to maintain backwards compatibility. Result: No user- or admin observable change, but dCache now supports a door requesting multiple checksum algorithms when a pool receives a file. Target: master Request: 10.2 Requires-notes: yes Requires-book: no Patch: https://rb.dcache.org/r/14341/ Acked-by: Tigran Mkrtchyan --- .../diskCacheV111/srm/dcache/Storage.java | 5 +- .../vehicles/HttpProtocolInfo.java | 42 +++++++++++++--- .../RemoteHttpDataTransferProtocolInfo.java | 48 +++++++++++++++---- .../RemoteHttpsDataTransferProtocolInfo.java | 36 ++++++++++---- .../dcache/webdav/DcacheResourceFactory.java | 5 +- .../transfer/RemoteTransferHandler.java | 11 +++-- .../java/diskCacheV111/doors/CopyManager.java | 3 +- .../dcache/http/HttpPoolRequestHandler.java | 2 +- .../RemoteHttpDataTransferProtocol.java | 2 +- 9 files changed, 122 insertions(+), 32 deletions(-) diff --git a/modules/dcache-srm/src/main/java/diskCacheV111/srm/dcache/Storage.java b/modules/dcache-srm/src/main/java/diskCacheV111/srm/dcache/Storage.java index bb3132bd30e..05409d0487b 100644 --- a/modules/dcache-srm/src/main/java/diskCacheV111/srm/dcache/Storage.java +++ b/modules/dcache-srm/src/main/java/diskCacheV111/srm/dcache/Storage.java @@ -151,6 +151,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; @@ -1784,7 +1785,7 @@ private String performRemoteTransfer(SRMUser srmUser, remoteTURL.toString(), isVerifyRequired(extraInfo), httpHeaders(extraInfo), credential, - Optional.empty()); + Collections.emptyList()); break; case "http": @@ -1792,7 +1793,7 @@ private String performRemoteTransfer(SRMUser srmUser, 1, 1, remoteAddr, remoteTURL.toString(), isVerifyRequired(extraInfo), httpHeaders(extraInfo), - Optional.empty()); + Collections.emptyList()); break; default: diff --git a/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/HttpProtocolInfo.java b/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/HttpProtocolInfo.java index 8e1cd76d557..ae4aa13ab2f 100644 --- a/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/HttpProtocolInfo.java +++ b/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/HttpProtocolInfo.java @@ -1,8 +1,13 @@ package diskCacheV111.vehicles; +import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; -import java.util.Optional; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.dcache.util.ChecksumType; /** @@ -42,7 +47,10 @@ public enum Disposition { private final String httpDoorDomainName; private final String path; private final URI _location; + @Nullable private final ChecksumType _wantedChecksum; + @Nonnull + private Set _wantedChecksums; private final Disposition _disposition; @@ -55,7 +63,7 @@ public HttpProtocolInfo(String protocol, int major, int minor, String path, URI location) { this(protocol, major, minor, clientSocketAddress, httpDoorCellName, - httpDoorDomainName, path, location, null, null); + httpDoorDomainName, path, location, null, Collections.emptyList()); } public HttpProtocolInfo(String protocol, int major, int minor, @@ -66,9 +74,15 @@ public HttpProtocolInfo(String protocol, int major, int minor, URI location, Disposition disposition) { this(protocol, major, minor, clientSocketAddress, httpDoorCellName, - httpDoorDomainName, path, location, disposition, null); + httpDoorDomainName, path, location, disposition, + Collections.emptyList()); } + /** + * @param wantedChecksums Desired checksum to calculate for this transfer. + * The first checksum is used as a fall-back for old pools that only support + * a single checksum. + */ public HttpProtocolInfo(String protocol, int major, int minor, InetSocketAddress clientSocketAddress, String httpDoorCellName, @@ -76,7 +90,8 @@ public HttpProtocolInfo(String protocol, int major, int minor, String path, URI location, Disposition disposition, - ChecksumType wantedChecksum) { + @Nonnull + List wantedChecksums) { _name = protocol; _minor = minor; _major = major; @@ -86,7 +101,8 @@ public HttpProtocolInfo(String protocol, int major, int minor, this.path = path; _location = location; _disposition = disposition; - _wantedChecksum = wantedChecksum; + _wantedChecksum = wantedChecksums.isEmpty() ? null : wantedChecksums.get(0); + _wantedChecksums = Set.copyOf(wantedChecksums); } public String getHttpDoorCellName() { @@ -109,8 +125,8 @@ public void setSessionId(int sessionId) { _sessionId = sessionId; } - public Optional getWantedChecksum() { - return Optional.ofNullable(_wantedChecksum); + public Set getWantedChecksums() { + return _wantedChecksums; } // @@ -197,6 +213,18 @@ public URI getLocation() { public Disposition getDisposition() { return _disposition != null ? _disposition : Disposition.ATTACHMENT; } + + private void readObject(java.io.ObjectInputStream stream) + throws ClassNotFoundException, IOException { + stream.defaultReadObject(); + + // Handle objects sent from old doors. + if (_wantedChecksums == null) { + _wantedChecksums = _wantedChecksum == null + ? Collections.emptySet() + : Set.of(_wantedChecksum); + } + } } diff --git a/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpDataTransferProtocolInfo.java b/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpDataTransferProtocolInfo.java index 724cacc519a..faa20ca1905 100644 --- a/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpDataTransferProtocolInfo.java +++ b/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpDataTransferProtocolInfo.java @@ -3,9 +3,14 @@ import static java.util.Objects.requireNonNull; import com.google.common.collect.ImmutableMap; +import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; -import java.util.Optional; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.dcache.auth.OpenIdCredential; import org.dcache.util.ChecksumType; @@ -24,22 +29,36 @@ public class RemoteHttpDataTransferProtocolInfo implements IpProtocolInfo { private final boolean isVerificationRequired; private final ImmutableMap headers; private final OpenIdCredential openIdCredential; + @Nullable private final ChecksumType desiredChecksum; + @Nonnull + private Set desiredChecksums; private static final long serialVersionUID = 4482469147378465931L; + /** + * @param desiredChecksums Desired checksum to calculate for this transfer. + * The first checksum is used as a fall-back for old pools that only support + * a single checksum. + */ public RemoteHttpDataTransferProtocolInfo(String protocol, int major, int minor, InetSocketAddress addr, String url, boolean isVerificationRequired, ImmutableMap headers, - Optional desiredChecksum) { - this(protocol, minor, major, addr, url, isVerificationRequired, headers, desiredChecksum, - null); + List desiredChecksums) { + this(protocol, minor, major, addr, url, isVerificationRequired, headers, + desiredChecksums, null); } + /** + * @param desiredChecksums Desired checksum to calculate for this transfer. + * The first checksum is used as a fall-back for old pools that only support + * a single checksum. + */ public RemoteHttpDataTransferProtocolInfo(String protocol, int major, int minor, InetSocketAddress addr, String url, boolean isVerificationRequired, ImmutableMap headers, - Optional desiredChecksum, + @Nonnull + List desiredChecksums, OpenIdCredential openIdCredential) { this.name = protocol; this.minor = minor; @@ -49,7 +68,8 @@ public RemoteHttpDataTransferProtocolInfo(String protocol, int major, this.isVerificationRequired = isVerificationRequired; this.headers = requireNonNull(headers); this.openIdCredential = openIdCredential; - this.desiredChecksum = desiredChecksum.orElse(null); + this.desiredChecksum = desiredChecksums.isEmpty() ? null : desiredChecksums.get(0); + this.desiredChecksums = Set.copyOf(desiredChecksums); } public URI getUri() { @@ -102,7 +122,19 @@ public boolean hasTokenCredential() { return openIdCredential != null; } - public Optional getDesiredChecksum() { - return Optional.ofNullable(desiredChecksum); + public Set getDesiredChecksums() { + return desiredChecksums; + } + + private void readObject(java.io.ObjectInputStream stream) + throws ClassNotFoundException, IOException { + stream.defaultReadObject(); + + // Handle objects sent from old doors. + if (desiredChecksums == null) { + desiredChecksums = desiredChecksum == null + ? Collections.emptySet() + : Set.of(desiredChecksum); + } } } diff --git a/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpsDataTransferProtocolInfo.java b/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpsDataTransferProtocolInfo.java index b6795a73eda..e24ec389ee3 100644 --- a/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpsDataTransferProtocolInfo.java +++ b/modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpsDataTransferProtocolInfo.java @@ -7,11 +7,12 @@ import java.security.KeyStoreException; import java.security.PrivateKey; import java.security.cert.X509Certificate; -import java.util.Optional; +import java.util.List; import org.dcache.auth.OpenIdCredential; import org.dcache.util.ChecksumType; import static java.util.Objects.requireNonNull; +import javax.annotation.Nonnull; /** * Provides information for HTTP transfer of a file using SSL/TLS encryption. @@ -23,34 +24,53 @@ public class RemoteHttpsDataTransferProtocolInfo extends RemoteHttpDataTransferP private final PrivateKey key; private final X509Certificate[] certChain; + /** + * @param desiredChecksums Desired checksum to calculate for this transfer. + * The first checksum is used as a fall-back for old pools that only support + * a single checksum. + */ public RemoteHttpsDataTransferProtocolInfo(String protocol, int major, int minor, InetSocketAddress addr, String url, boolean isVerificationRequired, ImmutableMap headers, X509Credential credential, - Optional desiredChecksum) { + @Nonnull + List desiredChecksums) { this(protocol, major, minor, addr, url, isVerificationRequired, headers, credential == null ? null : credential.getKey(), credential == null ? null : credential.getCertificateChain(), - desiredChecksum); + desiredChecksums); } + /** + * @param desiredChecksums Desired checksum to calculate for this transfer. + * The first checksum is used as a fall-back for old pools that only support + * a single checksum. + */ public RemoteHttpsDataTransferProtocolInfo(String protocol, int major, int minor, InetSocketAddress addr, String url, boolean isVerificationRequired, ImmutableMap headers, PrivateKey privateKey, X509Certificate[] certificateChain, - Optional desiredChecksum) { - super(protocol, major, minor, addr, url, isVerificationRequired, headers, desiredChecksum); + @Nonnull + List desiredChecksums) { + super(protocol, major, minor, addr, url, isVerificationRequired, headers, + desiredChecksums); key = privateKey; certChain = certificateChain; } + /** + * @param desiredChecksums Desired checksum to calculate for this transfer. + * The first checksum is used as a fall-back for old pools that only support + * a single checksum. + */ public RemoteHttpsDataTransferProtocolInfo(String protocol, int major, int minor, InetSocketAddress addr, String url, boolean isVerificationRequired, ImmutableMap headers, - Optional desiredChecksum, + @Nonnull + List desiredChecksums, OpenIdCredential token) { - super(protocol, major, minor, addr, url, isVerificationRequired, headers, desiredChecksum, - token); + super(protocol, major, minor, addr, url, isVerificationRequired, headers, + desiredChecksums, token); key = null; certChain = null; } diff --git a/modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java b/modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java index c9f966f38ad..ad2136fb2be 100644 --- a/modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java +++ b/modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java @@ -1696,6 +1696,9 @@ public HttpTransfer(PnfsHandler pnfs, Subject subject, } protected ProtocolInfo createProtocolInfo(InetSocketAddress address) { + List wantedChecksums = _wantedChecksum == null + ? Collections.emptyList() + : List.of(_wantedChecksum); HttpProtocolInfo protocolInfo = new HttpProtocolInfo( _isSSL ? PROTOCOL_INFO_SSL_NAME : PROTOCOL_INFO_NAME, @@ -1706,7 +1709,7 @@ protected ProtocolInfo createProtocolInfo(InetSocketAddress address) { _requestPath, _location, _disposition, - _wantedChecksum); + wantedChecksums); protocolInfo.setSessionId((int) getId()); return protocolInfo; } diff --git a/modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/RemoteTransferHandler.java b/modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/RemoteTransferHandler.java index fa5b1765433..3e75f221c2c 100644 --- a/modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/RemoteTransferHandler.java +++ b/modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/RemoteTransferHandler.java @@ -151,6 +151,7 @@ import static diskCacheV111.services.TransferManagerHandler.RECEIVED_FIRST_POOL_REPLY_STATE; import static dmg.util.CommandException.checkCommand; +import java.util.Collections; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.summingInt; @@ -1030,6 +1031,9 @@ private IpProtocolInfo buildProtocolInfo() throws ErrorResponseException { Optional desiredChecksum = _wantDigest.flatMap( Checksums::parseWantDigest); + var desiredChecksums = desiredChecksum + .map(List::of) + .orElseGet(Collections::emptyList); switch (_type) { case GSIFTP: @@ -1042,20 +1046,21 @@ private IpProtocolInfo buildProtocolInfo() throws ErrorResponseException { return new RemoteHttpDataTransferProtocolInfo("RemoteHttpDataTransfer", 1, 1, address, _destination.toASCIIString(), _flags.contains(TransferFlag.REQUIRE_VERIFICATION), - _transferHeaders, desiredChecksum); + _transferHeaders, desiredChecksums); case HTTPS: if (_source == CredentialSource.OIDC) { return new RemoteHttpsDataTransferProtocolInfo("RemoteHttpsDataTransfer", 1, 1, address, _destination.toASCIIString(), _flags.contains(TransferFlag.REQUIRE_VERIFICATION), - _transferHeaders, desiredChecksum, _oidCredential); + _transferHeaders, desiredChecksums, + _oidCredential); } else { return new RemoteHttpsDataTransferProtocolInfo("RemoteHttpsDataTransfer", 1, 1, address, _destination.toASCIIString(), _flags.contains(TransferFlag.REQUIRE_VERIFICATION), _transferHeaders, _privateKey, _certificateChain, - desiredChecksum); + desiredChecksums); } } diff --git a/modules/dcache/src/main/java/diskCacheV111/doors/CopyManager.java b/modules/dcache/src/main/java/diskCacheV111/doors/CopyManager.java index 129c7c25e1c..d3fb3635e1e 100644 --- a/modules/dcache/src/main/java/diskCacheV111/doors/CopyManager.java +++ b/modules/dcache/src/main/java/diskCacheV111/doors/CopyManager.java @@ -26,6 +26,7 @@ import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayDeque; +import java.util.Collections; import java.util.Map; import java.util.Optional; import java.util.Queue; @@ -460,7 +461,7 @@ private RemoteHttpDataTransferProtocolInfo createTargetProtocolInfo(String urlRe urlRemote, false, ImmutableMap.of(), - Optional.empty()); + Collections.emptyList()); } diff --git a/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java b/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java index 3208ee7d039..726d3c2ec2d 100644 --- a/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java +++ b/modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java @@ -521,7 +521,7 @@ protected ChannelFuture doOnPut(ChannelHandlerContext context, HttpRequest reque file.truncate(file.getFileAttributes().getSize()); } - file.getProtocolInfo().getWantedChecksum().ifPresent(file::addChecksumType); + file.getProtocolInfo().getWantedChecksums().forEach(file::addChecksumType); _wantedDigest = wantDigest(request).flatMap(Checksums::parseWantDigest); _wantedDigest.ifPresent(file::addChecksumType); diff --git a/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java b/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java index 8fe1b2949b3..47d5f7eda11 100644 --- a/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java +++ b/modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java @@ -238,7 +238,7 @@ public void runIO(FileAttributes attributes, RepositoryChannel channel, _channel = new MoverChannel<>(access, attributes, info, channel); channel.optionallyAs(ChecksumChannel.class).ifPresent(c -> { - info.getDesiredChecksum().ifPresent(t -> { + info.getDesiredChecksums().forEach(t -> { try { c.addType(t); } catch (IOException e) {