vehicles: support door requesting multiple checksums

Motivation:

A client that is uploading a file or initiating an HTTP TPC pull
transfer may wish to validate the transfer did not result in data
corruption, using the new file's checksum / digest value.  When
receiving a file, the pool will calculate a set of checksums (from
different checksum algorithms) based on the pool's configuration.  This
configured list of checksum algorithms might not include the client's
desired checksum algorithm.

One solution would be for the pool configuration to be updated, to
include the client's desired checksum.  Although technically possible,
this is impractical, as it would require the person transferring the
data to negotiate with the dCache admins, asking them to update the pool
configuration for all pools their transfer might use.  Such support
operations are undesirable.

As an alternative approach, the WebDAV door accepts a `Want-Digest` HTTP
request header on PUT and COPY (HTTP-TPC) requests.  The door uses this
header to select a single desired checksum algorithm.  This algorithm
(if provided) is transferred to the pool, which then ensures that this
algorithm is calculated during the file upload.

A dCache instance may provide storage to multiple communities, with
different conventions on which algorithm is use for data integrety.  A
client initiating a transfer between these communities (using dCache as
an intermediate) would require two checksums: one to verify data
integrety of the transfer to dCache and another to validate transferred
within the second community.

Modification:

The ProtocolInfo subclasses are updated to carry a collection of
algorithms, taking care to maintain backwards compatibility.

Result:

No user- or admin observable change, but dCache now supports a door
requesting multiple checksum algorithms when a pool receives a file.

Target: master
Request: 10.2
Requires-notes: yes
Requires-book: no
Patch: https://rb.dcache.org/r/14341/
Acked-by: Tigran Mkrtchyan

modules/dcache-srm/src/main/java/diskCacheV111/srm/dcache/Storage.java

@@ -151,6 +151,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -1784,15 +1785,15 @@ private String performRemoteTransfer(SRMUser srmUser,
                       remoteTURL.toString(), isVerifyRequired(extraInfo),
                       httpHeaders(extraInfo),
                       credential,
-                      Optional.empty());
+                      Collections.emptyList());
                 break;
 
             case "http":
                 protocolInfo = new RemoteHttpDataTransferProtocolInfo("RemoteHttpDataTransfer",
                       1, 1, remoteAddr,
                       remoteTURL.toString(), isVerifyRequired(extraInfo),
                       httpHeaders(extraInfo),
-                      Optional.empty());
+                      Collections.emptyList());
                 break;
 
             default:

modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/HttpProtocolInfo.java

@@ -1,8 +1,13 @@
 package diskCacheV111.vehicles;
 
+import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.URI;
-import java.util.Optional;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import org.dcache.util.ChecksumType;
 
 /**
@@ -42,7 +47,10 @@ public enum Disposition {
     private final String httpDoorDomainName;
     private final String path;
     private final URI _location;
+    @Nullable
     private final ChecksumType _wantedChecksum;
+    @Nonnull
+    private Set<ChecksumType> _wantedChecksums;
 
     private final Disposition _disposition;
 
@@ -55,7 +63,7 @@ public HttpProtocolInfo(String protocol, int major, int minor,
           String path,
           URI location) {
         this(protocol, major, minor, clientSocketAddress, httpDoorCellName,
-              httpDoorDomainName, path, location, null, null);
+              httpDoorDomainName, path, location, null, Collections.emptyList());
     }
 
     public HttpProtocolInfo(String protocol, int major, int minor,
@@ -66,17 +74,24 @@ public HttpProtocolInfo(String protocol, int major, int minor,
           URI location,
           Disposition disposition) {
         this(protocol, major, minor, clientSocketAddress, httpDoorCellName,
-              httpDoorDomainName, path, location, disposition, null);
+              httpDoorDomainName, path, location, disposition,
+              Collections.emptyList());
     }
 
+    /**
+     * @param wantedChecksums Desired checksum to calculate for this transfer.
+     * The first checksum is used as a fall-back for old pools that only support
+     * a single checksum.
+     */
     public HttpProtocolInfo(String protocol, int major, int minor,
           InetSocketAddress clientSocketAddress,
           String httpDoorCellName,
           String httpDoorDomainName,
           String path,
           URI location,
           Disposition disposition,
-          ChecksumType wantedChecksum) {
+          @Nonnull
+          List<ChecksumType> wantedChecksums) {
         _name = protocol;
         _minor = minor;
         _major = major;
@@ -86,7 +101,8 @@ public HttpProtocolInfo(String protocol, int major, int minor,
         this.path = path;
         _location = location;
         _disposition = disposition;
-        _wantedChecksum = wantedChecksum;
+        _wantedChecksum = wantedChecksums.isEmpty() ? null : wantedChecksums.get(0);
+        _wantedChecksums = Set.copyOf(wantedChecksums);
     }
 
     public String getHttpDoorCellName() {
@@ -109,8 +125,8 @@ public void setSessionId(int sessionId) {
         _sessionId = sessionId;
     }
 
-    public Optional<ChecksumType> getWantedChecksum() {
-        return Optional.ofNullable(_wantedChecksum);
+    public Set<ChecksumType> getWantedChecksums() {
+        return _wantedChecksums;
     }
 
     //
@@ -197,6 +213,18 @@ public URI getLocation() {
     public Disposition getDisposition() {
         return _disposition != null ? _disposition : Disposition.ATTACHMENT;
     }
+
+    private void readObject(java.io.ObjectInputStream stream)
+          throws ClassNotFoundException, IOException {
+        stream.defaultReadObject();
+
+        // Handle objects sent from old doors.
+        if (_wantedChecksums == null) {
+            _wantedChecksums = _wantedChecksum == null
+                    ? Collections.emptySet()
+                    : Set.of(_wantedChecksum);
+        }
+    }
 }
 
 

modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpDataTransferProtocolInfo.java

@@ -3,9 +3,14 @@
 import static java.util.Objects.requireNonNull;
 
 import com.google.common.collect.ImmutableMap;
+import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.URI;
-import java.util.Optional;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import org.dcache.auth.OpenIdCredential;
 import org.dcache.util.ChecksumType;
 
@@ -24,22 +29,36 @@ public class RemoteHttpDataTransferProtocolInfo implements IpProtocolInfo {
     private final boolean isVerificationRequired;
     private final ImmutableMap<String, String> headers;
     private final OpenIdCredential openIdCredential;
+    @Nullable
     private final ChecksumType desiredChecksum;
+    @Nonnull
+    private Set<ChecksumType> desiredChecksums;
 
     private static final long serialVersionUID = 4482469147378465931L;
 
+    /**
+     * @param desiredChecksums Desired checksum to calculate for this transfer.
+     * The first checksum is used as a fall-back for old pools that only support
+     * a single checksum.
+     */
     public RemoteHttpDataTransferProtocolInfo(String protocol, int major,
           int minor, InetSocketAddress addr, String url,
           boolean isVerificationRequired, ImmutableMap<String, String> headers,
-          Optional<ChecksumType> desiredChecksum) {
-        this(protocol, minor, major, addr, url, isVerificationRequired, headers, desiredChecksum,
-              null);
+          List<ChecksumType> desiredChecksums) {
+        this(protocol, minor, major, addr, url, isVerificationRequired, headers,
+                desiredChecksums, null);
     }
 
+    /**
+     * @param desiredChecksums Desired checksum to calculate for this transfer.
+     * The first checksum is used as a fall-back for old pools that only support
+     * a single checksum.
+     */
     public RemoteHttpDataTransferProtocolInfo(String protocol, int major,
           int minor, InetSocketAddress addr, String url,
           boolean isVerificationRequired, ImmutableMap<String, String> headers,
-          Optional<ChecksumType> desiredChecksum,
+          @Nonnull
+          List<ChecksumType> desiredChecksums,
           OpenIdCredential openIdCredential) {
         this.name = protocol;
         this.minor = minor;
@@ -49,7 +68,8 @@ public RemoteHttpDataTransferProtocolInfo(String protocol, int major,
         this.isVerificationRequired = isVerificationRequired;
         this.headers = requireNonNull(headers);
         this.openIdCredential = openIdCredential;
-        this.desiredChecksum = desiredChecksum.orElse(null);
+        this.desiredChecksum = desiredChecksums.isEmpty() ? null : desiredChecksums.get(0);
+        this.desiredChecksums = Set.copyOf(desiredChecksums);
     }
 
     public URI getUri() {
@@ -102,7 +122,19 @@ public boolean hasTokenCredential() {
         return openIdCredential != null;
     }
 
-    public Optional<ChecksumType> getDesiredChecksum() {
-        return Optional.ofNullable(desiredChecksum);
+    public Set<ChecksumType> getDesiredChecksums() {
+        return desiredChecksums;
+    }
+
+    private void readObject(java.io.ObjectInputStream stream)
+          throws ClassNotFoundException, IOException {
+        stream.defaultReadObject();
+
+        // Handle objects sent from old doors.
+        if (desiredChecksums == null) {
+            desiredChecksums = desiredChecksum == null
+                    ? Collections.emptySet()
+                    : Set.of(desiredChecksum);
+        }
     }
 }

modules/dcache-vehicles/src/main/java/diskCacheV111/vehicles/RemoteHttpsDataTransferProtocolInfo.java

@@ -7,11 +7,12 @@
 import java.security.KeyStoreException;
 import java.security.PrivateKey;
 import java.security.cert.X509Certificate;
-import java.util.Optional;
+import java.util.List;
 import org.dcache.auth.OpenIdCredential;
 import org.dcache.util.ChecksumType;
 
 import static java.util.Objects.requireNonNull;
+import javax.annotation.Nonnull;
 
 /**
  * Provides information for HTTP transfer of a file using SSL/TLS encryption.
@@ -23,34 +24,53 @@ public class RemoteHttpsDataTransferProtocolInfo extends RemoteHttpDataTransferP
     private final PrivateKey key;
     private final X509Certificate[] certChain;
 
+    /**
+     * @param desiredChecksums Desired checksum to calculate for this transfer.
+     * The first checksum is used as a fall-back for old pools that only support
+     * a single checksum.
+     */
     public RemoteHttpsDataTransferProtocolInfo(String protocol, int major,
           int minor, InetSocketAddress addr, String url,
           boolean isVerificationRequired, ImmutableMap<String, String> headers,
           X509Credential credential,
-          Optional<ChecksumType> desiredChecksum) {
+          @Nonnull
+          List<ChecksumType> desiredChecksums) {
         this(protocol, major, minor, addr, url, isVerificationRequired,
               headers, credential == null ? null : credential.getKey(),
               credential == null ? null : credential.getCertificateChain(),
-              desiredChecksum);
+              desiredChecksums);
     }
 
+    /**
+     * @param desiredChecksums Desired checksum to calculate for this transfer.
+     * The first checksum is used as a fall-back for old pools that only support
+     * a single checksum.
+     */
     public RemoteHttpsDataTransferProtocolInfo(String protocol, int major,
           int minor, InetSocketAddress addr, String url,
           boolean isVerificationRequired, ImmutableMap<String, String> headers,
           PrivateKey privateKey, X509Certificate[] certificateChain,
-          Optional<ChecksumType> desiredChecksum) {
-        super(protocol, major, minor, addr, url, isVerificationRequired, headers, desiredChecksum);
+          @Nonnull
+          List<ChecksumType> desiredChecksums) {
+        super(protocol, major, minor, addr, url, isVerificationRequired, headers,
+                desiredChecksums);
         key = privateKey;
         certChain = certificateChain;
     }
 
+    /**
+     * @param desiredChecksums Desired checksum to calculate for this transfer.
+     * The first checksum is used as a fall-back for old pools that only support
+     * a single checksum.
+     */
     public RemoteHttpsDataTransferProtocolInfo(String protocol, int major,
           int minor, InetSocketAddress addr, String url,
           boolean isVerificationRequired, ImmutableMap<String, String> headers,
-          Optional<ChecksumType> desiredChecksum,
+          @Nonnull
+          List<ChecksumType> desiredChecksums,
           OpenIdCredential token) {
-        super(protocol, major, minor, addr, url, isVerificationRequired, headers, desiredChecksum,
-              token);
+        super(protocol, major, minor, addr, url, isVerificationRequired, headers,
+                desiredChecksums, token);
         key = null;
         certChain = null;
     }

modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java

@@ -1696,6 +1696,9 @@ public HttpTransfer(PnfsHandler pnfs, Subject subject,
         }
 
         protected ProtocolInfo createProtocolInfo(InetSocketAddress address) {
+            List<ChecksumType> wantedChecksums = _wantedChecksum == null
+                    ? Collections.emptyList()
+                    : List.of(_wantedChecksum);
             HttpProtocolInfo protocolInfo =
                   new HttpProtocolInfo(
                         _isSSL ? PROTOCOL_INFO_SSL_NAME : PROTOCOL_INFO_NAME,
@@ -1706,7 +1709,7 @@ protected ProtocolInfo createProtocolInfo(InetSocketAddress address) {
                         _requestPath,
                         _location,
                         _disposition,
-                        _wantedChecksum);
+                        wantedChecksums);
             protocolInfo.setSessionId((int) getId());
             return protocolInfo;
         }

modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/RemoteTransferHandler.java

@@ -151,6 +151,7 @@
 
 import static diskCacheV111.services.TransferManagerHandler.RECEIVED_FIRST_POOL_REPLY_STATE;
 import static dmg.util.CommandException.checkCommand;
+import java.util.Collections;
 import static java.util.stream.Collectors.groupingBy;
 import static java.util.stream.Collectors.summingInt;
 
@@ -1030,6 +1031,9 @@ private IpProtocolInfo buildProtocolInfo() throws ErrorResponseException {
 
             Optional<ChecksumType> desiredChecksum = _wantDigest.flatMap(
                   Checksums::parseWantDigest);
+            var desiredChecksums = desiredChecksum
+                    .map(List::of)
+                    .orElseGet(Collections::emptyList);
 
             switch (_type) {
                 case GSIFTP:
@@ -1042,20 +1046,21 @@ private IpProtocolInfo buildProtocolInfo() throws ErrorResponseException {
                     return new RemoteHttpDataTransferProtocolInfo("RemoteHttpDataTransfer",
                           1, 1, address, _destination.toASCIIString(),
                           _flags.contains(TransferFlag.REQUIRE_VERIFICATION),
-                          _transferHeaders, desiredChecksum);
+                          _transferHeaders, desiredChecksums);
 
                 case HTTPS:
                     if (_source == CredentialSource.OIDC) {
                         return new RemoteHttpsDataTransferProtocolInfo("RemoteHttpsDataTransfer",
                               1, 1, address, _destination.toASCIIString(),
                               _flags.contains(TransferFlag.REQUIRE_VERIFICATION),
-                              _transferHeaders, desiredChecksum, _oidCredential);
+                              _transferHeaders, desiredChecksums,
+                              _oidCredential);
                     } else {
                         return new RemoteHttpsDataTransferProtocolInfo("RemoteHttpsDataTransfer",
                               1, 1, address, _destination.toASCIIString(),
                               _flags.contains(TransferFlag.REQUIRE_VERIFICATION),
                               _transferHeaders, _privateKey, _certificateChain,
-                              desiredChecksum);
+                              desiredChecksums);
                     }
             }
 

modules/dcache/src/main/java/diskCacheV111/doors/CopyManager.java

@@ -26,6 +26,7 @@
 import java.net.InetSocketAddress;
 import java.net.URI;
 import java.util.ArrayDeque;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Queue;
@@ -460,7 +461,7 @@ private RemoteHttpDataTransferProtocolInfo createTargetProtocolInfo(String urlRe
                   urlRemote,
                   false,
                   ImmutableMap.of(),
-                  Optional.empty());
+                  Collections.emptyList());
 
         }
 

modules/dcache/src/main/java/org/dcache/http/HttpPoolRequestHandler.java

@@ -521,7 +521,7 @@ protected ChannelFuture doOnPut(ChannelHandlerContext context, HttpRequest reque
                 file.truncate(file.getFileAttributes().getSize());
             }
 
-            file.getProtocolInfo().getWantedChecksum().ifPresent(file::addChecksumType);
+            file.getProtocolInfo().getWantedChecksums().forEach(file::addChecksumType);
             _wantedDigest = wantDigest(request).flatMap(Checksums::parseWantDigest);
             _wantedDigest.ifPresent(file::addChecksumType);
 

modules/dcache/src/main/java/org/dcache/pool/movers/RemoteHttpDataTransferProtocol.java

@@ -238,7 +238,7 @@ public void runIO(FileAttributes attributes, RepositoryChannel channel,
         _channel = new MoverChannel<>(access, attributes, info, channel);
 
         channel.optionallyAs(ChecksumChannel.class).ifPresent(c -> {
-            info.getDesiredChecksum().ifPresent(t -> {
+            info.getDesiredChecksums().forEach(t -> {
                 try {
                     c.addType(t);
                 } catch (IOException e) {