WIP

italiangrid · Mar 19, 2024 · d61329a · d61329a
1 parent 010d2c9
commit d61329a
Show file tree

Hide file tree

Showing 22 changed files with 485 additions and 115 deletions.
diff --git a/pom.xml b/pom.xml
@@ -57,6 +57,8 @@
     <nimbus-jose-jwt.version>6.0.2</nimbus-jose-jwt.version>
     <mock-server.version>5.5.1</mock-server.version>
 
+    <jnr-posix.version>3.1.18</jnr-posix.version>
+
     <start-class>org.italiangrid.storm.webdav.WebdavService</start-class>
 
   </properties>
@@ -448,6 +450,12 @@
       <version>${commons-cli.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>com.github.jnr</groupId>
+      <artifactId>jnr-posix</artifactId>
+      <version>${jnr-posix.version}</version>
+    </dependency>
+
     <dependency>
       <groupId>org.mockito</groupId>
       <artifactId>mockito-core</artifactId>

diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/DefaultFSStrategy.java b/src/main/java/org/italiangrid/storm/webdav/fs/DefaultFSStrategy.java
@@ -15,6 +15,8 @@
  */
 package org.italiangrid.storm.webdav.fs;
 
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME;
+
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -137,7 +139,8 @@ public File create(File file, InputStream in) {
       Adler32ChecksumInputStream cis = new Adler32ChecksumInputStream(in);
 
       IOUtils.copy(cis, fos);
-      attrsHelper.setChecksumAttribute(file, cis.getChecksumValue());
+      attrsHelper.setExtendedFileAttribute(file, STORM_ADLER32_CHECKSUM_ATTR_NAME,
+          cis.getChecksumValue());
 
       return file;
 

diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/DefaultExtendedFileAttributesHelper.java b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/DefaultExtendedFileAttributesHelper.java
@@ -15,9 +15,10 @@
  */
 package org.italiangrid.storm.webdav.fs.attrs;
 
-import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Strings.isNullOrEmpty;
+import static java.util.Objects.isNull;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_MIGRATED_ATTR_NAME;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_RECALL_IN_PROGRESS_ATTR_NAME;
 
 import java.io.File;
 import java.io.IOException;
@@ -28,21 +29,43 @@
 import java.nio.file.attribute.UserDefinedFileAttributeView;
 import java.util.List;
 
-public class DefaultExtendedFileAttributesHelper implements
-  ExtendedAttributesHelper {
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-  public static final String STORM_ADLER32_CHECKSUM_ATTR_NAME = "storm.checksum.adler32";
+import jnr.posix.FileStat;
+import jnr.posix.POSIX;
+import jnr.posix.POSIXFactory;
+
+public class DefaultExtendedFileAttributesHelper implements ExtendedAttributesHelper {
+
+  public static final Logger LOG =
+      LoggerFactory.getLogger(DefaultExtendedFileAttributesHelper.class);
+
+  private static POSIX posix;
 
   public DefaultExtendedFileAttributesHelper() {
+    posix = POSIXFactory.getPOSIX();
+  }
+
+  protected UserDefinedFileAttributeView getFileAttributeView(File f) throws IOException {
+
+    UserDefinedFileAttributeView faView =
+        Files.getFileAttributeView(f.toPath(), UserDefinedFileAttributeView.class);
+
+    if (faView == null) {
+      throw new IOException(
+          "UserDefinedFileAttributeView not supported on file " + f.getAbsolutePath());
+    }
 
+    return faView;
   }
 
-  protected String getAttributeValue(UserDefinedFileAttributeView view,
-    String attributeName) throws IOException {
+  protected String getAttributeValue(UserDefinedFileAttributeView view, String name)
+      throws IOException {
 
-    if (view.list().contains(attributeName)) {
-      ByteBuffer buffer = ByteBuffer.allocateDirect(view.size(attributeName));
-      view.read(attributeName, buffer);
+    if (view.list().contains(name)) {
+      ByteBuffer buffer = ByteBuffer.allocateDirect(view.size(name));
+      view.read(name, buffer);
       buffer.flip();
       return StandardCharsets.UTF_8.decode(buffer).toString();
     } else {
@@ -51,98 +74,128 @@ protected String getAttributeValue(UserDefinedFileAttributeView view,
   }
 
   @Override
-  public void setExtendedFileAttribute(File f, String attributeName,
-    String attributeValue) throws IOException {
+  public void setExtendedFileAttribute(Path p, ExtendedAttributes name, String value)
+      throws IOException {
+    setExtendedFileAttribute(p.toFile(), name, value);
+  }
+
+  @Override
+  public void setExtendedFileAttribute(File f, ExtendedAttributes name, String value)
+      throws IOException {
 
     checkNotNull(f);
-    checkArgument(!isNullOrEmpty(attributeName));
 
-    UserDefinedFileAttributeView faView = Files.getFileAttributeView(
-      f.toPath(), UserDefinedFileAttributeView.class);
+    UserDefinedFileAttributeView faView = getFileAttributeView(f);
 
-    if (faView == null) {
-      throw new IOException(
-        "UserDefinedFileAttributeView not supported on file "
-          + f.getAbsolutePath());
+    int bytes = faView.write(name.toString(), StandardCharsets.UTF_8.encode(value));
+    if (bytes > 0) {
+      LOG.debug("Setting '{}' extended attribute on file '{}': {} bytes written", name.toString(),
+          f.getAbsolutePath(), bytes);
     }
-
-    faView.write(attributeName, StandardCharsets.UTF_8.encode(attributeValue));
   }
 
   @Override
-  public String getExtendedFileAttributeValue(File f, String attributeName)
-    throws IOException {
+  public String getExtendedFileAttributeValue(File f, ExtendedAttributes attributeName)
+      throws IOException {
 
     checkNotNull(f);
-    checkArgument(!isNullOrEmpty(attributeName));
-
-    UserDefinedFileAttributeView faView = Files.getFileAttributeView(
-      f.toPath(), UserDefinedFileAttributeView.class);
 
-    if (faView == null) {
-      throw new IOException(
-        "UserDefinedFileAttributeView not supported on file "
-          + f.getAbsolutePath());
-    }
+    UserDefinedFileAttributeView faView = getFileAttributeView(f);
 
-    return getAttributeValue(faView, attributeName);
+    return getAttributeValue(faView, attributeName.toString());
 
   }
 
   @Override
   public List<String> getExtendedFileAttributeNames(File f) throws IOException {
 
     checkNotNull(f);
-    
-    UserDefinedFileAttributeView faView = Files.getFileAttributeView(
-      f.toPath(), UserDefinedFileAttributeView.class);
+
+    UserDefinedFileAttributeView faView =
+        Files.getFileAttributeView(f.toPath(), UserDefinedFileAttributeView.class);
 
     if (faView == null) {
       throw new IOException(
-        "UserDefinedFileAttributeView not supported on file "
-          + f.getAbsolutePath());
+          "UserDefinedFileAttributeView not supported on file " + f.getAbsolutePath());
     }
 
     return faView.list();
   }
 
   @Override
-  public void setChecksumAttribute(File f, String checksumValue)
-    throws IOException {
+  public boolean fileSupportsExtendedAttributes(File f) throws IOException {
 
-    if (fileSupportsExtendedAttributes(f)) {
-      setExtendedFileAttribute(f, STORM_ADLER32_CHECKSUM_ATTR_NAME,
-        checksumValue);
-    }
+    checkNotNull(f);
 
+    UserDefinedFileAttributeView faView =
+        Files.getFileAttributeView(f.toPath(), UserDefinedFileAttributeView.class);
+
+    return (faView != null);
   }
 
   @Override
-  public String getChecksumAttribute(File f) throws IOException {
+  public FileStat stat(Path p) throws IOException {
+    checkNotNull(p);
+    return stat(p.toFile());
+  }
 
-    return getExtendedFileAttributeValue(f, STORM_ADLER32_CHECKSUM_ATTR_NAME);
+  @Override
+  public FileStat stat(File f) throws IOException {
+    checkNotNull(f);
+    return posix.stat(f.getAbsolutePath());
   }
 
   @Override
-  public boolean fileSupportsExtendedAttributes(File f) throws IOException {
+  public FileStatus getFileStatus(Path p) throws IOException {
+    checkNotNull(p);
+    return getFileStatus(p.toFile());
+  }
 
+  @Override
+  public FileStatus getFileStatus(File f) throws IOException {
     checkNotNull(f);
-
-    UserDefinedFileAttributeView faView = Files.getFileAttributeView(
-      f.toPath(), UserDefinedFileAttributeView.class);
 
-    return (faView != null);
+    FileStat stat = stat(f);
+    if (isNull(stat)) {
+      throw new IOException("Unable to stat file " + f.toString());
+    }
+    if (stat.isDirectory()) {
+      return FileStatus.DISK;
+    }
+    List<String> attrs = getExtendedFileAttributeNames(f);
+
+    // check if file has been migrated to taoe
+    boolean hasMigrated = attrs.contains(STORM_MIGRATED_ATTR_NAME.toString());
+    // check if file is available with 0 latency
+    boolean isOnline = !isStub(stat);
+    if (isOnline) {
+      // file is available, check if it has a copy on tape
+      return hasMigrated ? FileStatus.DISK_AND_TAPE : FileStatus.DISK;
+    }
+    // file is a stub, no migrated attribute means an undefined situation
+    if (!hasMigrated) {
+      return FileStatus.UNDEFINED;
+    }
+    // file is on tape, check if a recall is in progress
+    boolean isRecallInProgress = attrs.contains(STORM_RECALL_IN_PROGRESS_ATTR_NAME.toString());
+    return isRecallInProgress ? FileStatus.TAPE_RECALL_IN_PROGRESS : FileStatus.TAPE;
   }
 
   @Override
-  public void setChecksumAttribute(Path p, String checksumValue) throws IOException {
-    setChecksumAttribute(p.toFile(), checksumValue);
-
+  public boolean isStub(Path p) throws IOException {
+    checkNotNull(p);
+    return isStub(p.toFile());
   }
 
   @Override
-  public String getChecksumAttribute(Path p) throws IOException {
-    return getChecksumAttribute(p.toFile());
+  public boolean isStub(File f) throws IOException {
+    checkNotNull(f);
+    return isStub(stat(f));
   }
 
+  @Override
+  public boolean isStub(FileStat fs) throws IOException {
+    checkNotNull(fs);
+    return fs.blockSize() * fs.blocks() < fs.st_size();
+  }
 }
diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributes.java b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributes.java
@@ -0,0 +1,20 @@
+package org.italiangrid.storm.webdav.fs.attrs;
+
+public enum ExtendedAttributes {
+
+  STORM_ADLER32_CHECKSUM_ATTR_NAME("storm.checksum.adler32"),
+  STORM_PREMIGRATE_ATTR_NAME("storm.premigrate"),
+  STORM_MIGRATED_ATTR_NAME("storm.migrated"),
+  STORM_RECALL_IN_PROGRESS_ATTR_NAME("storm.TSMRecT");
+
+  private final String attrName;
+
+  private ExtendedAttributes(String attrName) {
+    this.attrName = attrName;
+  }
+
+  @Override
+  public String toString() {
+    return attrName;
+  }
+}
diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributesHelper.java b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributesHelper.java
@@ -20,25 +20,31 @@
 import java.nio.file.Path;
 import java.util.List;
 
-public interface ExtendedAttributesHelper {
+import jnr.posix.FileStat;
 
-  public void setExtendedFileAttribute(File f, String attributeName,
-    String attributeValue) throws IOException;
+public interface ExtendedAttributesHelper {
 
-  public String getExtendedFileAttributeValue(File f, String attributeName)
-    throws IOException;
+  public boolean fileSupportsExtendedAttributes(File f) throws IOException;
 
   public List<String> getExtendedFileAttributeNames(File f) throws IOException;
 
-  public void setChecksumAttribute(Path p, String checksumValue)
-    throws IOException;
-
-  public void setChecksumAttribute(File f, String checksumValue)
-    throws IOException;
+  public void setExtendedFileAttribute(Path p, ExtendedAttributes name, String value) throws IOException;
 
-  public String getChecksumAttribute(File f) throws IOException;
-
-  public String getChecksumAttribute(Path p) throws IOException;
+  public void setExtendedFileAttribute(File f, ExtendedAttributes name, String value) throws IOException;
 
-  public boolean fileSupportsExtendedAttributes(File f) throws IOException;
+  public String getExtendedFileAttributeValue(File f, ExtendedAttributes name) throws IOException;
+
+  public FileStat stat(Path p) throws IOException;
+
+  public FileStat stat(File f) throws IOException;
+
+  public FileStatus getFileStatus(Path p) throws IOException;
+
+  public FileStatus getFileStatus(File f) throws IOException;
+
+  public boolean isStub(Path p) throws IOException;
+
+  public boolean isStub(File f) throws IOException;
+
+  public boolean isStub(FileStat fs) throws IOException;
 }
diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/FileStatus.java b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/FileStatus.java
@@ -0,0 +1,6 @@
+package org.italiangrid.storm.webdav.fs.attrs;
+
+public enum FileStatus {
+
+  TAPE, TAPE_RECALL_IN_PROGRESS, DISK, DISK_AND_TAPE, UNDEFINED;
+}
diff --git a/src/main/java/org/italiangrid/storm/webdav/milton/StoRMFileResource.java b/src/main/java/org/italiangrid/storm/webdav/milton/StoRMFileResource.java
@@ -17,6 +17,7 @@
 
 import static io.milton.property.PropertySource.PropertyAccessibility.READ_ONLY;
 import static java.util.Objects.isNull;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME;
 
 import java.io.BufferedInputStream;
 import java.io.File;
@@ -175,7 +176,8 @@ protected void calculateChecksum() {
         // do nothing, just read
       }
 
-      getExtendedAttributesHelper().setChecksumAttribute(getFile(), cis.getChecksumValue());
+      getExtendedAttributesHelper().setExtendedFileAttribute(getFile(),
+          STORM_ADLER32_CHECKSUM_ATTR_NAME, cis.getChecksumValue());
 
     } catch (IOException e) {
       throw new StoRMWebDAVError(e);
@@ -188,7 +190,8 @@ public Object getProperty(QName name) {
     if (name.getNamespaceURI().equals(STORM_NAMESPACE_URI)) {
       if (name.getLocalPart().equals(PROPERTY_CHECKSUM)) {
         try {
-          return getExtendedAttributesHelper().getChecksumAttribute(getFile());
+          return getExtendedAttributesHelper().getExtendedFileAttributeValue(getFile(),
+              STORM_ADLER32_CHECKSUM_ATTR_NAME);
         } catch (IOException e) {
           logger.warn("Errror getting checksum value for file: {}", getFile().getAbsolutePath(), e);
           return null;