diff --git a/pom.xml b/pom.xml
index 5db4b944..8ff09e51 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,6 +57,8 @@
6.0.2
5.5.1
+ 3.1.18
+
org.italiangrid.storm.webdav.WebdavService
@@ -448,6 +450,12 @@
${commons-cli.version}
+
+ com.github.jnr
+ jnr-posix
+ ${jnr-posix.version}
+
+
org.mockito
mockito-core
diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/DefaultFSStrategy.java b/src/main/java/org/italiangrid/storm/webdav/fs/DefaultFSStrategy.java
index aa3202d5..cd103728 100644
--- a/src/main/java/org/italiangrid/storm/webdav/fs/DefaultFSStrategy.java
+++ b/src/main/java/org/italiangrid/storm/webdav/fs/DefaultFSStrategy.java
@@ -15,6 +15,8 @@
*/
package org.italiangrid.storm.webdav.fs;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME;
+
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -137,7 +139,8 @@ public File create(File file, InputStream in) {
Adler32ChecksumInputStream cis = new Adler32ChecksumInputStream(in);
IOUtils.copy(cis, fos);
- attrsHelper.setChecksumAttribute(file, cis.getChecksumValue());
+ attrsHelper.setExtendedFileAttribute(file, STORM_ADLER32_CHECKSUM_ATTR_NAME,
+ cis.getChecksumValue());
return file;
diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/DefaultExtendedFileAttributesHelper.java b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/DefaultExtendedFileAttributesHelper.java
index 8bb6a7dc..e804e88f 100644
--- a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/DefaultExtendedFileAttributesHelper.java
+++ b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/DefaultExtendedFileAttributesHelper.java
@@ -15,9 +15,10 @@
*/
package org.italiangrid.storm.webdav.fs.attrs;
-import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.common.base.Strings.isNullOrEmpty;
+import static java.util.Objects.isNull;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_MIGRATED_ATTR_NAME;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_RECALL_IN_PROGRESS_ATTR_NAME;
import java.io.File;
import java.io.IOException;
@@ -28,21 +29,43 @@
import java.nio.file.attribute.UserDefinedFileAttributeView;
import java.util.List;
-public class DefaultExtendedFileAttributesHelper implements
- ExtendedAttributesHelper {
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
- public static final String STORM_ADLER32_CHECKSUM_ATTR_NAME = "storm.checksum.adler32";
+import jnr.posix.FileStat;
+import jnr.posix.POSIX;
+import jnr.posix.POSIXFactory;
+
+public class DefaultExtendedFileAttributesHelper implements ExtendedAttributesHelper {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(DefaultExtendedFileAttributesHelper.class);
+
+ private static POSIX posix;
public DefaultExtendedFileAttributesHelper() {
+ posix = POSIXFactory.getPOSIX();
+ }
+
+ protected UserDefinedFileAttributeView getFileAttributeView(File f) throws IOException {
+
+ UserDefinedFileAttributeView faView =
+ Files.getFileAttributeView(f.toPath(), UserDefinedFileAttributeView.class);
+
+ if (faView == null) {
+ throw new IOException(
+ "UserDefinedFileAttributeView not supported on file " + f.getAbsolutePath());
+ }
+ return faView;
}
- protected String getAttributeValue(UserDefinedFileAttributeView view,
- String attributeName) throws IOException {
+ protected String getAttributeValue(UserDefinedFileAttributeView view, String name)
+ throws IOException {
- if (view.list().contains(attributeName)) {
- ByteBuffer buffer = ByteBuffer.allocateDirect(view.size(attributeName));
- view.read(attributeName, buffer);
+ if (view.list().contains(name)) {
+ ByteBuffer buffer = ByteBuffer.allocateDirect(view.size(name));
+ view.read(name, buffer);
buffer.flip();
return StandardCharsets.UTF_8.decode(buffer).toString();
} else {
@@ -51,41 +74,35 @@ protected String getAttributeValue(UserDefinedFileAttributeView view,
}
@Override
- public void setExtendedFileAttribute(File f, String attributeName,
- String attributeValue) throws IOException {
+ public void setExtendedFileAttribute(Path p, ExtendedAttributes name, String value)
+ throws IOException {
+ setExtendedFileAttribute(p.toFile(), name, value);
+ }
+
+ @Override
+ public void setExtendedFileAttribute(File f, ExtendedAttributes name, String value)
+ throws IOException {
checkNotNull(f);
- checkArgument(!isNullOrEmpty(attributeName));
- UserDefinedFileAttributeView faView = Files.getFileAttributeView(
- f.toPath(), UserDefinedFileAttributeView.class);
+ UserDefinedFileAttributeView faView = getFileAttributeView(f);
- if (faView == null) {
- throw new IOException(
- "UserDefinedFileAttributeView not supported on file "
- + f.getAbsolutePath());
+ int bytes = faView.write(name.toString(), StandardCharsets.UTF_8.encode(value));
+ if (bytes > 0) {
+ LOG.debug("Setting '{}' extended attribute on file '{}': {} bytes written", name.toString(),
+ f.getAbsolutePath(), bytes);
}
-
- faView.write(attributeName, StandardCharsets.UTF_8.encode(attributeValue));
}
@Override
- public String getExtendedFileAttributeValue(File f, String attributeName)
- throws IOException {
+ public String getExtendedFileAttributeValue(File f, ExtendedAttributes attributeName)
+ throws IOException {
checkNotNull(f);
- checkArgument(!isNullOrEmpty(attributeName));
-
- UserDefinedFileAttributeView faView = Files.getFileAttributeView(
- f.toPath(), UserDefinedFileAttributeView.class);
- if (faView == null) {
- throw new IOException(
- "UserDefinedFileAttributeView not supported on file "
- + f.getAbsolutePath());
- }
+ UserDefinedFileAttributeView faView = getFileAttributeView(f);
- return getAttributeValue(faView, attributeName);
+ return getAttributeValue(faView, attributeName.toString());
}
@@ -93,56 +110,92 @@ public String getExtendedFileAttributeValue(File f, String attributeName)
public List getExtendedFileAttributeNames(File f) throws IOException {
checkNotNull(f);
-
- UserDefinedFileAttributeView faView = Files.getFileAttributeView(
- f.toPath(), UserDefinedFileAttributeView.class);
+
+ UserDefinedFileAttributeView faView =
+ Files.getFileAttributeView(f.toPath(), UserDefinedFileAttributeView.class);
if (faView == null) {
throw new IOException(
- "UserDefinedFileAttributeView not supported on file "
- + f.getAbsolutePath());
+ "UserDefinedFileAttributeView not supported on file " + f.getAbsolutePath());
}
return faView.list();
}
@Override
- public void setChecksumAttribute(File f, String checksumValue)
- throws IOException {
+ public boolean fileSupportsExtendedAttributes(File f) throws IOException {
- if (fileSupportsExtendedAttributes(f)) {
- setExtendedFileAttribute(f, STORM_ADLER32_CHECKSUM_ATTR_NAME,
- checksumValue);
- }
+ checkNotNull(f);
+ UserDefinedFileAttributeView faView =
+ Files.getFileAttributeView(f.toPath(), UserDefinedFileAttributeView.class);
+
+ return (faView != null);
}
@Override
- public String getChecksumAttribute(File f) throws IOException {
+ public FileStat stat(Path p) throws IOException {
+ checkNotNull(p);
+ return stat(p.toFile());
+ }
- return getExtendedFileAttributeValue(f, STORM_ADLER32_CHECKSUM_ATTR_NAME);
+ @Override
+ public FileStat stat(File f) throws IOException {
+ checkNotNull(f);
+ return posix.stat(f.getAbsolutePath());
}
@Override
- public boolean fileSupportsExtendedAttributes(File f) throws IOException {
+ public FileStatus getFileStatus(Path p) throws IOException {
+ checkNotNull(p);
+ return getFileStatus(p.toFile());
+ }
+ @Override
+ public FileStatus getFileStatus(File f) throws IOException {
checkNotNull(f);
-
- UserDefinedFileAttributeView faView = Files.getFileAttributeView(
- f.toPath(), UserDefinedFileAttributeView.class);
- return (faView != null);
+ FileStat stat = stat(f);
+ if (isNull(stat)) {
+ throw new IOException("Unable to stat file " + f.toString());
+ }
+ if (stat.isDirectory()) {
+ return FileStatus.DISK;
+ }
+ List attrs = getExtendedFileAttributeNames(f);
+
+ // check if file has been migrated to taoe
+ boolean hasMigrated = attrs.contains(STORM_MIGRATED_ATTR_NAME.toString());
+ // check if file is available with 0 latency
+ boolean isOnline = !isStub(stat);
+ if (isOnline) {
+ // file is available, check if it has a copy on tape
+ return hasMigrated ? FileStatus.DISK_AND_TAPE : FileStatus.DISK;
+ }
+ // file is a stub, no migrated attribute means an undefined situation
+ if (!hasMigrated) {
+ return FileStatus.UNDEFINED;
+ }
+ // file is on tape, check if a recall is in progress
+ boolean isRecallInProgress = attrs.contains(STORM_RECALL_IN_PROGRESS_ATTR_NAME.toString());
+ return isRecallInProgress ? FileStatus.TAPE_RECALL_IN_PROGRESS : FileStatus.TAPE;
}
@Override
- public void setChecksumAttribute(Path p, String checksumValue) throws IOException {
- setChecksumAttribute(p.toFile(), checksumValue);
-
+ public boolean isStub(Path p) throws IOException {
+ checkNotNull(p);
+ return isStub(p.toFile());
}
@Override
- public String getChecksumAttribute(Path p) throws IOException {
- return getChecksumAttribute(p.toFile());
+ public boolean isStub(File f) throws IOException {
+ checkNotNull(f);
+ return isStub(stat(f));
}
+ @Override
+ public boolean isStub(FileStat fs) throws IOException {
+ checkNotNull(fs);
+ return fs.blockSize() * fs.blocks() < fs.st_size();
+ }
}
diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributes.java b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributes.java
new file mode 100644
index 00000000..a5be8f43
--- /dev/null
+++ b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributes.java
@@ -0,0 +1,20 @@
+package org.italiangrid.storm.webdav.fs.attrs;
+
+public enum ExtendedAttributes {
+
+ STORM_ADLER32_CHECKSUM_ATTR_NAME("storm.checksum.adler32"),
+ STORM_PREMIGRATE_ATTR_NAME("storm.premigrate"),
+ STORM_MIGRATED_ATTR_NAME("storm.migrated"),
+ STORM_RECALL_IN_PROGRESS_ATTR_NAME("storm.TSMRecT");
+
+ private final String attrName;
+
+ private ExtendedAttributes(String attrName) {
+ this.attrName = attrName;
+ }
+
+ @Override
+ public String toString() {
+ return attrName;
+ }
+}
diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributesHelper.java b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributesHelper.java
index 39796d84..191e323f 100644
--- a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributesHelper.java
+++ b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributesHelper.java
@@ -20,25 +20,31 @@
import java.nio.file.Path;
import java.util.List;
-public interface ExtendedAttributesHelper {
+import jnr.posix.FileStat;
- public void setExtendedFileAttribute(File f, String attributeName,
- String attributeValue) throws IOException;
+public interface ExtendedAttributesHelper {
- public String getExtendedFileAttributeValue(File f, String attributeName)
- throws IOException;
+ public boolean fileSupportsExtendedAttributes(File f) throws IOException;
public List getExtendedFileAttributeNames(File f) throws IOException;
- public void setChecksumAttribute(Path p, String checksumValue)
- throws IOException;
-
- public void setChecksumAttribute(File f, String checksumValue)
- throws IOException;
+ public void setExtendedFileAttribute(Path p, ExtendedAttributes name, String value) throws IOException;
- public String getChecksumAttribute(File f) throws IOException;
-
- public String getChecksumAttribute(Path p) throws IOException;
+ public void setExtendedFileAttribute(File f, ExtendedAttributes name, String value) throws IOException;
- public boolean fileSupportsExtendedAttributes(File f) throws IOException;
+ public String getExtendedFileAttributeValue(File f, ExtendedAttributes name) throws IOException;
+
+ public FileStat stat(Path p) throws IOException;
+
+ public FileStat stat(File f) throws IOException;
+
+ public FileStatus getFileStatus(Path p) throws IOException;
+
+ public FileStatus getFileStatus(File f) throws IOException;
+
+ public boolean isStub(Path p) throws IOException;
+
+ public boolean isStub(File f) throws IOException;
+
+ public boolean isStub(FileStat fs) throws IOException;
}
diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/FileStatus.java b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/FileStatus.java
new file mode 100644
index 00000000..11d0417d
--- /dev/null
+++ b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/FileStatus.java
@@ -0,0 +1,6 @@
+package org.italiangrid.storm.webdav.fs.attrs;
+
+public enum FileStatus {
+
+ TAPE, TAPE_RECALL_IN_PROGRESS, DISK, DISK_AND_TAPE, UNDEFINED;
+}
diff --git a/src/main/java/org/italiangrid/storm/webdav/milton/StoRMFileResource.java b/src/main/java/org/italiangrid/storm/webdav/milton/StoRMFileResource.java
index aefbd19f..13c80f9b 100644
--- a/src/main/java/org/italiangrid/storm/webdav/milton/StoRMFileResource.java
+++ b/src/main/java/org/italiangrid/storm/webdav/milton/StoRMFileResource.java
@@ -17,6 +17,7 @@
import static io.milton.property.PropertySource.PropertyAccessibility.READ_ONLY;
import static java.util.Objects.isNull;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME;
import java.io.BufferedInputStream;
import java.io.File;
@@ -175,7 +176,8 @@ protected void calculateChecksum() {
// do nothing, just read
}
- getExtendedAttributesHelper().setChecksumAttribute(getFile(), cis.getChecksumValue());
+ getExtendedAttributesHelper().setExtendedFileAttribute(getFile(),
+ STORM_ADLER32_CHECKSUM_ATTR_NAME, cis.getChecksumValue());
} catch (IOException e) {
throw new StoRMWebDAVError(e);
@@ -188,7 +190,8 @@ public Object getProperty(QName name) {
if (name.getNamespaceURI().equals(STORM_NAMESPACE_URI)) {
if (name.getLocalPart().equals(PROPERTY_CHECKSUM)) {
try {
- return getExtendedAttributesHelper().getChecksumAttribute(getFile());
+ return getExtendedAttributesHelper().getExtendedFileAttributeValue(getFile(),
+ STORM_ADLER32_CHECKSUM_ATTR_NAME);
} catch (IOException e) {
logger.warn("Errror getting checksum value for file: {}", getFile().getAbsolutePath(), e);
return null;
diff --git a/src/main/java/org/italiangrid/storm/webdav/milton/util/EarlyChecksumStrategy.java b/src/main/java/org/italiangrid/storm/webdav/milton/util/EarlyChecksumStrategy.java
index aa5ea31e..a4b58d83 100644
--- a/src/main/java/org/italiangrid/storm/webdav/milton/util/EarlyChecksumStrategy.java
+++ b/src/main/java/org/italiangrid/storm/webdav/milton/util/EarlyChecksumStrategy.java
@@ -15,6 +15,8 @@
*/
package org.italiangrid.storm.webdav.milton.util;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME;
+
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -41,7 +43,8 @@ public void replaceContent(InputStream in, Long length, File targetFile) throws
throw new StoRMWebDAVError("Incomplete copy error!");
}
- attributesHelper.setChecksumAttribute(targetFile, cis.getChecksumValue());
+ attributesHelper.setExtendedFileAttribute(targetFile, STORM_ADLER32_CHECKSUM_ATTR_NAME,
+ cis.getChecksumValue());
}
}
diff --git a/src/main/java/org/italiangrid/storm/webdav/milton/util/LateChecksumStrategy.java b/src/main/java/org/italiangrid/storm/webdav/milton/util/LateChecksumStrategy.java
index 1a116ef8..7d132b54 100644
--- a/src/main/java/org/italiangrid/storm/webdav/milton/util/LateChecksumStrategy.java
+++ b/src/main/java/org/italiangrid/storm/webdav/milton/util/LateChecksumStrategy.java
@@ -15,6 +15,8 @@
*/
package org.italiangrid.storm.webdav.milton.util;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME;
+
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -55,7 +57,8 @@ protected void calculateChecksum(File targetFile) {
// do nothing, just read
}
- attributesHelper.setChecksumAttribute(targetFile, cis.getChecksumValue());
+ attributesHelper.setExtendedFileAttribute(targetFile, STORM_ADLER32_CHECKSUM_ATTR_NAME,
+ cis.getChecksumValue());
} catch (IOException e) {
throw new StoRMWebDAVError(e);
diff --git a/src/main/java/org/italiangrid/storm/webdav/server/servlet/ChecksumFilter.java b/src/main/java/org/italiangrid/storm/webdav/server/servlet/ChecksumFilter.java
index e3b33022..c3bc519b 100644
--- a/src/main/java/org/italiangrid/storm/webdav/server/servlet/ChecksumFilter.java
+++ b/src/main/java/org/italiangrid/storm/webdav/server/servlet/ChecksumFilter.java
@@ -17,6 +17,7 @@
import static com.google.common.base.Strings.isNullOrEmpty;
import static java.lang.String.format;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME;
import java.io.File;
import java.io.IOException;
@@ -34,7 +35,6 @@
import org.italiangrid.storm.webdav.server.PathResolver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
public class ChecksumFilter implements Filter {
@@ -43,7 +43,6 @@ public class ChecksumFilter implements Filter {
public static final Logger logger = LoggerFactory.getLogger(ChecksumFilter.class);
- @Autowired
public ChecksumFilter(ExtendedAttributesHelper attributeHelper,
PathResolver resolver) {
@@ -113,7 +112,8 @@ private void addChecksumHeader(HttpServletRequest request,
try {
- checksumValue = attributeHelper.getChecksumAttribute(f);
+ checksumValue =
+ attributeHelper.getExtendedFileAttributeValue(f, STORM_ADLER32_CHECKSUM_ATTR_NAME);
} catch (IOException e) {
diff --git a/src/main/java/org/italiangrid/storm/webdav/server/servlet/StoRMServlet.java b/src/main/java/org/italiangrid/storm/webdav/server/servlet/StoRMServlet.java
index 03f83598..778b7258 100644
--- a/src/main/java/org/italiangrid/storm/webdav/server/servlet/StoRMServlet.java
+++ b/src/main/java/org/italiangrid/storm/webdav/server/servlet/StoRMServlet.java
@@ -17,6 +17,7 @@
import java.io.File;
import java.io.IOException;
+import java.nio.file.Path;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
@@ -26,11 +27,14 @@
import org.eclipse.jetty.util.resource.Resource;
import org.italiangrid.storm.webdav.config.OAuthProperties;
import org.italiangrid.storm.webdav.config.ServiceConfigurationProperties;
+import org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributesHelper;
import org.italiangrid.storm.webdav.server.PathResolver;
import org.italiangrid.storm.webdav.server.servlet.resource.StormResourceService;
import org.italiangrid.storm.webdav.server.servlet.resource.StormResourceWrapper;
import org.thymeleaf.TemplateEngine;
+import jnr.posix.POSIX;
+
public class StoRMServlet extends DefaultServlet {
/**
@@ -43,15 +47,20 @@ public class StoRMServlet extends DefaultServlet {
final ServiceConfigurationProperties serviceConfig;
final OAuthProperties oauthProperties;
final StormResourceService resourceService;
+ final POSIX posix;
+ final ExtendedAttributesHelper attributesHelper;
public StoRMServlet(OAuthProperties oauthP, ServiceConfigurationProperties serviceConfig,
- PathResolver resolver, TemplateEngine engine, StormResourceService rs) {
+ PathResolver resolver, TemplateEngine engine, StormResourceService rs, POSIX posix,
+ ExtendedAttributesHelper attributesHelper) {
super(rs);
oauthProperties = oauthP;
resourceService = rs;
pathResolver = resolver;
templateEngine = engine;
this.serviceConfig = serviceConfig;
+ this.posix = posix;
+ this.attributesHelper = attributesHelper;
}
@Override
@@ -69,9 +78,9 @@ public Resource getResource(String pathInContext) {
return null;
}
- return new StormResourceWrapper(oauthProperties, serviceConfig, templateEngine,
- Resource.newResource(f));
-
+ return new StormResourceWrapper(oauthProperties, serviceConfig, templateEngine, resolvedPath,
+ Resource.newResource(Path.of(resolvedPath)), posix.stat(resolvedPath), posix,
+ attributesHelper);
}
@Override
diff --git a/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormFsResourceView.java b/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormFsResourceView.java
index 2add60c2..c7fae5fc 100644
--- a/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormFsResourceView.java
+++ b/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormFsResourceView.java
@@ -25,26 +25,33 @@ public class StormFsResourceView {
final String path;
+ final String fullPath;
+
final long sizeInBytes;
final Date lastModificationTime;
final Date creationTime;
+ final boolean isOnline;
+
+ final boolean isRecallInProgress;
+
private StormFsResourceView(Builder b) {
if (b.isDirectory && !b.name.endsWith("/")) {
- this.name = b.name +"/";
+ this.name = b.name + "/";
} else {
this.name = b.name;
}
-
+
this.isDirectory = b.isDirectory;
this.path = b.path;
+ this.fullPath = b.fullPath;
this.sizeInBytes = b.sizeInBytes;
this.lastModificationTime = b.lastModificationTime;
this.creationTime = b.creationTime;
-
-
+ this.isOnline = b.isOnline;
+ this.isRecallInProgress = b.isRecallInProgress;
}
public String getName() {
@@ -77,16 +84,32 @@ public Date getCreationTime() {
}
+ public String getFullPath() {
+ return fullPath;
+ }
+
+ public boolean getIsOnline() {
+ return isOnline;
+ }
+
+ public boolean getIsRecallInProgress() {
+ return isRecallInProgress;
+ }
+
public static Builder builder() {
return new Builder();
}
+
public static class Builder {
String name;
boolean isDirectory;
String path;
+ String fullPath;
long sizeInBytes;
Date lastModificationTime;
Date creationTime;
+ boolean isOnline;
+ boolean isRecallInProgress;
public Builder() {}
@@ -106,6 +129,11 @@ public Builder withPath(String path) {
return this;
}
+ public Builder withFullPath(String fullPath) {
+ this.fullPath = fullPath;
+ return this;
+ }
+
public Builder withSizeInBytes(long syzeInBytes) {
this.sizeInBytes = syzeInBytes;
return this;
@@ -121,6 +149,16 @@ public Builder withCreationTime(Date creationTime) {
return this;
}
+ public Builder withIsOnline(boolean isOnline) {
+ this.isOnline = isOnline;
+ return this;
+ }
+
+ public Builder withIsRecallInProgress(boolean isRecallInProgress) {
+ this.isRecallInProgress = isRecallInProgress;
+ return this;
+ }
+
public StormFsResourceView build() {
return new StormFsResourceView(this);
}
diff --git a/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResource.java b/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResource.java
new file mode 100644
index 00000000..67aa3b09
--- /dev/null
+++ b/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResource.java
@@ -0,0 +1,15 @@
+package org.italiangrid.storm.webdav.server.servlet.resource;
+
+import java.io.IOException;
+
+public interface StormResource {
+
+ public boolean isOnline() throws IOException;
+
+ public boolean isStub() throws IOException;
+
+ public boolean hasMigrated() throws IOException;
+
+ public boolean hasInProgressRecall() throws IOException;
+
+}
diff --git a/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResourceWrapper.java b/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResourceWrapper.java
index 73a030e9..bf1d34f9 100644
--- a/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResourceWrapper.java
+++ b/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResourceWrapper.java
@@ -15,6 +15,9 @@
*/
package org.italiangrid.storm.webdav.server.servlet.resource;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_MIGRATED_ATTR_NAME;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_RECALL_IN_PROGRESS_ATTR_NAME;
+
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -22,6 +25,7 @@
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.channels.ReadableByteChannel;
+import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
@@ -33,11 +37,16 @@
import org.italiangrid.storm.webdav.authn.AuthenticationUtils;
import org.italiangrid.storm.webdav.config.OAuthProperties;
import org.italiangrid.storm.webdav.config.ServiceConfigurationProperties;
+import org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributesHelper;
import org.springframework.security.core.context.SecurityContextHolder;
import org.thymeleaf.TemplateEngine;
import org.thymeleaf.context.Context;
-public class StormResourceWrapper extends Resource {
+import jnr.posix.FileStat;
+import jnr.posix.POSIX;
+import jnr.posix.POSIXFactory;
+
+public class StormResourceWrapper extends Resource implements StormResource {
public static final String JETTY_DIR_TEMPLATE = "jetty-dir";
@@ -45,15 +54,23 @@ public class StormResourceWrapper extends Resource {
final TemplateEngine engine;
final OAuthProperties oauthProperties;
final ServiceConfigurationProperties serviceConfig;
+ final String absoluteFilePath;
+ final FileStat fileStat;
+ final ExtendedAttributesHelper attrsHelper;
+ final POSIX posix;
public StormResourceWrapper(OAuthProperties oauth, ServiceConfigurationProperties serviceConfig,
- TemplateEngine engine, Resource delegate) {
+ TemplateEngine engine, String absoluteFilePath, Resource delegate, FileStat fileStat,
+ POSIX posix, ExtendedAttributesHelper attributesHelper) {
this.oauthProperties = oauth;
this.engine = engine;
this.delegate = delegate;
this.serviceConfig = serviceConfig;
-
+ this.absoluteFilePath = absoluteFilePath;
+ this.fileStat = fileStat;
+ this.attrsHelper = attributesHelper;
+ this.posix = posix;
}
/**
@@ -136,6 +153,7 @@ public String getListHTML(String base, boolean parent, String query) throws IOEx
context.setVariable("oidcEnabled", oauthProperties.isEnableOidc());
String encodedBase = hrefEncodeURI(decodedBase);
+ String encodedFullBasePath = hrefEncodeURI(delegate.getFile().getCanonicalPath());
String parentDir = URIUtil.addPaths(encodedBase, "../");
@@ -144,13 +162,27 @@ public String getListHTML(String base, boolean parent, String query) throws IOEx
List resources = new ArrayList<>();
for (String l : rawListing) {
- Resource r = addPath(l);
+
+ String absolutePath = String.format("%s/%s", absoluteFilePath, l);
+ File f = new File(absolutePath);
+ FileStat fStat = POSIXFactory.getPOSIX().stat(absolutePath);
+
+ boolean isOnline = true;
+ boolean hasRecallInProgress = false;
+ boolean isDirectory = fStat.isDirectory();
+ if (!isDirectory) {
+ isOnline = !isStub(fStat);
+ hasRecallInProgress = attrsHelper.getExtendedFileAttributeNames(f).contains(STORM_RECALL_IN_PROGRESS_ATTR_NAME.toString());
+ }
resources.add(StormFsResourceView.builder()
.withName(l)
.withPath(URIUtil.addEncodedPaths(encodedBase, URIUtil.encodePath(l)))
- .withIsDirectory(r.isDirectory())
- .withLastModificationTime(new Date(r.lastModified()))
- .withSizeInBytes(r.length())
+ .withFullPath(URIUtil.addEncodedPaths(encodedFullBasePath, URIUtil.encodePath(l)))
+ .withIsOnline(isOnline)
+ .withIsRecallInProgress(hasRecallInProgress)
+ .withIsDirectory(isDirectory)
+ .withLastModificationTime(Date.from(Instant.ofEpochMilli(fStat.ctime() + fStat.mtime())))
+ .withSizeInBytes(fStat.st_size())
.build());
}
@@ -189,7 +221,8 @@ public long lastModified() {
@Override
public long length() {
- return delegate.length();
+ return fileStat.st_size();
+ // return delegate.length();
}
@SuppressWarnings("deprecation")
@@ -289,5 +322,29 @@ private void internalCopy(InputStream in, OutputStream out) throws IOException {
internalCopy(in, out, -1);
}
+ @Override
+ public boolean isOnline() {
+ return !isStub(fileStat);
+ }
+
+ @Override
+ public boolean isStub() {
+ return isStub(fileStat);
+ }
+
+ @Override
+ public boolean hasMigrated() throws IOException {
+ return attrsHelper.getExtendedFileAttributeNames(getFile())
+ .contains(STORM_MIGRATED_ATTR_NAME.toString());
+ }
+ @Override
+ public boolean hasInProgressRecall() throws IOException {
+ return attrsHelper.getExtendedFileAttributeNames(getFile())
+ .contains(STORM_RECALL_IN_PROGRESS_ATTR_NAME.toString());
+ }
+
+ public static boolean isStub(FileStat fileStat) {
+ return fileStat.blockSize() * fileStat.blocks() < fileStat.st_size();
+ }
}
diff --git a/src/main/java/org/italiangrid/storm/webdav/spring/web/ServletConfiguration.java b/src/main/java/org/italiangrid/storm/webdav/spring/web/ServletConfiguration.java
index b6813811..cccc1391 100644
--- a/src/main/java/org/italiangrid/storm/webdav/spring/web/ServletConfiguration.java
+++ b/src/main/java/org/italiangrid/storm/webdav/spring/web/ServletConfiguration.java
@@ -25,6 +25,7 @@
import org.italiangrid.storm.webdav.config.StorageAreaConfiguration;
import org.italiangrid.storm.webdav.config.ThirdPartyCopyProperties;
import org.italiangrid.storm.webdav.fs.FilesystemAccess;
+import org.italiangrid.storm.webdav.fs.attrs.DefaultExtendedFileAttributesHelper;
import org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributesHelper;
import org.italiangrid.storm.webdav.macaroon.MacaroonIssuerService;
import org.italiangrid.storm.webdav.macaroon.MacaroonRequestFilter;
@@ -61,7 +62,7 @@
import com.codahale.metrics.servlets.MetricsServlet;
import com.fasterxml.jackson.databind.ObjectMapper;
-
+import jnr.posix.POSIXFactory;
@Configuration
public class ServletConfiguration {
@@ -225,7 +226,8 @@ ServletRegistrationBean stormServlet(OAuthProperties oauthProperti
ServletRegistrationBean stormServlet =
new ServletRegistrationBean<>(new StoRMServlet(oauthProperties, serviceConfig, pathResolver,
- templateEngine, new StormResourceService()));
+ templateEngine, new StormResourceService(), POSIXFactory.getPOSIX(),
+ new DefaultExtendedFileAttributesHelper()));
stormServlet.addInitParameter("acceptRanges", "true");
stormServlet.addInitParameter("dirAllowed", "true");
diff --git a/src/main/java/org/italiangrid/storm/webdav/tape/WlcgTapeRestApiController.java b/src/main/java/org/italiangrid/storm/webdav/tape/WlcgTapeRestApiController.java
index ed355ef9..a25e6742 100644
--- a/src/main/java/org/italiangrid/storm/webdav/tape/WlcgTapeRestApiController.java
+++ b/src/main/java/org/italiangrid/storm/webdav/tape/WlcgTapeRestApiController.java
@@ -18,7 +18,7 @@
import static org.springframework.http.HttpStatus.NOT_FOUND;
-import org.italiangrid.storm.webdav.tape.model.WlcgTapeRestApi;
+import org.italiangrid.storm.webdav.tape.model.WlcgTapeRestApiMetadata;
import org.italiangrid.storm.webdav.tape.service.WlcgTapeRestApiService;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
@@ -34,9 +34,9 @@ public WlcgTapeRestApiController(WlcgTapeRestApiService service) {
}
@GetMapping({".well-known/wlcg-tape-rest-api"})
- public WlcgTapeRestApi getMetadata() {
+ public WlcgTapeRestApiMetadata getMetadata() {
- WlcgTapeRestApi metadata = service.getMetadata();
+ WlcgTapeRestApiMetadata metadata = service.getMetadata();
if (metadata == null) {
throw new ResponseStatusException(NOT_FOUND, "Unable to find resource");
}
diff --git a/src/main/java/org/italiangrid/storm/webdav/tape/model/WlcgTapeRestApi.java b/src/main/java/org/italiangrid/storm/webdav/tape/model/WlcgTapeRestApiMetadata.java
similarity index 97%
rename from src/main/java/org/italiangrid/storm/webdav/tape/model/WlcgTapeRestApi.java
rename to src/main/java/org/italiangrid/storm/webdav/tape/model/WlcgTapeRestApiMetadata.java
index c3557680..d5eeece0 100644
--- a/src/main/java/org/italiangrid/storm/webdav/tape/model/WlcgTapeRestApi.java
+++ b/src/main/java/org/italiangrid/storm/webdav/tape/model/WlcgTapeRestApiMetadata.java
@@ -20,7 +20,7 @@
import com.google.common.collect.Lists;
-public class WlcgTapeRestApi {
+public class WlcgTapeRestApiMetadata {
private String sitename;
private String description;
diff --git a/src/main/java/org/italiangrid/storm/webdav/tape/service/WlcgTapeRestApiService.java b/src/main/java/org/italiangrid/storm/webdav/tape/service/WlcgTapeRestApiService.java
index 44158ba3..c5fc71c1 100644
--- a/src/main/java/org/italiangrid/storm/webdav/tape/service/WlcgTapeRestApiService.java
+++ b/src/main/java/org/italiangrid/storm/webdav/tape/service/WlcgTapeRestApiService.java
@@ -20,7 +20,7 @@
import java.io.IOException;
import org.italiangrid.storm.webdav.config.ServiceConfigurationProperties;
-import org.italiangrid.storm.webdav.tape.model.WlcgTapeRestApi;
+import org.italiangrid.storm.webdav.tape.model.WlcgTapeRestApiMetadata;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@@ -36,7 +36,7 @@ public class WlcgTapeRestApiService {
private static final String LOG_ERROR_PREFIX = "Error loading WLCG Tape REST API well-known endpoint from file: {}";
private static final String LOG_INFO_NOFILEFOUND = "No WLCG Tape REST API well-known file found at '{}'";
- private WlcgTapeRestApi metadata;
+ private WlcgTapeRestApiMetadata metadata;
public WlcgTapeRestApiService(ServiceConfigurationProperties props) {
@@ -45,7 +45,7 @@ public WlcgTapeRestApiService(ServiceConfigurationProperties props) {
if (source.exists()) {
LOG.info(LOG_INFO_LOADING, source);
try {
- metadata = (new ObjectMapper()).readValue(source, WlcgTapeRestApi.class);
+ metadata = (new ObjectMapper()).readValue(source, WlcgTapeRestApiMetadata.class);
} catch (IOException e) {
LOG.error(LOG_ERROR_PREFIX, e.getMessage());
}
@@ -54,7 +54,7 @@ public WlcgTapeRestApiService(ServiceConfigurationProperties props) {
}
}
- public WlcgTapeRestApi getMetadata() {
+ public WlcgTapeRestApiMetadata getMetadata() {
return metadata;
}
diff --git a/src/main/java/org/italiangrid/storm/webdav/tpc/http/GetResponseHandler.java b/src/main/java/org/italiangrid/storm/webdav/tpc/http/GetResponseHandler.java
index 1a2941d6..41fada38 100644
--- a/src/main/java/org/italiangrid/storm/webdav/tpc/http/GetResponseHandler.java
+++ b/src/main/java/org/italiangrid/storm/webdav/tpc/http/GetResponseHandler.java
@@ -16,6 +16,7 @@
package org.italiangrid.storm.webdav.tpc.http;
import static java.util.Objects.isNull;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME;
import java.io.IOException;
import java.io.InputStream;
@@ -110,8 +111,8 @@ public Boolean handleResponse(HttpResponse response) throws IOException {
writeEntityToStream(entity, os);
if (computeChecksum) {
- attributesHelper.setChecksumAttribute(fileStream.getPath(),
- checkedStream.getChecksumValue());
+ attributesHelper.setExtendedFileAttribute(fileStream.getPath(),
+ STORM_ADLER32_CHECKSUM_ATTR_NAME, checkedStream.getChecksumValue());
}
}
diff --git a/src/main/resources/templates/jetty-dir.html b/src/main/resources/templates/jetty-dir.html
index 6963c475..a603271e 100644
--- a/src/main/resources/templates/jetty-dir.html
+++ b/src/main/resources/templates/jetty-dir.html
@@ -17,15 +17,22 @@ Directory
Name |
Last modified |
Size (in bytes) |
+ Full Path |
+ Is Online |
+ Is Recalling |
-
- fake
+ |
+ fake
+ fake
|
Last modification time |
Size in bytes |
+ Full Path |
+ Is Online |
+ Is Recalling |
diff --git a/src/test/java/org/italiangrid/storm/webdav/test/tape/FileStubTest.java b/src/test/java/org/italiangrid/storm/webdav/test/tape/FileStubTest.java
new file mode 100644
index 00000000..1ea3110e
--- /dev/null
+++ b/src/test/java/org/italiangrid/storm/webdav/test/tape/FileStubTest.java
@@ -0,0 +1,134 @@
+/**
+ * Copyright (c) Istituto Nazionale di Fisica Nucleare, 2014-2023.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.italiangrid.storm.webdav.test.tape;
+
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_MIGRATED_ATTR_NAME;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_RECALL_IN_PROGRESS_ATTR_NAME;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import org.italiangrid.storm.webdav.fs.attrs.DefaultExtendedFileAttributesHelper;
+import org.italiangrid.storm.webdav.fs.attrs.FileStatus;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+@ExtendWith(MockitoExtension.class)
+public class FileStubTest {
+
+ private final static String RESOURCE_BASE_PATH = "storage/tape";
+
+ private File stubFile;
+ private File undefinedFile;
+ private File onlineFile;
+ private File onlineAndMigratedFile;
+ private File recallInProgressFile;
+ private DefaultExtendedFileAttributesHelper helper = new DefaultExtendedFileAttributesHelper();
+
+ private void printProcess(Process p) throws IOException {
+ BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream()));
+ String str = br.readLine();
+ while (str != null) {
+ System.out.println(str);
+ str = br.readLine();
+ }
+ }
+
+ private File createUndefinedFile(String fileName) throws IOException {
+
+ ClassLoader classLoader = getClass().getClassLoader();
+ File resourceBaseDir = new File(classLoader.getResource(RESOURCE_BASE_PATH).getFile());
+ File undefinedFile = new File(resourceBaseDir + "/" + fileName);
+ printProcess(Runtime.getRuntime()
+ .exec("dd if=/dev/zero conv=sparse bs=1M count=1 of=" + undefinedFile.getAbsolutePath()));
+ return undefinedFile;
+ }
+
+ private File createStubFile(String fileName) throws IOException {
+
+ File stubFile = createUndefinedFile(fileName);
+ helper.setExtendedFileAttribute(stubFile, STORM_MIGRATED_ATTR_NAME, "y");
+ return stubFile;
+ }
+
+ private File createStubRecalledFile(String fileName) throws IOException {
+
+ File stubRecalledFile = createStubFile(fileName);
+ helper.setExtendedFileAttribute(stubRecalledFile, STORM_RECALL_IN_PROGRESS_ATTR_NAME, "y");
+ return stubRecalledFile;
+ }
+
+ private File createOnlineFile(String fileName) throws IOException {
+
+ ClassLoader classLoader = getClass().getClassLoader();
+ File resourceBaseDir = new File(classLoader.getResource(RESOURCE_BASE_PATH).getFile());
+ File onlineFile = new File(resourceBaseDir + "/" + fileName);
+ return onlineFile;
+ }
+
+ private File createOnlineAndMigratedFile(String fileName) throws IOException {
+
+ File migratedFile = createOnlineFile(fileName);
+ helper.setExtendedFileAttribute(migratedFile, STORM_MIGRATED_ATTR_NAME, "y");
+ return migratedFile;
+ }
+
+ @BeforeEach
+ public void setup() throws IOException {
+
+ undefinedFile = createUndefinedFile("undefined.dat");
+ stubFile = createStubFile("tape.dat");
+ recallInProgressFile = createStubRecalledFile("tape-recalled.dat");
+ onlineFile = createOnlineFile("disk.dat");
+ onlineAndMigratedFile = createOnlineAndMigratedFile("disk-and-tape.dat");
+ }
+
+ @AfterEach
+ public void finalize() throws IOException {
+
+ }
+
+ @Test
+ public void testUndefinedFile() throws IOException {
+ assertEquals(FileStatus.UNDEFINED, helper.getFileStatus(undefinedFile));
+ }
+
+ @Test
+ public void testStubFile() throws IOException {
+ assertEquals(FileStatus.TAPE, helper.getFileStatus(stubFile));
+ }
+
+ @Test
+ public void testOnlineFile() throws IOException {
+ assertEquals(FileStatus.DISK, helper.getFileStatus(onlineFile));
+ }
+
+ @Test
+ public void testOnlineAndMigratedFile() throws IOException {
+ assertEquals(FileStatus.DISK_AND_TAPE, helper.getFileStatus(onlineAndMigratedFile));
+ }
+
+ @Test
+ public void testRecallInProgressFile() throws IOException {
+ assertEquals(FileStatus.TAPE_RECALL_IN_PROGRESS, helper.getFileStatus(recallInProgressFile));
+ }
+}
diff --git a/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/GetResponseHandlerTest.java b/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/GetResponseHandlerTest.java
index 257b060c..a111ec98 100644
--- a/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/GetResponseHandlerTest.java
+++ b/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/GetResponseHandlerTest.java
@@ -15,6 +15,7 @@
*/
package org.italiangrid.storm.webdav.test.tpc.http;
+import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.verify;
@@ -26,6 +27,7 @@
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.StatusLine;
+import org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes;
import org.italiangrid.storm.webdav.tpc.http.GetResponseHandler;
import org.italiangrid.storm.webdav.tpc.utils.StormCountingOutputStream;
import org.junit.jupiter.api.BeforeEach;
@@ -40,26 +42,26 @@ public class GetResponseHandlerTest extends ClientTestSupport {
@Mock
StatusLine status;
-
+
@Mock
HttpEntity entity;
-
+
@Mock
HttpResponse response;
-
+
@Mock
StormCountingOutputStream os;
-
+
GetResponseHandler handler;
@BeforeEach
public void setup() {
-
+
handler = new GetResponseHandler(null, os, eah);
lenient().when(response.getStatusLine()).thenReturn(status);
lenient().when(response.getEntity()).thenReturn(entity);
}
-
+
@Test
public void handlerWritesToStream() throws IOException {
when(status.getStatusCode()).thenReturn(200);
@@ -67,6 +69,6 @@ public void handlerWritesToStream() throws IOException {
handler.handleResponse(response);
verify(entity).getContent();
- verify(eah).setChecksumAttribute(ArgumentMatchers.any(), any());
+ verify(eah).setExtendedFileAttribute(ArgumentMatchers.any(), ArgumentMatchers.eq(STORM_ADLER32_CHECKSUM_ATTR_NAME), any());
}
}