diff --git a/pom.xml b/pom.xml index 5db4b944..8ff09e51 100644 --- a/pom.xml +++ b/pom.xml @@ -57,6 +57,8 @@ 6.0.2 5.5.1 + 3.1.18 + org.italiangrid.storm.webdav.WebdavService @@ -448,6 +450,12 @@ ${commons-cli.version} + + com.github.jnr + jnr-posix + ${jnr-posix.version} + + org.mockito mockito-core diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/DefaultFSStrategy.java b/src/main/java/org/italiangrid/storm/webdav/fs/DefaultFSStrategy.java index aa3202d5..cd103728 100644 --- a/src/main/java/org/italiangrid/storm/webdav/fs/DefaultFSStrategy.java +++ b/src/main/java/org/italiangrid/storm/webdav/fs/DefaultFSStrategy.java @@ -15,6 +15,8 @@ */ package org.italiangrid.storm.webdav.fs; +import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -137,7 +139,8 @@ public File create(File file, InputStream in) { Adler32ChecksumInputStream cis = new Adler32ChecksumInputStream(in); IOUtils.copy(cis, fos); - attrsHelper.setChecksumAttribute(file, cis.getChecksumValue()); + attrsHelper.setExtendedFileAttribute(file, STORM_ADLER32_CHECKSUM_ATTR_NAME, + cis.getChecksumValue()); return file; diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/DefaultExtendedFileAttributesHelper.java b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/DefaultExtendedFileAttributesHelper.java index 8bb6a7dc..e804e88f 100644 --- a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/DefaultExtendedFileAttributesHelper.java +++ b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/DefaultExtendedFileAttributesHelper.java @@ -15,9 +15,10 @@ */ package org.italiangrid.storm.webdav.fs.attrs; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.base.Strings.isNullOrEmpty; +import static java.util.Objects.isNull; +import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_MIGRATED_ATTR_NAME; +import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_RECALL_IN_PROGRESS_ATTR_NAME; import java.io.File; import java.io.IOException; @@ -28,21 +29,43 @@ import java.nio.file.attribute.UserDefinedFileAttributeView; import java.util.List; -public class DefaultExtendedFileAttributesHelper implements - ExtendedAttributesHelper { +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; - public static final String STORM_ADLER32_CHECKSUM_ATTR_NAME = "storm.checksum.adler32"; +import jnr.posix.FileStat; +import jnr.posix.POSIX; +import jnr.posix.POSIXFactory; + +public class DefaultExtendedFileAttributesHelper implements ExtendedAttributesHelper { + + public static final Logger LOG = + LoggerFactory.getLogger(DefaultExtendedFileAttributesHelper.class); + + private static POSIX posix; public DefaultExtendedFileAttributesHelper() { + posix = POSIXFactory.getPOSIX(); + } + + protected UserDefinedFileAttributeView getFileAttributeView(File f) throws IOException { + + UserDefinedFileAttributeView faView = + Files.getFileAttributeView(f.toPath(), UserDefinedFileAttributeView.class); + + if (faView == null) { + throw new IOException( + "UserDefinedFileAttributeView not supported on file " + f.getAbsolutePath()); + } + return faView; } - protected String getAttributeValue(UserDefinedFileAttributeView view, - String attributeName) throws IOException { + protected String getAttributeValue(UserDefinedFileAttributeView view, String name) + throws IOException { - if (view.list().contains(attributeName)) { - ByteBuffer buffer = ByteBuffer.allocateDirect(view.size(attributeName)); - view.read(attributeName, buffer); + if (view.list().contains(name)) { + ByteBuffer buffer = ByteBuffer.allocateDirect(view.size(name)); + view.read(name, buffer); buffer.flip(); return StandardCharsets.UTF_8.decode(buffer).toString(); } else { @@ -51,41 +74,35 @@ protected String getAttributeValue(UserDefinedFileAttributeView view, } @Override - public void setExtendedFileAttribute(File f, String attributeName, - String attributeValue) throws IOException { + public void setExtendedFileAttribute(Path p, ExtendedAttributes name, String value) + throws IOException { + setExtendedFileAttribute(p.toFile(), name, value); + } + + @Override + public void setExtendedFileAttribute(File f, ExtendedAttributes name, String value) + throws IOException { checkNotNull(f); - checkArgument(!isNullOrEmpty(attributeName)); - UserDefinedFileAttributeView faView = Files.getFileAttributeView( - f.toPath(), UserDefinedFileAttributeView.class); + UserDefinedFileAttributeView faView = getFileAttributeView(f); - if (faView == null) { - throw new IOException( - "UserDefinedFileAttributeView not supported on file " - + f.getAbsolutePath()); + int bytes = faView.write(name.toString(), StandardCharsets.UTF_8.encode(value)); + if (bytes > 0) { + LOG.debug("Setting '{}' extended attribute on file '{}': {} bytes written", name.toString(), + f.getAbsolutePath(), bytes); } - - faView.write(attributeName, StandardCharsets.UTF_8.encode(attributeValue)); } @Override - public String getExtendedFileAttributeValue(File f, String attributeName) - throws IOException { + public String getExtendedFileAttributeValue(File f, ExtendedAttributes attributeName) + throws IOException { checkNotNull(f); - checkArgument(!isNullOrEmpty(attributeName)); - - UserDefinedFileAttributeView faView = Files.getFileAttributeView( - f.toPath(), UserDefinedFileAttributeView.class); - if (faView == null) { - throw new IOException( - "UserDefinedFileAttributeView not supported on file " - + f.getAbsolutePath()); - } + UserDefinedFileAttributeView faView = getFileAttributeView(f); - return getAttributeValue(faView, attributeName); + return getAttributeValue(faView, attributeName.toString()); } @@ -93,56 +110,92 @@ public String getExtendedFileAttributeValue(File f, String attributeName) public List getExtendedFileAttributeNames(File f) throws IOException { checkNotNull(f); - - UserDefinedFileAttributeView faView = Files.getFileAttributeView( - f.toPath(), UserDefinedFileAttributeView.class); + + UserDefinedFileAttributeView faView = + Files.getFileAttributeView(f.toPath(), UserDefinedFileAttributeView.class); if (faView == null) { throw new IOException( - "UserDefinedFileAttributeView not supported on file " - + f.getAbsolutePath()); + "UserDefinedFileAttributeView not supported on file " + f.getAbsolutePath()); } return faView.list(); } @Override - public void setChecksumAttribute(File f, String checksumValue) - throws IOException { + public boolean fileSupportsExtendedAttributes(File f) throws IOException { - if (fileSupportsExtendedAttributes(f)) { - setExtendedFileAttribute(f, STORM_ADLER32_CHECKSUM_ATTR_NAME, - checksumValue); - } + checkNotNull(f); + UserDefinedFileAttributeView faView = + Files.getFileAttributeView(f.toPath(), UserDefinedFileAttributeView.class); + + return (faView != null); } @Override - public String getChecksumAttribute(File f) throws IOException { + public FileStat stat(Path p) throws IOException { + checkNotNull(p); + return stat(p.toFile()); + } - return getExtendedFileAttributeValue(f, STORM_ADLER32_CHECKSUM_ATTR_NAME); + @Override + public FileStat stat(File f) throws IOException { + checkNotNull(f); + return posix.stat(f.getAbsolutePath()); } @Override - public boolean fileSupportsExtendedAttributes(File f) throws IOException { + public FileStatus getFileStatus(Path p) throws IOException { + checkNotNull(p); + return getFileStatus(p.toFile()); + } + @Override + public FileStatus getFileStatus(File f) throws IOException { checkNotNull(f); - - UserDefinedFileAttributeView faView = Files.getFileAttributeView( - f.toPath(), UserDefinedFileAttributeView.class); - return (faView != null); + FileStat stat = stat(f); + if (isNull(stat)) { + throw new IOException("Unable to stat file " + f.toString()); + } + if (stat.isDirectory()) { + return FileStatus.DISK; + } + List attrs = getExtendedFileAttributeNames(f); + + // check if file has been migrated to taoe + boolean hasMigrated = attrs.contains(STORM_MIGRATED_ATTR_NAME.toString()); + // check if file is available with 0 latency + boolean isOnline = !isStub(stat); + if (isOnline) { + // file is available, check if it has a copy on tape + return hasMigrated ? FileStatus.DISK_AND_TAPE : FileStatus.DISK; + } + // file is a stub, no migrated attribute means an undefined situation + if (!hasMigrated) { + return FileStatus.UNDEFINED; + } + // file is on tape, check if a recall is in progress + boolean isRecallInProgress = attrs.contains(STORM_RECALL_IN_PROGRESS_ATTR_NAME.toString()); + return isRecallInProgress ? FileStatus.TAPE_RECALL_IN_PROGRESS : FileStatus.TAPE; } @Override - public void setChecksumAttribute(Path p, String checksumValue) throws IOException { - setChecksumAttribute(p.toFile(), checksumValue); - + public boolean isStub(Path p) throws IOException { + checkNotNull(p); + return isStub(p.toFile()); } @Override - public String getChecksumAttribute(Path p) throws IOException { - return getChecksumAttribute(p.toFile()); + public boolean isStub(File f) throws IOException { + checkNotNull(f); + return isStub(stat(f)); } + @Override + public boolean isStub(FileStat fs) throws IOException { + checkNotNull(fs); + return fs.blockSize() * fs.blocks() < fs.st_size(); + } } diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributes.java b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributes.java new file mode 100644 index 00000000..a5be8f43 --- /dev/null +++ b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributes.java @@ -0,0 +1,20 @@ +package org.italiangrid.storm.webdav.fs.attrs; + +public enum ExtendedAttributes { + + STORM_ADLER32_CHECKSUM_ATTR_NAME("storm.checksum.adler32"), + STORM_PREMIGRATE_ATTR_NAME("storm.premigrate"), + STORM_MIGRATED_ATTR_NAME("storm.migrated"), + STORM_RECALL_IN_PROGRESS_ATTR_NAME("storm.TSMRecT"); + + private final String attrName; + + private ExtendedAttributes(String attrName) { + this.attrName = attrName; + } + + @Override + public String toString() { + return attrName; + } +} diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributesHelper.java b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributesHelper.java index 39796d84..191e323f 100644 --- a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributesHelper.java +++ b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/ExtendedAttributesHelper.java @@ -20,25 +20,31 @@ import java.nio.file.Path; import java.util.List; -public interface ExtendedAttributesHelper { +import jnr.posix.FileStat; - public void setExtendedFileAttribute(File f, String attributeName, - String attributeValue) throws IOException; +public interface ExtendedAttributesHelper { - public String getExtendedFileAttributeValue(File f, String attributeName) - throws IOException; + public boolean fileSupportsExtendedAttributes(File f) throws IOException; public List getExtendedFileAttributeNames(File f) throws IOException; - public void setChecksumAttribute(Path p, String checksumValue) - throws IOException; - - public void setChecksumAttribute(File f, String checksumValue) - throws IOException; + public void setExtendedFileAttribute(Path p, ExtendedAttributes name, String value) throws IOException; - public String getChecksumAttribute(File f) throws IOException; - - public String getChecksumAttribute(Path p) throws IOException; + public void setExtendedFileAttribute(File f, ExtendedAttributes name, String value) throws IOException; - public boolean fileSupportsExtendedAttributes(File f) throws IOException; + public String getExtendedFileAttributeValue(File f, ExtendedAttributes name) throws IOException; + + public FileStat stat(Path p) throws IOException; + + public FileStat stat(File f) throws IOException; + + public FileStatus getFileStatus(Path p) throws IOException; + + public FileStatus getFileStatus(File f) throws IOException; + + public boolean isStub(Path p) throws IOException; + + public boolean isStub(File f) throws IOException; + + public boolean isStub(FileStat fs) throws IOException; } diff --git a/src/main/java/org/italiangrid/storm/webdav/fs/attrs/FileStatus.java b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/FileStatus.java new file mode 100644 index 00000000..11d0417d --- /dev/null +++ b/src/main/java/org/italiangrid/storm/webdav/fs/attrs/FileStatus.java @@ -0,0 +1,6 @@ +package org.italiangrid.storm.webdav.fs.attrs; + +public enum FileStatus { + + TAPE, TAPE_RECALL_IN_PROGRESS, DISK, DISK_AND_TAPE, UNDEFINED; +} diff --git a/src/main/java/org/italiangrid/storm/webdav/milton/StoRMFileResource.java b/src/main/java/org/italiangrid/storm/webdav/milton/StoRMFileResource.java index aefbd19f..13c80f9b 100644 --- a/src/main/java/org/italiangrid/storm/webdav/milton/StoRMFileResource.java +++ b/src/main/java/org/italiangrid/storm/webdav/milton/StoRMFileResource.java @@ -17,6 +17,7 @@ import static io.milton.property.PropertySource.PropertyAccessibility.READ_ONLY; import static java.util.Objects.isNull; +import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME; import java.io.BufferedInputStream; import java.io.File; @@ -175,7 +176,8 @@ protected void calculateChecksum() { // do nothing, just read } - getExtendedAttributesHelper().setChecksumAttribute(getFile(), cis.getChecksumValue()); + getExtendedAttributesHelper().setExtendedFileAttribute(getFile(), + STORM_ADLER32_CHECKSUM_ATTR_NAME, cis.getChecksumValue()); } catch (IOException e) { throw new StoRMWebDAVError(e); @@ -188,7 +190,8 @@ public Object getProperty(QName name) { if (name.getNamespaceURI().equals(STORM_NAMESPACE_URI)) { if (name.getLocalPart().equals(PROPERTY_CHECKSUM)) { try { - return getExtendedAttributesHelper().getChecksumAttribute(getFile()); + return getExtendedAttributesHelper().getExtendedFileAttributeValue(getFile(), + STORM_ADLER32_CHECKSUM_ATTR_NAME); } catch (IOException e) { logger.warn("Errror getting checksum value for file: {}", getFile().getAbsolutePath(), e); return null; diff --git a/src/main/java/org/italiangrid/storm/webdav/milton/util/EarlyChecksumStrategy.java b/src/main/java/org/italiangrid/storm/webdav/milton/util/EarlyChecksumStrategy.java index aa5ea31e..a4b58d83 100644 --- a/src/main/java/org/italiangrid/storm/webdav/milton/util/EarlyChecksumStrategy.java +++ b/src/main/java/org/italiangrid/storm/webdav/milton/util/EarlyChecksumStrategy.java @@ -15,6 +15,8 @@ */ package org.italiangrid.storm.webdav.milton.util; +import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME; + import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -41,7 +43,8 @@ public void replaceContent(InputStream in, Long length, File targetFile) throws throw new StoRMWebDAVError("Incomplete copy error!"); } - attributesHelper.setChecksumAttribute(targetFile, cis.getChecksumValue()); + attributesHelper.setExtendedFileAttribute(targetFile, STORM_ADLER32_CHECKSUM_ATTR_NAME, + cis.getChecksumValue()); } } diff --git a/src/main/java/org/italiangrid/storm/webdav/milton/util/LateChecksumStrategy.java b/src/main/java/org/italiangrid/storm/webdav/milton/util/LateChecksumStrategy.java index 1a116ef8..7d132b54 100644 --- a/src/main/java/org/italiangrid/storm/webdav/milton/util/LateChecksumStrategy.java +++ b/src/main/java/org/italiangrid/storm/webdav/milton/util/LateChecksumStrategy.java @@ -15,6 +15,8 @@ */ package org.italiangrid.storm.webdav.milton.util; +import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME; + import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; @@ -55,7 +57,8 @@ protected void calculateChecksum(File targetFile) { // do nothing, just read } - attributesHelper.setChecksumAttribute(targetFile, cis.getChecksumValue()); + attributesHelper.setExtendedFileAttribute(targetFile, STORM_ADLER32_CHECKSUM_ATTR_NAME, + cis.getChecksumValue()); } catch (IOException e) { throw new StoRMWebDAVError(e); diff --git a/src/main/java/org/italiangrid/storm/webdav/server/servlet/ChecksumFilter.java b/src/main/java/org/italiangrid/storm/webdav/server/servlet/ChecksumFilter.java index e3b33022..c3bc519b 100644 --- a/src/main/java/org/italiangrid/storm/webdav/server/servlet/ChecksumFilter.java +++ b/src/main/java/org/italiangrid/storm/webdav/server/servlet/ChecksumFilter.java @@ -17,6 +17,7 @@ import static com.google.common.base.Strings.isNullOrEmpty; import static java.lang.String.format; +import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME; import java.io.File; import java.io.IOException; @@ -34,7 +35,6 @@ import org.italiangrid.storm.webdav.server.PathResolver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; public class ChecksumFilter implements Filter { @@ -43,7 +43,6 @@ public class ChecksumFilter implements Filter { public static final Logger logger = LoggerFactory.getLogger(ChecksumFilter.class); - @Autowired public ChecksumFilter(ExtendedAttributesHelper attributeHelper, PathResolver resolver) { @@ -113,7 +112,8 @@ private void addChecksumHeader(HttpServletRequest request, try { - checksumValue = attributeHelper.getChecksumAttribute(f); + checksumValue = + attributeHelper.getExtendedFileAttributeValue(f, STORM_ADLER32_CHECKSUM_ATTR_NAME); } catch (IOException e) { diff --git a/src/main/java/org/italiangrid/storm/webdav/server/servlet/StoRMServlet.java b/src/main/java/org/italiangrid/storm/webdav/server/servlet/StoRMServlet.java index 03f83598..778b7258 100644 --- a/src/main/java/org/italiangrid/storm/webdav/server/servlet/StoRMServlet.java +++ b/src/main/java/org/italiangrid/storm/webdav/server/servlet/StoRMServlet.java @@ -17,6 +17,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Path; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; @@ -26,11 +27,14 @@ import org.eclipse.jetty.util.resource.Resource; import org.italiangrid.storm.webdav.config.OAuthProperties; import org.italiangrid.storm.webdav.config.ServiceConfigurationProperties; +import org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributesHelper; import org.italiangrid.storm.webdav.server.PathResolver; import org.italiangrid.storm.webdav.server.servlet.resource.StormResourceService; import org.italiangrid.storm.webdav.server.servlet.resource.StormResourceWrapper; import org.thymeleaf.TemplateEngine; +import jnr.posix.POSIX; + public class StoRMServlet extends DefaultServlet { /** @@ -43,15 +47,20 @@ public class StoRMServlet extends DefaultServlet { final ServiceConfigurationProperties serviceConfig; final OAuthProperties oauthProperties; final StormResourceService resourceService; + final POSIX posix; + final ExtendedAttributesHelper attributesHelper; public StoRMServlet(OAuthProperties oauthP, ServiceConfigurationProperties serviceConfig, - PathResolver resolver, TemplateEngine engine, StormResourceService rs) { + PathResolver resolver, TemplateEngine engine, StormResourceService rs, POSIX posix, + ExtendedAttributesHelper attributesHelper) { super(rs); oauthProperties = oauthP; resourceService = rs; pathResolver = resolver; templateEngine = engine; this.serviceConfig = serviceConfig; + this.posix = posix; + this.attributesHelper = attributesHelper; } @Override @@ -69,9 +78,9 @@ public Resource getResource(String pathInContext) { return null; } - return new StormResourceWrapper(oauthProperties, serviceConfig, templateEngine, - Resource.newResource(f)); - + return new StormResourceWrapper(oauthProperties, serviceConfig, templateEngine, resolvedPath, + Resource.newResource(Path.of(resolvedPath)), posix.stat(resolvedPath), posix, + attributesHelper); } @Override diff --git a/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormFsResourceView.java b/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormFsResourceView.java index 2add60c2..c7fae5fc 100644 --- a/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormFsResourceView.java +++ b/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormFsResourceView.java @@ -25,26 +25,33 @@ public class StormFsResourceView { final String path; + final String fullPath; + final long sizeInBytes; final Date lastModificationTime; final Date creationTime; + final boolean isOnline; + + final boolean isRecallInProgress; + private StormFsResourceView(Builder b) { if (b.isDirectory && !b.name.endsWith("/")) { - this.name = b.name +"/"; + this.name = b.name + "/"; } else { this.name = b.name; } - + this.isDirectory = b.isDirectory; this.path = b.path; + this.fullPath = b.fullPath; this.sizeInBytes = b.sizeInBytes; this.lastModificationTime = b.lastModificationTime; this.creationTime = b.creationTime; - - + this.isOnline = b.isOnline; + this.isRecallInProgress = b.isRecallInProgress; } public String getName() { @@ -77,16 +84,32 @@ public Date getCreationTime() { } + public String getFullPath() { + return fullPath; + } + + public boolean getIsOnline() { + return isOnline; + } + + public boolean getIsRecallInProgress() { + return isRecallInProgress; + } + public static Builder builder() { return new Builder(); } + public static class Builder { String name; boolean isDirectory; String path; + String fullPath; long sizeInBytes; Date lastModificationTime; Date creationTime; + boolean isOnline; + boolean isRecallInProgress; public Builder() {} @@ -106,6 +129,11 @@ public Builder withPath(String path) { return this; } + public Builder withFullPath(String fullPath) { + this.fullPath = fullPath; + return this; + } + public Builder withSizeInBytes(long syzeInBytes) { this.sizeInBytes = syzeInBytes; return this; @@ -121,6 +149,16 @@ public Builder withCreationTime(Date creationTime) { return this; } + public Builder withIsOnline(boolean isOnline) { + this.isOnline = isOnline; + return this; + } + + public Builder withIsRecallInProgress(boolean isRecallInProgress) { + this.isRecallInProgress = isRecallInProgress; + return this; + } + public StormFsResourceView build() { return new StormFsResourceView(this); } diff --git a/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResource.java b/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResource.java new file mode 100644 index 00000000..67aa3b09 --- /dev/null +++ b/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResource.java @@ -0,0 +1,15 @@ +package org.italiangrid.storm.webdav.server.servlet.resource; + +import java.io.IOException; + +public interface StormResource { + + public boolean isOnline() throws IOException; + + public boolean isStub() throws IOException; + + public boolean hasMigrated() throws IOException; + + public boolean hasInProgressRecall() throws IOException; + +} diff --git a/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResourceWrapper.java b/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResourceWrapper.java index 73a030e9..bf1d34f9 100644 --- a/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResourceWrapper.java +++ b/src/main/java/org/italiangrid/storm/webdav/server/servlet/resource/StormResourceWrapper.java @@ -15,6 +15,9 @@ */ package org.italiangrid.storm.webdav.server.servlet.resource; +import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_MIGRATED_ATTR_NAME; +import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_RECALL_IN_PROGRESS_ATTR_NAME; + import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -22,6 +25,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.nio.channels.ReadableByteChannel; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -33,11 +37,16 @@ import org.italiangrid.storm.webdav.authn.AuthenticationUtils; import org.italiangrid.storm.webdav.config.OAuthProperties; import org.italiangrid.storm.webdav.config.ServiceConfigurationProperties; +import org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributesHelper; import org.springframework.security.core.context.SecurityContextHolder; import org.thymeleaf.TemplateEngine; import org.thymeleaf.context.Context; -public class StormResourceWrapper extends Resource { +import jnr.posix.FileStat; +import jnr.posix.POSIX; +import jnr.posix.POSIXFactory; + +public class StormResourceWrapper extends Resource implements StormResource { public static final String JETTY_DIR_TEMPLATE = "jetty-dir"; @@ -45,15 +54,23 @@ public class StormResourceWrapper extends Resource { final TemplateEngine engine; final OAuthProperties oauthProperties; final ServiceConfigurationProperties serviceConfig; + final String absoluteFilePath; + final FileStat fileStat; + final ExtendedAttributesHelper attrsHelper; + final POSIX posix; public StormResourceWrapper(OAuthProperties oauth, ServiceConfigurationProperties serviceConfig, - TemplateEngine engine, Resource delegate) { + TemplateEngine engine, String absoluteFilePath, Resource delegate, FileStat fileStat, + POSIX posix, ExtendedAttributesHelper attributesHelper) { this.oauthProperties = oauth; this.engine = engine; this.delegate = delegate; this.serviceConfig = serviceConfig; - + this.absoluteFilePath = absoluteFilePath; + this.fileStat = fileStat; + this.attrsHelper = attributesHelper; + this.posix = posix; } /** @@ -136,6 +153,7 @@ public String getListHTML(String base, boolean parent, String query) throws IOEx context.setVariable("oidcEnabled", oauthProperties.isEnableOidc()); String encodedBase = hrefEncodeURI(decodedBase); + String encodedFullBasePath = hrefEncodeURI(delegate.getFile().getCanonicalPath()); String parentDir = URIUtil.addPaths(encodedBase, "../"); @@ -144,13 +162,27 @@ public String getListHTML(String base, boolean parent, String query) throws IOEx List resources = new ArrayList<>(); for (String l : rawListing) { - Resource r = addPath(l); + + String absolutePath = String.format("%s/%s", absoluteFilePath, l); + File f = new File(absolutePath); + FileStat fStat = POSIXFactory.getPOSIX().stat(absolutePath); + + boolean isOnline = true; + boolean hasRecallInProgress = false; + boolean isDirectory = fStat.isDirectory(); + if (!isDirectory) { + isOnline = !isStub(fStat); + hasRecallInProgress = attrsHelper.getExtendedFileAttributeNames(f).contains(STORM_RECALL_IN_PROGRESS_ATTR_NAME.toString()); + } resources.add(StormFsResourceView.builder() .withName(l) .withPath(URIUtil.addEncodedPaths(encodedBase, URIUtil.encodePath(l))) - .withIsDirectory(r.isDirectory()) - .withLastModificationTime(new Date(r.lastModified())) - .withSizeInBytes(r.length()) + .withFullPath(URIUtil.addEncodedPaths(encodedFullBasePath, URIUtil.encodePath(l))) + .withIsOnline(isOnline) + .withIsRecallInProgress(hasRecallInProgress) + .withIsDirectory(isDirectory) + .withLastModificationTime(Date.from(Instant.ofEpochMilli(fStat.ctime() + fStat.mtime()))) + .withSizeInBytes(fStat.st_size()) .build()); } @@ -189,7 +221,8 @@ public long lastModified() { @Override public long length() { - return delegate.length(); + return fileStat.st_size(); + // return delegate.length(); } @SuppressWarnings("deprecation") @@ -289,5 +322,29 @@ private void internalCopy(InputStream in, OutputStream out) throws IOException { internalCopy(in, out, -1); } + @Override + public boolean isOnline() { + return !isStub(fileStat); + } + + @Override + public boolean isStub() { + return isStub(fileStat); + } + + @Override + public boolean hasMigrated() throws IOException { + return attrsHelper.getExtendedFileAttributeNames(getFile()) + .contains(STORM_MIGRATED_ATTR_NAME.toString()); + } + @Override + public boolean hasInProgressRecall() throws IOException { + return attrsHelper.getExtendedFileAttributeNames(getFile()) + .contains(STORM_RECALL_IN_PROGRESS_ATTR_NAME.toString()); + } + + public static boolean isStub(FileStat fileStat) { + return fileStat.blockSize() * fileStat.blocks() < fileStat.st_size(); + } } diff --git a/src/main/java/org/italiangrid/storm/webdav/spring/web/ServletConfiguration.java b/src/main/java/org/italiangrid/storm/webdav/spring/web/ServletConfiguration.java index b6813811..cccc1391 100644 --- a/src/main/java/org/italiangrid/storm/webdav/spring/web/ServletConfiguration.java +++ b/src/main/java/org/italiangrid/storm/webdav/spring/web/ServletConfiguration.java @@ -25,6 +25,7 @@ import org.italiangrid.storm.webdav.config.StorageAreaConfiguration; import org.italiangrid.storm.webdav.config.ThirdPartyCopyProperties; import org.italiangrid.storm.webdav.fs.FilesystemAccess; +import org.italiangrid.storm.webdav.fs.attrs.DefaultExtendedFileAttributesHelper; import org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributesHelper; import org.italiangrid.storm.webdav.macaroon.MacaroonIssuerService; import org.italiangrid.storm.webdav.macaroon.MacaroonRequestFilter; @@ -61,7 +62,7 @@ import com.codahale.metrics.servlets.MetricsServlet; import com.fasterxml.jackson.databind.ObjectMapper; - +import jnr.posix.POSIXFactory; @Configuration public class ServletConfiguration { @@ -225,7 +226,8 @@ ServletRegistrationBean stormServlet(OAuthProperties oauthProperti ServletRegistrationBean stormServlet = new ServletRegistrationBean<>(new StoRMServlet(oauthProperties, serviceConfig, pathResolver, - templateEngine, new StormResourceService())); + templateEngine, new StormResourceService(), POSIXFactory.getPOSIX(), + new DefaultExtendedFileAttributesHelper())); stormServlet.addInitParameter("acceptRanges", "true"); stormServlet.addInitParameter("dirAllowed", "true"); diff --git a/src/main/java/org/italiangrid/storm/webdav/tape/WlcgTapeRestApiController.java b/src/main/java/org/italiangrid/storm/webdav/tape/WlcgTapeRestApiController.java index ed355ef9..a25e6742 100644 --- a/src/main/java/org/italiangrid/storm/webdav/tape/WlcgTapeRestApiController.java +++ b/src/main/java/org/italiangrid/storm/webdav/tape/WlcgTapeRestApiController.java @@ -18,7 +18,7 @@ import static org.springframework.http.HttpStatus.NOT_FOUND; -import org.italiangrid.storm.webdav.tape.model.WlcgTapeRestApi; +import org.italiangrid.storm.webdav.tape.model.WlcgTapeRestApiMetadata; import org.italiangrid.storm.webdav.tape.service.WlcgTapeRestApiService; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; @@ -34,9 +34,9 @@ public WlcgTapeRestApiController(WlcgTapeRestApiService service) { } @GetMapping({".well-known/wlcg-tape-rest-api"}) - public WlcgTapeRestApi getMetadata() { + public WlcgTapeRestApiMetadata getMetadata() { - WlcgTapeRestApi metadata = service.getMetadata(); + WlcgTapeRestApiMetadata metadata = service.getMetadata(); if (metadata == null) { throw new ResponseStatusException(NOT_FOUND, "Unable to find resource"); } diff --git a/src/main/java/org/italiangrid/storm/webdav/tape/model/WlcgTapeRestApi.java b/src/main/java/org/italiangrid/storm/webdav/tape/model/WlcgTapeRestApiMetadata.java similarity index 97% rename from src/main/java/org/italiangrid/storm/webdav/tape/model/WlcgTapeRestApi.java rename to src/main/java/org/italiangrid/storm/webdav/tape/model/WlcgTapeRestApiMetadata.java index c3557680..d5eeece0 100644 --- a/src/main/java/org/italiangrid/storm/webdav/tape/model/WlcgTapeRestApi.java +++ b/src/main/java/org/italiangrid/storm/webdav/tape/model/WlcgTapeRestApiMetadata.java @@ -20,7 +20,7 @@ import com.google.common.collect.Lists; -public class WlcgTapeRestApi { +public class WlcgTapeRestApiMetadata { private String sitename; private String description; diff --git a/src/main/java/org/italiangrid/storm/webdav/tape/service/WlcgTapeRestApiService.java b/src/main/java/org/italiangrid/storm/webdav/tape/service/WlcgTapeRestApiService.java index 44158ba3..c5fc71c1 100644 --- a/src/main/java/org/italiangrid/storm/webdav/tape/service/WlcgTapeRestApiService.java +++ b/src/main/java/org/italiangrid/storm/webdav/tape/service/WlcgTapeRestApiService.java @@ -20,7 +20,7 @@ import java.io.IOException; import org.italiangrid.storm.webdav.config.ServiceConfigurationProperties; -import org.italiangrid.storm.webdav.tape.model.WlcgTapeRestApi; +import org.italiangrid.storm.webdav.tape.model.WlcgTapeRestApiMetadata; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -36,7 +36,7 @@ public class WlcgTapeRestApiService { private static final String LOG_ERROR_PREFIX = "Error loading WLCG Tape REST API well-known endpoint from file: {}"; private static final String LOG_INFO_NOFILEFOUND = "No WLCG Tape REST API well-known file found at '{}'"; - private WlcgTapeRestApi metadata; + private WlcgTapeRestApiMetadata metadata; public WlcgTapeRestApiService(ServiceConfigurationProperties props) { @@ -45,7 +45,7 @@ public WlcgTapeRestApiService(ServiceConfigurationProperties props) { if (source.exists()) { LOG.info(LOG_INFO_LOADING, source); try { - metadata = (new ObjectMapper()).readValue(source, WlcgTapeRestApi.class); + metadata = (new ObjectMapper()).readValue(source, WlcgTapeRestApiMetadata.class); } catch (IOException e) { LOG.error(LOG_ERROR_PREFIX, e.getMessage()); } @@ -54,7 +54,7 @@ public WlcgTapeRestApiService(ServiceConfigurationProperties props) { } } - public WlcgTapeRestApi getMetadata() { + public WlcgTapeRestApiMetadata getMetadata() { return metadata; } diff --git a/src/main/java/org/italiangrid/storm/webdav/tpc/http/GetResponseHandler.java b/src/main/java/org/italiangrid/storm/webdav/tpc/http/GetResponseHandler.java index 1a2941d6..41fada38 100644 --- a/src/main/java/org/italiangrid/storm/webdav/tpc/http/GetResponseHandler.java +++ b/src/main/java/org/italiangrid/storm/webdav/tpc/http/GetResponseHandler.java @@ -16,6 +16,7 @@ package org.italiangrid.storm.webdav.tpc.http; import static java.util.Objects.isNull; +import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME; import java.io.IOException; import java.io.InputStream; @@ -110,8 +111,8 @@ public Boolean handleResponse(HttpResponse response) throws IOException { writeEntityToStream(entity, os); if (computeChecksum) { - attributesHelper.setChecksumAttribute(fileStream.getPath(), - checkedStream.getChecksumValue()); + attributesHelper.setExtendedFileAttribute(fileStream.getPath(), + STORM_ADLER32_CHECKSUM_ATTR_NAME, checkedStream.getChecksumValue()); } } diff --git a/src/main/resources/templates/jetty-dir.html b/src/main/resources/templates/jetty-dir.html index 6963c475..a603271e 100644 --- a/src/main/resources/templates/jetty-dir.html +++ b/src/main/resources/templates/jetty-dir.html @@ -17,15 +17,22 @@

Directory

Name Last modified Size (in bytes) + Full Path + Is Online + Is Recalling - - fake + + fake + fake Last modification time Size in bytes + Full Path + Is Online + Is Recalling diff --git a/src/test/java/org/italiangrid/storm/webdav/test/tape/FileStubTest.java b/src/test/java/org/italiangrid/storm/webdav/test/tape/FileStubTest.java new file mode 100644 index 00000000..1ea3110e --- /dev/null +++ b/src/test/java/org/italiangrid/storm/webdav/test/tape/FileStubTest.java @@ -0,0 +1,134 @@ +/** + * Copyright (c) Istituto Nazionale di Fisica Nucleare, 2014-2023. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.italiangrid.storm.webdav.test.tape; + +import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_MIGRATED_ATTR_NAME; +import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_RECALL_IN_PROGRESS_ATTR_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; + +import org.italiangrid.storm.webdav.fs.attrs.DefaultExtendedFileAttributesHelper; +import org.italiangrid.storm.webdav.fs.attrs.FileStatus; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class FileStubTest { + + private final static String RESOURCE_BASE_PATH = "storage/tape"; + + private File stubFile; + private File undefinedFile; + private File onlineFile; + private File onlineAndMigratedFile; + private File recallInProgressFile; + private DefaultExtendedFileAttributesHelper helper = new DefaultExtendedFileAttributesHelper(); + + private void printProcess(Process p) throws IOException { + BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream())); + String str = br.readLine(); + while (str != null) { + System.out.println(str); + str = br.readLine(); + } + } + + private File createUndefinedFile(String fileName) throws IOException { + + ClassLoader classLoader = getClass().getClassLoader(); + File resourceBaseDir = new File(classLoader.getResource(RESOURCE_BASE_PATH).getFile()); + File undefinedFile = new File(resourceBaseDir + "/" + fileName); + printProcess(Runtime.getRuntime() + .exec("dd if=/dev/zero conv=sparse bs=1M count=1 of=" + undefinedFile.getAbsolutePath())); + return undefinedFile; + } + + private File createStubFile(String fileName) throws IOException { + + File stubFile = createUndefinedFile(fileName); + helper.setExtendedFileAttribute(stubFile, STORM_MIGRATED_ATTR_NAME, "y"); + return stubFile; + } + + private File createStubRecalledFile(String fileName) throws IOException { + + File stubRecalledFile = createStubFile(fileName); + helper.setExtendedFileAttribute(stubRecalledFile, STORM_RECALL_IN_PROGRESS_ATTR_NAME, "y"); + return stubRecalledFile; + } + + private File createOnlineFile(String fileName) throws IOException { + + ClassLoader classLoader = getClass().getClassLoader(); + File resourceBaseDir = new File(classLoader.getResource(RESOURCE_BASE_PATH).getFile()); + File onlineFile = new File(resourceBaseDir + "/" + fileName); + return onlineFile; + } + + private File createOnlineAndMigratedFile(String fileName) throws IOException { + + File migratedFile = createOnlineFile(fileName); + helper.setExtendedFileAttribute(migratedFile, STORM_MIGRATED_ATTR_NAME, "y"); + return migratedFile; + } + + @BeforeEach + public void setup() throws IOException { + + undefinedFile = createUndefinedFile("undefined.dat"); + stubFile = createStubFile("tape.dat"); + recallInProgressFile = createStubRecalledFile("tape-recalled.dat"); + onlineFile = createOnlineFile("disk.dat"); + onlineAndMigratedFile = createOnlineAndMigratedFile("disk-and-tape.dat"); + } + + @AfterEach + public void finalize() throws IOException { + + } + + @Test + public void testUndefinedFile() throws IOException { + assertEquals(FileStatus.UNDEFINED, helper.getFileStatus(undefinedFile)); + } + + @Test + public void testStubFile() throws IOException { + assertEquals(FileStatus.TAPE, helper.getFileStatus(stubFile)); + } + + @Test + public void testOnlineFile() throws IOException { + assertEquals(FileStatus.DISK, helper.getFileStatus(onlineFile)); + } + + @Test + public void testOnlineAndMigratedFile() throws IOException { + assertEquals(FileStatus.DISK_AND_TAPE, helper.getFileStatus(onlineAndMigratedFile)); + } + + @Test + public void testRecallInProgressFile() throws IOException { + assertEquals(FileStatus.TAPE_RECALL_IN_PROGRESS, helper.getFileStatus(recallInProgressFile)); + } +} diff --git a/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/GetResponseHandlerTest.java b/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/GetResponseHandlerTest.java index 257b060c..a111ec98 100644 --- a/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/GetResponseHandlerTest.java +++ b/src/test/java/org/italiangrid/storm/webdav/test/tpc/http/GetResponseHandlerTest.java @@ -15,6 +15,7 @@ */ package org.italiangrid.storm.webdav.test.tpc.http; +import static org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes.STORM_ADLER32_CHECKSUM_ATTR_NAME; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.verify; @@ -26,6 +27,7 @@ import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.StatusLine; +import org.italiangrid.storm.webdav.fs.attrs.ExtendedAttributes; import org.italiangrid.storm.webdav.tpc.http.GetResponseHandler; import org.italiangrid.storm.webdav.tpc.utils.StormCountingOutputStream; import org.junit.jupiter.api.BeforeEach; @@ -40,26 +42,26 @@ public class GetResponseHandlerTest extends ClientTestSupport { @Mock StatusLine status; - + @Mock HttpEntity entity; - + @Mock HttpResponse response; - + @Mock StormCountingOutputStream os; - + GetResponseHandler handler; @BeforeEach public void setup() { - + handler = new GetResponseHandler(null, os, eah); lenient().when(response.getStatusLine()).thenReturn(status); lenient().when(response.getEntity()).thenReturn(entity); } - + @Test public void handlerWritesToStream() throws IOException { when(status.getStatusCode()).thenReturn(200); @@ -67,6 +69,6 @@ public void handlerWritesToStream() throws IOException { handler.handleResponse(response); verify(entity).getContent(); - verify(eah).setChecksumAttribute(ArgumentMatchers.any(), any()); + verify(eah).setExtendedFileAttribute(ArgumentMatchers.any(), ArgumentMatchers.eq(STORM_ADLER32_CHECKSUM_ATTR_NAME), any()); } }