diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml index 5f8e99dc..875d9e18 100644 --- a/dependency-reduced-pom.xml +++ b/dependency-reduced-pom.xml @@ -10,7 +10,7 @@ org.embl.mobie mobie-io MoBIE IO - 2.2.2-SNAPSHOT + 2.2.3-SNAPSHOT Readers and writers for image data in MoBIE projects https://github.com/mobie/mobie-io @@ -179,6 +179,18 @@ 2.2.7 compile + + ome + bio-formats_plugins + 6.14.0 + compile + + + logback-classic + ch.qos.logback + + + com.google.http-client google-http-client diff --git a/pom.xml b/pom.xml index 0fa3f668..0e86bbb5 100644 --- a/pom.xml +++ b/pom.xml @@ -197,6 +197,10 @@ sc.fiji spim_data + + ome + bio-formats_plugins + org.janelia.saalfeldlab n5-aws-s3 diff --git a/src/main/java/org/embl/mobie/io/ImageDataFormat.java b/src/main/java/org/embl/mobie/io/ImageDataFormat.java index c8d9425c..301138fc 100644 --- a/src/main/java/org/embl/mobie/io/ImageDataFormat.java +++ b/src/main/java/org/embl/mobie/io/ImageDataFormat.java @@ -30,22 +30,7 @@ import com.google.gson.annotations.SerializedName; -import static org.embl.mobie.io.ImageDataFormatNames.BDV; -import static org.embl.mobie.io.ImageDataFormatNames.BDVHDF5; -import static org.embl.mobie.io.ImageDataFormatNames.BDVN5; -import static org.embl.mobie.io.ImageDataFormatNames.BDVN5S3; -import static org.embl.mobie.io.ImageDataFormatNames.BDVOMEZARR; -import static org.embl.mobie.io.ImageDataFormatNames.BDVOMEZARRS3; -import static org.embl.mobie.io.ImageDataFormatNames.BIOFORMATS; -import static org.embl.mobie.io.ImageDataFormatNames.ILASTIKHDF5; -import static org.embl.mobie.io.ImageDataFormatNames.IMAGEJ; -import static org.embl.mobie.io.ImageDataFormatNames.SPIMDATA; -import static org.embl.mobie.io.ImageDataFormatNames.IMARIS; -import static org.embl.mobie.io.ImageDataFormatNames.OMEZARR; -import static org.embl.mobie.io.ImageDataFormatNames.OMEZARRS3; -import static org.embl.mobie.io.ImageDataFormatNames.OPENORGANELLES3; -import static org.embl.mobie.io.ImageDataFormatNames.TIFF; -import static org.embl.mobie.io.ImageDataFormatNames.TOML; +import static org.embl.mobie.io.ImageDataFormatNames.*; /** * Currently mobie-io supports the following data formats: @@ -94,6 +79,8 @@ public enum ImageDataFormat { ImageJ, @SerializedName(BIOFORMATS) BioFormats, + @SerializedName(BIOFORMATSS3) + BioFormatsS3, @SerializedName(BDV) Bdv, @SerializedName(BDVHDF5) @@ -127,6 +114,8 @@ public static ImageDataFormat fromString(String string) { return ImageJ; case BIOFORMATS: return BioFormats; + case BIOFORMATSS3: + return BioFormatsS3; case BDV: return Bdv; case BDVHDF5: @@ -167,6 +156,8 @@ public String toString() { return IMAGEJ; case BioFormats: return BIOFORMATS; + case BioFormatsS3: + return BIOFORMATSS3; case Bdv: return BDV; case BdvHDF5: @@ -231,6 +222,7 @@ public boolean isRemote() { case OmeZarrS3: case BdvOmeZarrS3: case OpenOrganelleS3: + case BioFormatsS3: return true; case BdvN5: case BdvOmeZarr: diff --git a/src/main/java/org/embl/mobie/io/ImageDataFormatNames.java b/src/main/java/org/embl/mobie/io/ImageDataFormatNames.java index 53c9e9d9..e9a271a3 100644 --- a/src/main/java/org/embl/mobie/io/ImageDataFormatNames.java +++ b/src/main/java/org/embl/mobie/io/ImageDataFormatNames.java @@ -34,6 +34,7 @@ public class ImageDataFormatNames public static final String TIFF = "tiff"; public static final String IMAGEJ = "imagej"; public static final String BIOFORMATS = "bioformats"; + public static final String BIOFORMATSS3 = "bioformats.s3"; public static final String BDV = "bdv"; public static final String BDVN5 = "bdv.n5"; public static final String BDVN5S3 = "bdv.n5.s3"; diff --git a/src/main/java/org/embl/mobie/io/SpimDataOpener.java b/src/main/java/org/embl/mobie/io/SpimDataOpener.java index cfcaef20..5f673027 100644 --- a/src/main/java/org/embl/mobie/io/SpimDataOpener.java +++ b/src/main/java/org/embl/mobie/io/SpimDataOpener.java @@ -98,7 +98,9 @@ public SpimDataOpener() { case ImageJ: return open(IJ.openImage(imagePath)); case BioFormats: - return openWithBioFormats(imagePath); + return openWithBDVBioFormats(imagePath); + case BioFormatsS3: + return openWithBioFormatsFromS3(imagePath, 0, null ); case Imaris: return openImaris(imagePath); case Bdv: @@ -130,12 +132,12 @@ public SpimDataOpener() { case ImageJ: ImagePlus imagePlus = IJ.openImage( imagePath ); if ( imagePlus == null ) - { throw new RuntimeException("Could not open " + imagePath ); - } return open( imagePlus, sharedQueue); case BioFormats: - return openWithBioFormats(imagePath, sharedQueue); + return openWithBDVBioFormats(imagePath, sharedQueue); + case BioFormatsS3: + return openWithBioFormatsFromS3(imagePath, 0, sharedQueue ); case BdvN5: return openBdvN5(imagePath, sharedQueue); case BdvN5S3: @@ -162,9 +164,7 @@ public AbstractSpimData open( ImagePlus imagePlus ) public AbstractSpimData open( ImagePlus imagePlus, SharedQueue sharedQueue ) { final AbstractSpimData< ? > spimData = open( imagePlus ); - setSharedQueue( sharedQueue, spimData ); - return spimData; } @@ -298,7 +298,7 @@ private SpimData openBdvOmeZarrS3(String path, SharedQueue queue) { } } - public AbstractSpimData< ? > openWithBioFormats( String path ) + public AbstractSpimData< ? > openWithBDVBioFormats( String path ) { final File file = new File( path ); List< OpenerSettings > openerSettings = new ArrayList<>(); @@ -312,20 +312,28 @@ private SpimData openBdvOmeZarrS3(String path, SharedQueue queue) { return OpenersToSpimData.getSpimData( openerSettings ); } - public AbstractSpimData< ? > openWithBioFormats( String path, SharedQueue sharedQueue ) + public AbstractSpimData< ? > openWithBDVBioFormats( String path, SharedQueue sharedQueue ) { - final AbstractSpimData< ? > spimData = openWithBioFormats( path ); + final AbstractSpimData< ? > spimData = openWithBDVBioFormats( path ); setSharedQueue( sharedQueue, spimData ); return spimData; } -// public static SpimData asSpimData( AbstractSpimData< ? > abstractSpimData ) -// { -// final AbstractSequenceDescription< ?, ?, ? > abstractSequenceDescription = abstractSpimData.getSequenceDescription(); -// final SequenceDescription sequenceDescription = new SequenceDescription( abstractSequenceDescription.getTimePoints(), abstractSequenceDescription.getViewSetups() ); -// final SpimData spimData = new SpimData( abstractSpimData.getBasePath(), sequenceDescription, abstractSpimData.getViewRegistrations() ); -// return spimData; -// } + // TODO: Currently does not support resolution pyramids + // + public AbstractSpimData< ? > openWithBioFormatsFromS3( String path, int seriesIndex, SharedQueue sharedQueue ) + { + ImagePlus imagePlus = IOHelper.openWithBioformatsFromS3( path, seriesIndex ); + + if ( sharedQueue != null ) + { + return open( imagePlus, sharedQueue ); + } + else + { + return open( imagePlus ); + } + } private SpimData openBdvOmeZarr(String path, @Nullable SharedQueue sharedQueue) throws SpimDataException { SpimData spimData = openBdvXml(path); diff --git a/src/main/java/org/embl/mobie/io/util/IOHelper.java b/src/main/java/org/embl/mobie/io/util/IOHelper.java index 8da615bf..00f9999a 100644 --- a/src/main/java/org/embl/mobie/io/util/IOHelper.java +++ b/src/main/java/org/embl/mobie/io/util/IOHelper.java @@ -28,14 +28,7 @@ */ package org.embl.mobie.io.util; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; +import java.io.*; import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; @@ -55,11 +48,15 @@ import ij.ImagePlus; import ij.io.Opener; +import loci.common.ByteArrayHandle; +import loci.common.Location; +import loci.plugins.in.ImagePlusReader; +import loci.plugins.in.ImportProcess; +import loci.plugins.in.ImporterOptions; import org.apache.commons.io.IOUtils; import org.embl.mobie.io.github.GitHubUtils; import com.amazonaws.services.s3.AmazonS3; -import org.jetbrains.annotations.NotNull; import static org.embl.mobie.io.github.GitHubUtils.isGithub; import static org.embl.mobie.io.github.GitHubUtils.selectGitHubPathFromDirectory; @@ -134,6 +131,26 @@ public static List getFiles(File inputDirectory, String filePattern) { return paths; } + public static ImagePlus openWithBioFormats( String path, int seriesIndex ) + { + try + { + ImporterOptions opts = new ImporterOptions(); + opts.setId( path ); + opts.setVirtual( true ); + opts.setSeriesOn( seriesIndex, true ); + ImportProcess process = new ImportProcess( opts ); + process.execute(); + ImagePlusReader impReader = new ImagePlusReader( process ); + ImagePlus[] imps = impReader.openImagePlus(); + return imps[ 0 ]; + } + catch ( Exception e ) + { + throw new RuntimeException("Could not open " + path ); + } + } + public static String getSeparator(String uri) { IOHelper.ResourceType type = getType(uri); switch (type) { @@ -306,7 +323,7 @@ public static String[] getFileNames(String uri) { } case FILE: List files = getFileList(new File(uri), ".*", false); - if (files.size() > 0) { + if ( !files.isEmpty() ) { String[] fileNames = new String[files.size()]; for (int i = 0; i < files.size(); i++) { fileNames[i] = files.get(i).getName(); @@ -476,6 +493,31 @@ public static List< String > getPaths( String dir, String regex, int maxDepth ) } } + public static ImagePlus openWithBioformatsFromS3( String path, int seriesIndex ) + { + try + { + InputStream inputStream = getInputStream( path ); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + int nRead; + byte[] data = new byte[ 1024 ]; + while ( ( nRead = inputStream.read( data, 0, data.length ) ) != -1 ) + buffer.write( data, 0, nRead ); + buffer.flush(); + byte[] byteArray = buffer.toByteArray(); + //System.out.println( byteArray.length + " bytes read from S3." ); + Location.mapFile( "mapped_" + path, new ByteArrayHandle( byteArray ) ); + ImagePlus imagePlus = openWithBioFormats( "mapped_" + path, seriesIndex ); + //System.out.println( "S3 [ms]: " + ( System.currentTimeMillis() - start ) ); + return imagePlus; + } + catch ( IOException e ) + { + throw new RuntimeException( e ); + } + + } + public enum ResourceType { FILE, // resource is a file on the file system diff --git a/src/main/java/org/embl/mobie/io/util/S3Utils.java b/src/main/java/org/embl/mobie/io/util/S3Utils.java index d7d73569..3e77642f 100644 --- a/src/main/java/org/embl/mobie/io/util/S3Utils.java +++ b/src/main/java/org/embl/mobie/io/util/S3Utils.java @@ -44,11 +44,8 @@ import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.HeadBucketRequest; -import com.amazonaws.services.s3.model.HeadBucketResult; -import com.amazonaws.services.s3.model.ListObjectsV2Request; -import com.amazonaws.services.s3.model.ListObjectsV2Result; -import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.iterable.S3Objects; +import com.amazonaws.services.s3.model.*; import com.google.api.client.http.HttpStatusCodes; import ij.gui.GenericDialog; @@ -194,17 +191,13 @@ public static ArrayList getS3FilePaths(String directory) { final String[] bucketAndObject = getBucketAndObject(directory); final String bucket = bucketAndObject[0]; - final String prefix = (bucketAndObject[1] == "") ? "" : (bucketAndObject[1] + "/"); + final String prefix = bucketAndObject[ 1 ].isEmpty() ? "" : ( bucketAndObject[1] + "/" ); - ListObjectsV2Request request = new ListObjectsV2Request() - .withBucketName(bucket) - .withPrefix(prefix) - .withDelimiter("/"); - ListObjectsV2Result files = s3.listObjectsV2(request); final ArrayList paths = new ArrayList<>(); - for (S3ObjectSummary summary : files.getObjectSummaries()) { - paths.add(summary.getKey()); - } + S3Objects.withPrefix( s3, bucket, prefix ).forEach( (S3ObjectSummary objectSummary ) -> { + String path = IOHelper.combinePath( directory, objectSummary.getKey().replace( prefix, "" ) ); + paths.add( path ); + }); return paths; } diff --git a/src/test/java/develop/ListS3BucketContents.java b/src/test/java/develop/ListS3BucketContents.java new file mode 100644 index 00000000..7611223c --- /dev/null +++ b/src/test/java/develop/ListS3BucketContents.java @@ -0,0 +1,13 @@ +package develop; + +import org.embl.mobie.io.util.S3Utils; + +import java.util.ArrayList; + +public class ListS3BucketContents +{ + public static void main( String[] args ) + { + ArrayList< String > s3FilePaths = S3Utils.getS3FilePaths( "https://s3.embl.de/i2k-2020/incu-test-data/2207/19" ); + } +}