Skip to content

Commit

Permalink
Add functionality to open with BioFormats from S3
Browse files Browse the repository at this point in the history
  • Loading branch information
@tischi
tischi committed Dec 16, 2023
1 parent 1fe0c15 commit b9c9daf
Show file tree
Hide file tree
Showing 8 changed files with 122 additions and 57 deletions.
14 changes: 13 additions & 1 deletion dependency-reduced-pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
<groupId>org.embl.mobie</groupId>
<artifactId>mobie-io</artifactId>
<name>MoBIE IO</name>
<version>2.2.2-SNAPSHOT</version>
<version>2.2.3-SNAPSHOT</version>
<description>Readers and writers for image data in MoBIE projects</description>
<url>https://github.com/mobie/mobie-io</url>
<issueManagement>
Expand Down Expand Up @@ -179,6 +179,18 @@
<version>2.2.7</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>ome</groupId>
<artifactId>bio-formats_plugins</artifactId>
<version>6.14.0</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<artifactId>logback-classic</artifactId>
<groupId>ch.qos.logback</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>
Expand Down
4 changes: 4 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,10 @@
<groupId>sc.fiji</groupId>
<artifactId>spim_data</artifactId>
</dependency>
<dependency>
<groupId>ome</groupId>
<artifactId>bio-formats_plugins</artifactId>
</dependency>
<dependency>
<groupId>org.janelia.saalfeldlab</groupId>
<artifactId>n5-aws-s3</artifactId>
Expand Down
24 changes: 8 additions & 16 deletions src/main/java/org/embl/mobie/io/ImageDataFormat.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,22 +30,7 @@

import com.google.gson.annotations.SerializedName;

import static org.embl.mobie.io.ImageDataFormatNames.BDV;
import static org.embl.mobie.io.ImageDataFormatNames.BDVHDF5;
import static org.embl.mobie.io.ImageDataFormatNames.BDVN5;
import static org.embl.mobie.io.ImageDataFormatNames.BDVN5S3;
import static org.embl.mobie.io.ImageDataFormatNames.BDVOMEZARR;
import static org.embl.mobie.io.ImageDataFormatNames.BDVOMEZARRS3;
import static org.embl.mobie.io.ImageDataFormatNames.BIOFORMATS;
import static org.embl.mobie.io.ImageDataFormatNames.ILASTIKHDF5;
import static org.embl.mobie.io.ImageDataFormatNames.IMAGEJ;
import static org.embl.mobie.io.ImageDataFormatNames.SPIMDATA;
import static org.embl.mobie.io.ImageDataFormatNames.IMARIS;
import static org.embl.mobie.io.ImageDataFormatNames.OMEZARR;
import static org.embl.mobie.io.ImageDataFormatNames.OMEZARRS3;
import static org.embl.mobie.io.ImageDataFormatNames.OPENORGANELLES3;
import static org.embl.mobie.io.ImageDataFormatNames.TIFF;
import static org.embl.mobie.io.ImageDataFormatNames.TOML;
import static org.embl.mobie.io.ImageDataFormatNames.*;

/**
* Currently mobie-io supports the following data formats:
Expand Down Expand Up @@ -94,6 +79,8 @@ public enum ImageDataFormat {
ImageJ,
@SerializedName(BIOFORMATS)
BioFormats,
@SerializedName(BIOFORMATSS3)
BioFormatsS3,
@SerializedName(BDV)
Bdv,
@SerializedName(BDVHDF5)
Expand Down Expand Up @@ -127,6 +114,8 @@ public static ImageDataFormat fromString(String string) {
return ImageJ;
case BIOFORMATS:
return BioFormats;
case BIOFORMATSS3:
return BioFormatsS3;
case BDV:
return Bdv;
case BDVHDF5:
Expand Down Expand Up @@ -167,6 +156,8 @@ public String toString() {
return IMAGEJ;
case BioFormats:
return BIOFORMATS;
case BioFormatsS3:
return BIOFORMATSS3;
case Bdv:
return BDV;
case BdvHDF5:
Expand Down Expand Up @@ -231,6 +222,7 @@ public boolean isRemote() {
case OmeZarrS3:
case BdvOmeZarrS3:
case OpenOrganelleS3:
case BioFormatsS3:
return true;
case BdvN5:
case BdvOmeZarr:
Expand Down
1 change: 1 addition & 0 deletions src/main/java/org/embl/mobie/io/ImageDataFormatNames.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ public class ImageDataFormatNames
public static final String TIFF = "tiff";
public static final String IMAGEJ = "imagej";
public static final String BIOFORMATS = "bioformats";
public static final String BIOFORMATSS3 = "bioformats.s3";
public static final String BDV = "bdv";
public static final String BDVN5 = "bdv.n5";
public static final String BDVN5S3 = "bdv.n5.s3";
Expand Down
40 changes: 24 additions & 16 deletions src/main/java/org/embl/mobie/io/SpimDataOpener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,9 @@ public SpimDataOpener() {
case ImageJ:
return open(IJ.openImage(imagePath));
case BioFormats:
return openWithBioFormats(imagePath);
return openWithBDVBioFormats(imagePath);
case BioFormatsS3:
return openWithBioFormatsFromS3(imagePath, 0, null );
case Imaris:
return openImaris(imagePath);
case Bdv:
Expand Down Expand Up @@ -130,12 +132,12 @@ public SpimDataOpener() {
case ImageJ:
ImagePlus imagePlus = IJ.openImage( imagePath );
if ( imagePlus == null )
{
throw new RuntimeException("Could not open " + imagePath );
}
return open( imagePlus, sharedQueue);
case BioFormats:
return openWithBioFormats(imagePath, sharedQueue);
return openWithBDVBioFormats(imagePath, sharedQueue);
case BioFormatsS3:
return openWithBioFormatsFromS3(imagePath, 0, sharedQueue );
case BdvN5:
return openBdvN5(imagePath, sharedQueue);
case BdvN5S3:
Expand All @@ -162,9 +164,7 @@ public AbstractSpimData open( ImagePlus imagePlus )
public AbstractSpimData open( ImagePlus imagePlus, SharedQueue sharedQueue )
{
final AbstractSpimData< ? > spimData = open( imagePlus );

setSharedQueue( sharedQueue, spimData );

return spimData;
}

Expand Down Expand Up @@ -298,7 +298,7 @@ private SpimData openBdvOmeZarrS3(String path, SharedQueue queue) {
}
}

public AbstractSpimData< ? > openWithBioFormats( String path )
public AbstractSpimData< ? > openWithBDVBioFormats( String path )
{
final File file = new File( path );
List< OpenerSettings > openerSettings = new ArrayList<>();
Expand All @@ -312,20 +312,28 @@ private SpimData openBdvOmeZarrS3(String path, SharedQueue queue) {
return OpenersToSpimData.getSpimData( openerSettings );
}

public AbstractSpimData< ? > openWithBioFormats( String path, SharedQueue sharedQueue )
public AbstractSpimData< ? > openWithBDVBioFormats( String path, SharedQueue sharedQueue )
{
final AbstractSpimData< ? > spimData = openWithBioFormats( path );
final AbstractSpimData< ? > spimData = openWithBDVBioFormats( path );
setSharedQueue( sharedQueue, spimData );
return spimData;
}

// public static SpimData asSpimData( AbstractSpimData< ? > abstractSpimData )
// {
// final AbstractSequenceDescription< ?, ?, ? > abstractSequenceDescription = abstractSpimData.getSequenceDescription();
// final SequenceDescription sequenceDescription = new SequenceDescription( abstractSequenceDescription.getTimePoints(), abstractSequenceDescription.getViewSetups() );
// final SpimData spimData = new SpimData( abstractSpimData.getBasePath(), sequenceDescription, abstractSpimData.getViewRegistrations() );
// return spimData;
// }
// TODO: Currently does not support resolution pyramids
//
public AbstractSpimData< ? > openWithBioFormatsFromS3( String path, int seriesIndex, SharedQueue sharedQueue )
{
ImagePlus imagePlus = IOHelper.openWithBioformatsFromS3( path, seriesIndex );

if ( sharedQueue != null )
{
return open( imagePlus, sharedQueue );
}
else
{
return open( imagePlus );
}
}

private SpimData openBdvOmeZarr(String path, @Nullable SharedQueue sharedQueue) throws SpimDataException {
SpimData spimData = openBdvXml(path);
Expand Down
62 changes: 52 additions & 10 deletions src/main/java/org/embl/mobie/io/util/IOHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,7 @@
*/
package org.embl.mobie.io.util;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URISyntaxException;
Expand All @@ -55,11 +48,15 @@

import ij.ImagePlus;
import ij.io.Opener;
import loci.common.ByteArrayHandle;
import loci.common.Location;
import loci.plugins.in.ImagePlusReader;
import loci.plugins.in.ImportProcess;
import loci.plugins.in.ImporterOptions;
import org.apache.commons.io.IOUtils;
import org.embl.mobie.io.github.GitHubUtils;

import com.amazonaws.services.s3.AmazonS3;
import org.jetbrains.annotations.NotNull;

import static org.embl.mobie.io.github.GitHubUtils.isGithub;
import static org.embl.mobie.io.github.GitHubUtils.selectGitHubPathFromDirectory;
Expand Down Expand Up @@ -134,6 +131,26 @@ public static List<String> getFiles(File inputDirectory, String filePattern) {
return paths;
}

public static ImagePlus openWithBioFormats( String path, int seriesIndex )
{
try
{
ImporterOptions opts = new ImporterOptions();
opts.setId( path );
opts.setVirtual( true );
opts.setSeriesOn( seriesIndex, true );
ImportProcess process = new ImportProcess( opts );
process.execute();
ImagePlusReader impReader = new ImagePlusReader( process );
ImagePlus[] imps = impReader.openImagePlus();
return imps[ 0 ];
}
catch ( Exception e )
{
throw new RuntimeException("Could not open " + path );
}
}

public static String getSeparator(String uri) {
IOHelper.ResourceType type = getType(uri);
switch (type) {
Expand Down Expand Up @@ -306,7 +323,7 @@ public static String[] getFileNames(String uri) {
}
case FILE:
List<File> files = getFileList(new File(uri), ".*", false);
if (files.size() > 0) {
if ( !files.isEmpty() ) {
String[] fileNames = new String[files.size()];
for (int i = 0; i < files.size(); i++) {
fileNames[i] = files.get(i).getName();
Expand Down Expand Up @@ -476,6 +493,31 @@ public static List< String > getPaths( String dir, String regex, int maxDepth )
}
}

public static ImagePlus openWithBioformatsFromS3( String path, int seriesIndex )
{
try
{
InputStream inputStream = getInputStream( path );
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
int nRead;
byte[] data = new byte[ 1024 ];
while ( ( nRead = inputStream.read( data, 0, data.length ) ) != -1 )
buffer.write( data, 0, nRead );
buffer.flush();
byte[] byteArray = buffer.toByteArray();
//System.out.println( byteArray.length + " bytes read from S3." );
Location.mapFile( "mapped_" + path, new ByteArrayHandle( byteArray ) );
ImagePlus imagePlus = openWithBioFormats( "mapped_" + path, seriesIndex );
//System.out.println( "S3 [ms]: " + ( System.currentTimeMillis() - start ) );
return imagePlus;
}
catch ( IOException e )
{
throw new RuntimeException( e );
}

}


public enum ResourceType {
FILE, // resource is a file on the file system
Expand Down
21 changes: 7 additions & 14 deletions src/main/java/org/embl/mobie/io/util/S3Utils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,8 @@
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.HeadBucketRequest;
import com.amazonaws.services.s3.model.HeadBucketResult;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.iterable.S3Objects;
import com.amazonaws.services.s3.model.*;
import com.google.api.client.http.HttpStatusCodes;

import ij.gui.GenericDialog;
Expand Down Expand Up @@ -194,17 +191,13 @@ public static ArrayList<String> getS3FilePaths(String directory) {
final String[] bucketAndObject = getBucketAndObject(directory);

final String bucket = bucketAndObject[0];
final String prefix = (bucketAndObject[1] == "") ? "" : (bucketAndObject[1] + "/");
final String prefix = bucketAndObject[ 1 ].isEmpty() ? "" : ( bucketAndObject[1] + "/" );

ListObjectsV2Request request = new ListObjectsV2Request()
.withBucketName(bucket)
.withPrefix(prefix)
.withDelimiter("/");
ListObjectsV2Result files = s3.listObjectsV2(request);
final ArrayList<String> paths = new ArrayList<>();
for (S3ObjectSummary summary : files.getObjectSummaries()) {
paths.add(summary.getKey());
}
S3Objects.withPrefix( s3, bucket, prefix ).forEach( (S3ObjectSummary objectSummary ) -> {
String path = IOHelper.combinePath( directory, objectSummary.getKey().replace( prefix, "" ) );
paths.add( path );
});
return paths;
}

Expand Down
13 changes: 13 additions & 0 deletions src/test/java/develop/ListS3BucketContents.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package develop;

import org.embl.mobie.io.util.S3Utils;

import java.util.ArrayList;

public class ListS3BucketContents
{
public static void main( String[] args )
{
ArrayList< String > s3FilePaths = S3Utils.getS3FilePaths( "https://s3.embl.de/i2k-2020/incu-test-data/2207/19" );
}
}

0 comments on commit b9c9daf

Please sign in to comment.