Skip to content

Commit

Permalink
introduced an abstraction layer for all N5 dataset parameters and to … (
Browse files Browse the repository at this point in the history 
#201)

* introduced an abstraction layer for all N5 dataset parameters and to potentially expose parts of the image (5D -> 3D for OME-ZARR), this allows us to directly use the N5ImageLoader for example for reading OME-ZARR based datasets. It also supports pre-fetching of all dataset attributes, which can slow down cloud processing if not done.

I also added an instantiateN5Reader() method.

* Revise, clean up, add javadoc.

* BdvN5Format doesn't have to extend N5Properties.
  Added DefaultN5Properties instead.

* 5D container slicing happens by overriding the (now exposed)
  N5ImageLoader.prepareCachedImage(...) method.

* N5ImageLoader.prefetch() must be called by the client if desired.
  This will open a N5Reader if necessary, so can happen independent of
  N5ImageLoader.open().

---------

Co-authored-by: tpietzsch <[email protected]>
  • Loading branch information
@StephanPreibisch @tpietzsch
StephanPreibisch and tpietzsch authored Nov 6, 2024
1 parent 4d308f7 commit fe3f2da
Show file tree
Hide file tree
Showing 3 changed files with 258 additions and 48 deletions.
62 changes: 62 additions & 0 deletions src/main/java/bdv/img/n5/DefaultN5Properties.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*-
* #%L
* BigDataViewer core classes with minimal dependencies.
* %%
* Copyright (C) 2012 - 2024 BigDataViewer developers.
* %%
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
* #L%
*/
package bdv.img.n5;

import org.janelia.saalfeldlab.n5.DataType;
import org.janelia.saalfeldlab.n5.N5Reader;

public class DefaultN5Properties implements N5Properties
{
@Override
public String getDatasetPath( final int setupId, final int timepointId, final int level )
{
return BdvN5Format.getPathName( setupId, timepointId, level );
}

@Override
public DataType getDataType( final N5Reader n5, final int setupId )
{
final String path = BdvN5Format.getPathName( setupId );
return n5.getAttribute( path, BdvN5Format.DATA_TYPE_KEY, DataType.class );
}

@Override
public double[][] getMipmapResolutions( final N5Reader n5, final int setupId )
{
final String path = BdvN5Format.getPathName( setupId );
return n5.getAttribute( path, BdvN5Format.DOWNSAMPLING_FACTORS_KEY, double[][].class );
}

@Override
public long[] getDimensions( final N5Reader n5, final int setupId, final int timepointId, final int level )
{
final String path = getDatasetPath( setupId, timepointId, level );
return n5.getDatasetAttributes( path ).getDimensions();
}
}
222 changes: 174 additions & 48 deletions src/main/java/bdv/img/n5/N5ImageLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,19 +28,26 @@
*/
package bdv.img.n5;

import static bdv.img.n5.BdvN5Format.DATA_TYPE_KEY;
import static bdv.img.n5.BdvN5Format.DOWNSAMPLING_FACTORS_KEY;
import static bdv.img.n5.BdvN5Format.getPathName;
import static net.imglib2.cache.volatiles.LoadingStrategy.BLOCKING;
import static net.imglib2.cache.volatiles.LoadingStrategy.BUDGETED;

import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Future;
import java.util.function.Function;
import java.util.function.IntFunction;
import java.util.stream.IntStream;
import java.util.stream.Stream;

import org.janelia.saalfeldlab.n5.DataBlock;
import org.janelia.saalfeldlab.n5.DataType;
Expand All @@ -62,6 +69,8 @@
import mpicbg.spim.data.generic.sequence.ImgLoaderHint;
import mpicbg.spim.data.sequence.MultiResolutionImgLoader;
import mpicbg.spim.data.sequence.MultiResolutionSetupImgLoader;
import mpicbg.spim.data.sequence.TimePoint;
import mpicbg.spim.data.sequence.ViewId;
import mpicbg.spim.data.sequence.VoxelDimensions;
import net.imglib2.Dimensions;
import net.imglib2.FinalDimensions;
Expand Down Expand Up @@ -120,11 +129,109 @@ public File getN5File()
return new File( n5URI );
}

/**
* Create a new N5Reader.
* <p>
* Can be overridden by subclasses (rather than only having the option to
* provide an instance at construction time).
*
* @return a new N5Reader instance
*/
protected N5Reader instantiateN5Reader()
{
return new N5FSReader( getN5File().getAbsolutePath() );
}

/**
* Create a new N5Properties (provides metadata and resolves dataset paths).
* <p>
* Can be overridden by subclasses to adapt to different metadata schemes
* and storage layouts.
*
* @return a new N5Properties instance.
*/
protected N5Properties createN5PropertiesInstance()
{
return new DefaultN5Properties();
}

/**
* Touch all metadata in advance in parallel so the N5-API caches them.
* <p>
* {@code prefetch()} completes immediately. The prefetching is done
* asynchronously in a new {@code ForkJoinPool} with the indicated {@code
* parallelism} level. The returned {@code Future} can be used to wait for
* completion of the async prefetching.
*
* @param parallelism
* the parallelism level
*
* @return represents completion of the prefetching
*/
public Future< Void > prefetch( final int parallelism )
{
openReader();

final CompletableFuture< Void > future = new CompletableFuture<>();

new Thread( () -> {
final ForkJoinPool myPool = new ForkJoinPool( parallelism );
try
{
// prefetch all datatypes and MipmapResolutions
final Collection< ? extends BasicViewSetup > vss = seq.getViewSetupsOrdered();
final Map< Integer, Integer > setupIdToNumLevels = new ConcurrentHashMap<>();
myPool.submit( () ->
vss.parallelStream()
.mapToInt( BasicViewSetup::getId )
.forEach( setup -> {
n5properties.getDataType( n5, setup );
setupIdToNumLevels.put( setup, n5properties.getMipmapResolutions( n5, setup ).length );
} )
).join();
// System.out.println( "Pre-fetched " + vss.size() + " data types and mipmap resolutions." );

// prefetch all DatasetAttributes for all views
myPool.submit( () -> {
final Collection< TimePoint > tps = seq.getTimePoints().getTimePointsOrdered();
Stream< ViewId > viewIds = vss.parallelStream().flatMap(
setup -> tps.parallelStream().map(
tp -> new ViewId( tp.getId(), setup.getId() ) ) );

// filter missing ViewIds
if ( seq.getMissingViews() != null )
{
final Set< ViewId > missing = seq.getMissingViews().getMissingViews();
viewIds = viewIds.filter( v -> !missing.contains( v ) );
}

viewIds.forEach( viewId -> {
final int setup = viewId.getViewSetupId();
final int tp = viewId.getTimePointId();
final int numLevels = setupIdToNumLevels.get( setup );
IntStream.range( 0, numLevels ).parallel().forEach( level -> n5.getDatasetAttributes( n5properties.getDatasetPath( setup, tp, level ) ) );
} );
} ).join();
// System.out.println( "Pre-fetched dataset attributes." );

future.complete(null);
}
catch ( Exception e )
{
// no drama if this fails ... could be that some data is actually missing, one can still look at what's there
future.completeExceptionally(e);
}
myPool.shutdown();
} ).start();

return future;
}

private volatile boolean isOpen = false;
private SharedQueue createdSharedQueue;
private VolatileGlobalCellCache cache;
private N5Reader n5;

private N5Properties n5properties;

private int requestedNumFetcherThreads = -1;
private SharedQueue requestedSharedQueue;
Expand All @@ -141,6 +248,19 @@ public void setCreatedSharedQueue( final SharedQueue createdSharedQueue )
requestedSharedQueue = createdSharedQueue;
}

private synchronized void openReader()
{
if ( n5 == null )
{
n5 = instantiateN5Reader();
}

if ( n5properties == null )
{
n5properties = createN5PropertiesInstance();
}
}

private void open()
{
if ( !isOpen )
Expand All @@ -152,10 +272,7 @@ private void open()

try
{
if ( n5 == null )
{
n5 = new N5FSReader( getN5File().getAbsolutePath() );
}
openReader();

int maxNumLevels = 0;
final List< ? extends BasicViewSetup > setups = seq.getViewSetupsOrdered();
Expand Down Expand Up @@ -185,6 +302,7 @@ private void open()
}
}

// TODO: the N5 is not re-opened
/**
* Clear the cache. Images that were obtained from
* this loader before {@link #close()} will stop working. Requesting images
Expand Down Expand Up @@ -219,17 +337,15 @@ public void close()

private < T extends NativeType< T >, V extends Volatile< T > & NativeType< V > > SetupImgLoader< T, V > createSetupImgLoader( final int setupId ) throws IOException
{
final String pathName = getPathName( setupId );
final DataType dataType;
try
{
dataType = n5.getAttribute( pathName, DATA_TYPE_KEY, DataType.class );
final DataType dataType = n5properties.getDataType( n5, setupId );
return new SetupImgLoader<>( setupId, Cast.unchecked( DataTypeProperties.of( dataType ) ) );
}
catch ( final N5Exception e )
{
throw new IOException( e );
}
return new SetupImgLoader<>( setupId, Cast.unchecked( DataTypeProperties.of( dataType ) ) );
}

@Override
Expand All @@ -251,17 +367,16 @@ public class SetupImgLoader< T extends NativeType< T >, V extends Volatile< T >

public SetupImgLoader( final int setupId, final DataTypeProperties< T, V, ?, ? > props ) throws IOException
{
this(setupId, props.type(), props.volatileType() );
this( setupId, props.type(), props.volatileType() );
}

public SetupImgLoader( final int setupId, final T type, final V volatileType ) throws IOException
{
super( type, volatileType );
this.setupId = setupId;
final String pathName = getPathName( setupId );
try
{
mipmapResolutions = n5.getAttribute( pathName, DOWNSAMPLING_FACTORS_KEY, double[][].class );
mipmapResolutions = n5properties.getMipmapResolutions( n5, setupId );
}
catch ( final N5Exception e )
{
Expand All @@ -275,25 +390,34 @@ public SetupImgLoader( final int setupId, final T type, final V volatileType ) t
@Override
public RandomAccessibleInterval< V > getVolatileImage( final int timepointId, final int level, final ImgLoaderHint... hints )
{
return prepareCachedImage( timepointId, level, LoadingStrategy.BUDGETED, volatileType );
return prepareCachedImage( timepointId, level, BUDGETED, volatileType );
}

@Override
public RandomAccessibleInterval< T > getImage( final int timepointId, final int level, final ImgLoaderHint... hints )
{
return prepareCachedImage( timepointId, level, LoadingStrategy.BLOCKING, type );
return prepareCachedImage( timepointId, level, BLOCKING, type );
}

/**
* Create a {@link CellImg} backed by the cache.
*/
private < T extends NativeType< T > > RandomAccessibleInterval< T > prepareCachedImage( final int timepointId, final int level, final LoadingStrategy loadingStrategy, final T type )
{
final int priority = numMipmapLevels() - 1 - level;
final CacheHints cacheHints = new CacheHints( loadingStrategy, priority, false );
final String datasetPath = n5properties.getDatasetPath( setupId, timepointId, level );
return N5ImageLoader.this.prepareCachedImage( datasetPath, setupId, timepointId, level, cacheHints, type );
}

@Override
public Dimensions getImageSize( final int timepointId, final int level )
{
try
{
final String pathName = getPathName( setupId, timepointId, level );
final DatasetAttributes attributes = n5.getDatasetAttributes( pathName );
return new FinalDimensions( attributes.getDimensions() );
return new FinalDimensions( n5properties.getDimensions( n5, setupId, timepointId, level ) );
}
catch( final RuntimeException e )
catch ( final RuntimeException e )
{
return null;
}
Expand Down Expand Up @@ -322,35 +446,37 @@ public VoxelDimensions getVoxelSize( final int timepointId )
{
return null;
}
}

/**
* Create a {@link CellImg} backed by the cache.
*/
private < T extends NativeType< T > > RandomAccessibleInterval< T > prepareCachedImage( final int timepointId, final int level, final LoadingStrategy loadingStrategy, final T type )
/**
* Create a {@link CellImg} backed by the cache.
*/
protected < T extends NativeType< T > > RandomAccessibleInterval< T > prepareCachedImage(
final String datasetPath,
final int setupId,
final int timepointId,
final int level,
final CacheHints cacheHints,
final T type )
{
try
{
try
{
final String pathName = getPathName( setupId, timepointId, level );
final DatasetAttributes attributes = n5.getDatasetAttributes( pathName );
final long[] dimensions = attributes.getDimensions();
final int[] cellDimensions = attributes.getBlockSize();
final CellGrid grid = new CellGrid( dimensions, cellDimensions );

final int priority = numMipmapLevels() - 1 - level;
final CacheHints cacheHints = new CacheHints( loadingStrategy, priority, false );

final SimpleCacheArrayLoader< ? > loader = createCacheArrayLoader( n5, pathName );
return cache.createImg( grid, timepointId, setupId, level, cacheHints, loader, type );
}
catch ( final IOException | N5Exception e )
{
System.err.println( String.format(
"image data for timepoint %d setup %d level %d could not be found.",
timepointId, setupId, level ) );
return Views.interval(
new ConstantRandomAccessible<>( type.createVariable(), 3 ),
new FinalInterval( 1, 1, 1 ) );
}
final DatasetAttributes attributes = n5.getDatasetAttributes( datasetPath );
final long[] dimensions = attributes.getDimensions();
final int[] cellDimensions = attributes.getBlockSize();
final CellGrid grid = new CellGrid( dimensions, cellDimensions );
final DataTypeProperties< ?, ?, ?, ? > dataTypeProperties = DataTypeProperties.of( attributes.getDataType() );
final SimpleCacheArrayLoader< ? > loader = new N5CacheArrayLoader<>( n5, datasetPath, attributes, dataTypeProperties );
return cache.createImg( grid, timepointId, setupId, level, cacheHints, loader, type );
}
catch ( final N5Exception e )
{
System.err.println( String.format(
"image data for timepoint %d setup %d level %d could not be found.",
timepointId, setupId, level ) );
return Views.interval(
new ConstantRandomAccessible<>( type.createVariable(), 3 ),
new FinalInterval( 1, 1, 1 ) );
}
}

Expand Down
Loading

0 comments on commit fe3f2da

Please sign in to comment.