Hdf5 types

src/main/java/bdv/export/WriteSequenceToHdf5.java

@@ -610,7 +610,7 @@ public void flush( final Object dataset )
 		@Override
 		public RandomAccessibleInterval< UnsignedShortType > getImage( final int level )
 		{
-			return loopback.getSetupImgLoader( viewIdPartition.getViewSetupId() ).getImage( viewIdPartition.getTimePointId(), level );
+			return Cast.unchecked( loopback.getSetupImgLoader( viewIdPartition.getViewSetupId() ).getImage( viewIdPartition.getTimePointId(), level ) );
 		}
 	}
 

src/main/java/bdv/img/hdf5/DimsAndExistence.java

@@ -28,6 +28,9 @@
  */
 package bdv.img.hdf5;
 
+import java.util.Arrays;
+import java.util.StringJoiner;
+
 /**
  * The dimensions of an image and a flag indicating whether that image
  * exists (can be loaded)
@@ -38,11 +41,14 @@ public class DimsAndExistence
 {
 	private final long[] dimensions;
 
+	private final int[] blockSize;
+
 	private final boolean exists;
 
-	public DimsAndExistence( final long[] dimensions, final boolean exists )
+	public DimsAndExistence( final long[] dimensions, final int[] blockSize, final boolean exists )
 	{
 		this.dimensions = dimensions;
+		this.blockSize = blockSize;
 		this.exists = exists;
 	}
 
@@ -51,8 +57,23 @@ public long[] getDimensions()
 		return dimensions;
 	}
 
+	public int[] getBlockSize()
+	{
+		return blockSize;
+	}
+
 	public boolean exists()
 	{
 		return exists;
 	}
+
+	@Override
+	public String toString()
+	{
+		return new StringJoiner( ", ", DimsAndExistence.class.getSimpleName() + "[", "]" )
+				.add( "dimensions=" + Arrays.toString( dimensions ) )
+				.add( "blockSize=" + Arrays.toString( blockSize ) )
+				.add( "exists=" + exists )
+				.toString();
+	}
 }

src/main/java/bdv/img/hdf5/HDF5Access.java

@@ -28,96 +28,259 @@
  */
 package bdv.img.hdf5;
 
-import static bdv.img.hdf5.Util.reorder;
-import ch.systemsx.cisd.base.mdarray.MDFloatArray;
-import ch.systemsx.cisd.base.mdarray.MDShortArray;
-import ch.systemsx.cisd.hdf5.HDF5DataSetInformation;
+import ch.systemsx.cisd.hdf5.IHDF5FileLevelReadOnlyHandler;
 import ch.systemsx.cisd.hdf5.IHDF5Reader;
+import ch.systemsx.cisd.hdf5.hdf5lib.HDFHelper;
+import hdf.hdf5lib.H5;
+import hdf.hdf5lib.HDF5Constants;
+import java.io.File;
+import java.util.LinkedHashMap;
+import java.util.Map.Entry;
+import org.janelia.saalfeldlab.n5.DataBlock;
+import org.janelia.saalfeldlab.n5.DataType;
+
+import static bdv.img.hdf5.Util.reorder;
+import static hdf.hdf5lib.H5.H5Dclose;
+import static hdf.hdf5lib.H5.H5Dget_space;
+import static hdf.hdf5lib.H5.H5Dopen;
+import static hdf.hdf5lib.H5.H5Dread;
+import static hdf.hdf5lib.H5.H5Fclose;
+import static hdf.hdf5lib.H5.H5Fopen;
+import static hdf.hdf5lib.H5.H5Pclose;
+import static hdf.hdf5lib.H5.H5Sclose;
+import static hdf.hdf5lib.H5.H5Screate_simple;
+import static hdf.hdf5lib.H5.H5Sget_simple_extent_dims;
+import static hdf.hdf5lib.H5.H5Sget_simple_extent_ndims;
+import static hdf.hdf5lib.H5.H5Sselect_hyperslab;
+import static hdf.hdf5lib.H5.H5open;
+import static hdf.hdf5lib.HDF5Constants.H5F_ACC_RDONLY;
+import static hdf.hdf5lib.HDF5Constants.H5O_TYPE_DATASET;
+import static hdf.hdf5lib.HDF5Constants.H5O_TYPE_GROUP;
+import static hdf.hdf5lib.HDF5Constants.H5P_DEFAULT;
+import static hdf.hdf5lib.HDF5Constants.H5S_SELECT_SET;
+import static hdf.hdf5lib.HDF5Constants.H5T_NATIVE_DOUBLE;
+import static hdf.hdf5lib.HDF5Constants.H5T_NATIVE_FLOAT;
+import static hdf.hdf5lib.HDF5Constants.H5T_NATIVE_INT16;
+import static hdf.hdf5lib.HDF5Constants.H5T_NATIVE_INT32;
+import static hdf.hdf5lib.HDF5Constants.H5T_NATIVE_INT64;
+import static hdf.hdf5lib.HDF5Constants.H5T_NATIVE_INT8;
+import static hdf.hdf5lib.HDF5Constants.H5T_NATIVE_UINT16;
+import static hdf.hdf5lib.HDF5Constants.H5T_NATIVE_UINT32;
+import static hdf.hdf5lib.HDF5Constants.H5T_NATIVE_UINT64;
+import static hdf.hdf5lib.HDF5Constants.H5T_NATIVE_UINT8;
+import static net.imglib2.util.Util.long2int;
 
-class HDF5Access implements IHDF5Access
+/**
+ * Access chunked data-sets through lower-level HDF5. This avoids opening and
+ * closing the dataset for each chunk when accessing through jhdf5 (This is a
+ * huge bottleneck when accessing many small chunks).
+ *
+ * The HDF5 fileId is extracted from a jhdf5 HDF5Reader to avoid having to do
+ * everything ourselves.
+ *
+ * @author Tobias Pietzsch
+ */
+class HDF5Access
 {
 	private final IHDF5Reader hdf5Reader;
 
-	private final int[] reorderedDimensions = new int[ 3 ];
+	private final long fileId;
+
+	private final long numericConversionXferPropertyListID;
+
+	private final long[] reorderedDimensions = new long[ 3 ];
 
 	private final long[] reorderedMin = new long[ 3 ];
 
+	private static final int MAX_OPEN_DATASETS = 48;
+
+	private class OpenDataSet
+	{
+		final long dataSetId;
+
+		final long fileSpaceId;
+
+		public OpenDataSet( final String pathName )
+		{
+			dataSetId = H5Dopen( fileId, pathName, H5P_DEFAULT );
+			fileSpaceId = H5Dget_space( dataSetId );
+		}
+
+		public void close()
+		{
+			H5Sclose( fileSpaceId );
+			H5Dclose( dataSetId );
+		}
+	}
+
+	private class OpenDataSetCache extends LinkedHashMap< String, OpenDataSet >
+	{
+		private static final long serialVersionUID = 1L;
+
+		public OpenDataSetCache()
+		{
+			super( MAX_OPEN_DATASETS, 0.75f, true );
+		}
+
+		@Override
+		protected boolean removeEldestEntry( final Entry< String, OpenDataSet > eldest )
+		{
+			if ( size() > MAX_OPEN_DATASETS )
+			{
+				eldest.getValue().close();
+				return true;
+			}
+			else
+				return false;
+		}
+
+		public OpenDataSet getDataSet( final String pathName )
+		{
+			OpenDataSet openDataSet = super.get( pathName );
+			if ( openDataSet == null && datasetExists( pathName ) )
+			{
+				openDataSet = new OpenDataSet( pathName );
+				put( pathName, openDataSet );
+			}
+			return openDataSet;
+		}
+
+		private boolean datasetExists( final String pathName )
+		{
+			final int objectTypeId = "/".equals(pathName) ? H5O_TYPE_GROUP : HDFHelper.H5Oget_info_by_name(fileId, pathName, false).type;
+			return objectTypeId == H5O_TYPE_DATASET ;
+		}
+	}
+
+	private final OpenDataSetCache openDataSetCache;
+
 	public HDF5Access( final IHDF5Reader hdf5Reader )
 	{
 		this.hdf5Reader = hdf5Reader;
+
+		// TODO: Do see ch.systemsx.cisd.hdf5.HDF5.createFileAccessPropertyListId for version bounds checking
+		long fileAccessPropertyListId = H5P_DEFAULT;
+
+		IHDF5FileLevelReadOnlyHandler fileHandler = hdf5Reader.file();
+		boolean performNumericConversions = fileHandler.isPerformNumericConversions();
+		File file = fileHandler.getFile();
+
+		// Make sure library is initialized. This can be called multiple times.
+		H5open();
+
+		// See ch.systemsx.cisd.hdf5.HDF5 constructor
+		// Make sure to close the numericConversionXferPropertyListID property list created below. See close()
+		if (performNumericConversions)
+		{
+			this.numericConversionXferPropertyListID = HDFHelper.H5Pcreate_xfer_abort_overflow();
+		} else {
+			this.numericConversionXferPropertyListID = HDFHelper.H5Pcreate_xfer_abort();
+		}
+
+		// Make sure to close the fileID created below. See close()
+		fileId = H5Fopen(file.getAbsolutePath(), H5F_ACC_RDONLY, fileAccessPropertyListId);
+
+		openDataSetCache = new OpenDataSetCache();
 	}
 
-	@Override
-	public synchronized DimsAndExistence getDimsAndExistence( final ViewLevelId id )
+	public synchronized DimsAndExistence getDimsAndExistence( final String pathName )
 	{
-		final String cellsPath = Util.getCellsPath( id );
-		HDF5DataSetInformation info = null;
-		boolean exists = false;
-		try
+		final OpenDataSet dataset = openDataSetCache.getDataSet( pathName );
+		if ( dataset == null )
 		{
-			info = hdf5Reader.getDataSetInformation( cellsPath );
-			exists = true;
+			return new DimsAndExistence( new long[] { 1, 1, 1 }, null, false );
 		}
-		catch ( final Exception e )
-		{}
-		if ( exists )
-			return new DimsAndExistence( reorder( info.getDimensions() ), true );
 		else
-			return new DimsAndExistence( new long[] { 1, 1, 1 }, false );
+		{
+			final int nDims = H5Sget_simple_extent_ndims( dataset.fileSpaceId );
+			final long[] dimensions = new long[ nDims ];
+			H5Sget_simple_extent_dims( dataset.fileSpaceId, dimensions, null );
+
+			int[] chunkSizes = null;
+			final long creationPropertyList = H5.H5Dget_create_plist( dataset.dataSetId );
+			if ( H5.H5Pget_layout( creationPropertyList ) == HDF5Constants.H5D_CHUNKED )
+			{
+				final long[] longChunkSizes = new long[ nDims ];
+				H5.H5Pget_chunk( creationPropertyList, nDims, longChunkSizes );
+				chunkSizes = long2int( longChunkSizes );
+			}
+			H5.H5Pclose( creationPropertyList );
+
+			return new DimsAndExistence( reorder( dimensions ), reorder( chunkSizes ), true );
+		}
 	}
 
-	@Override
-	public synchronized short[] readShortMDArrayBlockWithOffset( final int timepoint, final int setup, final int level, final int[] dimensions, final long[] min ) throws InterruptedException
+	public void closeAllDataSets()
 	{
-		if ( Thread.interrupted() )
-			throw new InterruptedException();
-		Util.reorder( dimensions, reorderedDimensions );
-		Util.reorder( min, reorderedMin );
-		final MDShortArray array = hdf5Reader.int16().readMDArrayBlockWithOffset( Util.getCellsPath( timepoint, setup, level ), reorderedDimensions, reorderedMin );
-		return array.getAsFlatArray();
+		for ( final OpenDataSet dataset : openDataSetCache.values() )
+			dataset.close();
+		openDataSetCache.clear();
 	}
 
-	@Override
-	public short[] readShortMDArrayBlockWithOffset( final int timepoint, final int setup, final int level, final int[] dimensions, final long[] min, final short[] dataBlock ) throws InterruptedException
+	public void close()
 	{
-		System.arraycopy( readShortMDArrayBlockWithOffset( timepoint, setup, level, dimensions, min ), 0, dataBlock, 0, dataBlock.length );
-		return dataBlock;
+		closeAllDataSets();
+		int status = 0;
+		status = H5Pclose(numericConversionXferPropertyListID);
+		if (status < 0) {
+			System.err.println("HDF5AccessHack: Error closing property list");
+		}
+		status = H5Fclose(fileId);
+		if (status < 0) {
+			System.err.println("HDF5AccessHack: Error closing file");
+		}
+		hdf5Reader.close();
 	}
 
-	@Override
-	public float[] readShortMDArrayBlockWithOffsetAsFloat( final int timepoint, final int setup, final int level, final int[] dimensions, final long[] min ) throws InterruptedException
+	public synchronized DataBlock< ? > readBlock(
+			final String pathName,
+			final DataType dataType,
+			final long memTypeId,
+			final int[] dimensions,
+			final long[] min ) throws InterruptedException
 	{
 		if ( Thread.interrupted() )
 			throw new InterruptedException();
+
 		Util.reorder( dimensions, reorderedDimensions );
 		Util.reorder( min, reorderedMin );
-		final MDFloatArray array = hdf5Reader.float32().readMDArrayBlockWithOffset( Util.getCellsPath( timepoint, setup, level ), reorderedDimensions, reorderedMin );
-		final float[] pixels = array.getAsFlatArray();
-		unsignedShort( pixels );
-		return pixels;
-	}
-
-	@Override
-	public float[] readShortMDArrayBlockWithOffsetAsFloat( final int timepoint, final int setup, final int level, final int[] dimensions, final long[] min, final float[] dataBlock ) throws InterruptedException
-	{
-		System.arraycopy( readShortMDArrayBlockWithOffsetAsFloat( timepoint, setup, level, dimensions, min ), 0, dataBlock, 0, dataBlock.length );
-		return dataBlock;
-	}
 
-	@Override
-	public void closeAllDataSets()
-	{}
+		final DataBlock< ? > block = dataType.createDataBlock( dimensions, min );
+		final OpenDataSet dataset = openDataSetCache.getDataSet( pathName );
+		final long memorySpaceId = H5Screate_simple( reorderedDimensions.length, reorderedDimensions, null );
+		H5Sselect_hyperslab( dataset.fileSpaceId, H5S_SELECT_SET, reorderedMin, null, reorderedDimensions, null );
+		H5Dread( dataset.dataSetId, memTypeId, memorySpaceId, dataset.fileSpaceId, numericConversionXferPropertyListID, block.getData() );
+		H5Sclose( memorySpaceId );
 
-	@Override
-	public void close()
-	{
-		closeAllDataSets();
-		hdf5Reader.close();
+		return block;
 	}
 
-	protected static final void unsignedShort( final float[] pixels )
+	public static long memTypeId( final DataType dataType )
 	{
-		for ( int j = 0; j < pixels.length; ++j )
-			pixels[ j ] = ((short)pixels[ j ]) & 0xffff;
+		switch ( dataType )
+		{
+		case INT8:
+			return H5T_NATIVE_INT8;
+		case UINT8:
+			return H5T_NATIVE_UINT8;
+		case INT16:
+			return H5T_NATIVE_INT16;
+		case UINT16:
+			return H5T_NATIVE_UINT16;
+		case INT32:
+			return H5T_NATIVE_INT32;
+		case UINT32:
+			return H5T_NATIVE_UINT32;
+		case INT64:
+			return H5T_NATIVE_INT64;
+		case UINT64:
+			return H5T_NATIVE_UINT64;
+		case FLOAT32:
+			return H5T_NATIVE_FLOAT;
+		case FLOAT64:
+			return H5T_NATIVE_DOUBLE;
+		default:
+			throw new IllegalArgumentException();
+		}
 	}
 }