Skip to content

Commit

Permalink
use faster yaml loader and better delayed loading
Browse files Browse the repository at this point in the history
  • Loading branch information
toloudis committed Sep 14, 2024
1 parent f955d03 commit 063ef31
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 4 deletions.
8 changes: 7 additions & 1 deletion bioio_sldy/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import pathlib
import typing

import dask
import dask.array as da
import numpy as np
import xarray as xr
Expand Down Expand Up @@ -149,9 +150,14 @@ def _read_delayed(self) -> xr.DataArray:
for timepoint in timepoints:
data_for_timepoint: typing.List[da.Array] = []
for channel in channels:
data = image.get_data(
value = dask.delayed(image.get_data)(
timepoint=timepoint, channel=channel, delayed=True
)
data = da.from_delayed(
value,
shape=(image.sizeZ, image.sizeY, image.sizeX),
dtype=image.dtype,
)
data_for_timepoint.append(data)

data_as_list.append(da.stack(data_for_timepoint))
Expand Down
16 changes: 13 additions & 3 deletions bioio_sldy/sldy_image.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ class SldyImage:
_data_paths: typing.Set[pathlib.Path] = set()

@staticmethod
def _yaml_mapping(loader: yaml.Loader, node: yaml.Node, deep: bool = False) -> dict:
def _yaml_mapping(
loader: yaml.CLoader, node: yaml.Node, deep: bool = False
) -> dict:
"""
Static method intended to map key-value pairs found in image
metadata yaml files to Python dictionaries.
Expand Down Expand Up @@ -108,7 +110,7 @@ def _get_yaml_contents(
"""
try:
with fs.open(yaml_path) as f:
return yaml.load(f, Loader=yaml.Loader)
return yaml.load(f, Loader=yaml.CLoader)
except FileNotFoundError:
if is_required:
raise
Expand Down Expand Up @@ -171,7 +173,7 @@ def __init__(
yaml.add_constructor(
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
SldyImage._yaml_mapping,
yaml.Loader,
yaml.CLoader,
)

self._fs = fs
Expand Down Expand Up @@ -219,6 +221,14 @@ def __init__(
self.timepoints = sorted(self._timepoint_to_data_paths.keys())
self.channels = sorted(self._channel_to_data_paths.keys())

self.sizeT = self._image_record["CImageRecord70"]["mNumTimepoints"]
self.sizeC = self._image_record["CImageRecord70"]["mNumChannels"]
self.sizeZ = self._image_record["CImageRecord70"]["mNumPlanes"]
self.sizeY = self._image_record["CImageRecord70"]["mHeight"]
self.sizeX = self._image_record["CImageRecord70"]["mWidth"]
# TODO check this but are all sldys uint16? bioformats seems to say so
self.dtype = np.dtype(np.uint16)

@property
def metadata(self) -> typing.Dict[str, typing.Optional[dict]]:
"""
Expand Down

0 comments on commit 063ef31

Please sign in to comment.