From 3dd4ed3563d360553b9b78d11b8738d1f5bd4da2 Mon Sep 17 00:00:00 2001 From: Eric Prestat Date: Fri, 28 Jun 2024 16:08:47 +0100 Subject: [PATCH 1/3] EMD Velox: fix reading EDS stream detector lazily with `sum_EDS_detectors=True`. When summing the stream, the same stream was used several times --- rsciio/emd/_emd_velox.py | 5 ++--- rsciio/tests/test_emd_velox.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/rsciio/emd/_emd_velox.py b/rsciio/emd/_emd_velox.py index 48632e843..0a153eec2 100644 --- a/rsciio/emd/_emd_velox.py +++ b/rsciio/emd/_emd_velox.py @@ -950,7 +950,6 @@ def stream_to_sparse_array(self, stream_data): """ # Here we load the stream data into memory, which is fine is the # arrays are small. We could load them lazily when lazy. - stream_data = self.stream_group["Data"][:].T[0] sparse_array = stream_readers.stream_to_sparse_COO_array( stream_data=stream_data, spatial_shape=self.reader.spatial_shape, @@ -967,8 +966,8 @@ def stream_to_array(self, stream_data, spectrum_image=None): Parameters ---------- - stream_data: array - spectrum_image: array or None + stream_data : numpy.ndarray + spectrum_image : numpy.ndarray or None If array, the data from the stream are added to the array. Otherwise it creates a new array and returns it. diff --git a/rsciio/tests/test_emd_velox.py b/rsciio/tests/test_emd_velox.py index 68ad2950c..4acb333da 100644 --- a/rsciio/tests/test_emd_velox.py +++ b/rsciio/tests/test_emd_velox.py @@ -405,6 +405,25 @@ def test_fei_si_4detectors(self, lazy, sum_EDS_detectors): assert len(signal) == length # TODO: add parsing azimuth_angle + @pytest.mark.parametrize("lazy", (False, True)) + def test_fei_si_4detectors_compare(self, lazy): + fname = self.fei_files_path / "fei_SI_EDS-HAADF-4detectors_2frames.emd" + s_sum_EDS = hs.load(fname, sum_EDS_detectors=True, lazy=lazy)[-1] + s = hs.load(fname, sum_EDS_detectors=False, lazy=lazy)[-4:] + if lazy: + s_sum_EDS.compute() + for s_ in s: + s_.compute() + + s2 = hs.stack(s, new_axis_name="detector").sum("detector") + + np.testing.assert_allclose(s[-1].data.sum(), 865236) + np.testing.assert_allclose(s[-2].data.sum(), 913682) + np.testing.assert_allclose(s[-3].data.sum(), 867647) + np.testing.assert_allclose(s[-4].data.sum(), 916174) + np.testing.assert_allclose(s2.data.sum(), 3562739) + np.testing.assert_allclose(s2.data, s_sum_EDS.data) + def test_fei_emd_ceta_camera(self): signal = hs.load(self.fei_files_path / "1532 Camera Ceta.emd") np.testing.assert_allclose(signal.data, np.zeros((64, 64))) From 4c7b914303671f5113a8741d55c75d7804daed98 Mon Sep 17 00:00:00 2001 From: Eric Prestat Date: Thu, 4 Jul 2024 16:46:55 +0100 Subject: [PATCH 2/3] Fix reading separate EDS detector with individual frames --- rsciio/tests/test_emd_velox.py | 8 +++++--- rsciio/utils/fei_stream_readers.py | 14 +++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/rsciio/tests/test_emd_velox.py b/rsciio/tests/test_emd_velox.py index 4acb333da..c14e2903d 100644 --- a/rsciio/tests/test_emd_velox.py +++ b/rsciio/tests/test_emd_velox.py @@ -406,10 +406,12 @@ def test_fei_si_4detectors(self, lazy, sum_EDS_detectors): # TODO: add parsing azimuth_angle @pytest.mark.parametrize("lazy", (False, True)) - def test_fei_si_4detectors_compare(self, lazy): + @pytest.mark.parametrize("sum_frames", (False, True)) + def test_fei_si_4detectors_compare(self, lazy, sum_frames): fname = self.fei_files_path / "fei_SI_EDS-HAADF-4detectors_2frames.emd" - s_sum_EDS = hs.load(fname, sum_EDS_detectors=True, lazy=lazy)[-1] - s = hs.load(fname, sum_EDS_detectors=False, lazy=lazy)[-4:] + kwargs = dict(lazy=lazy, sum_frames=sum_frames) + s_sum_EDS = hs.load(fname, sum_EDS_detectors=True, **kwargs)[-1] + s = hs.load(fname, sum_EDS_detectors=False, **kwargs)[-4:] if lazy: s_sum_EDS.compute() for s_ in s: diff --git a/rsciio/utils/fei_stream_readers.py b/rsciio/utils/fei_stream_readers.py index 05bf54881..42d6991f4 100644 --- a/rsciio/utils/fei_stream_readers.py +++ b/rsciio/utils/fei_stream_readers.py @@ -363,13 +363,13 @@ def stream_to_array( dtype=dtype, ) - _fill_array_with_stream( - spectrum_image=spectrum_image, - stream=stream, - first_frame=first_frame, - last_frame=last_frame, - rebin_energy=rebin_energy, - ) + _fill_array_with_stream( + spectrum_image=spectrum_image, + stream=stream, + first_frame=first_frame, + last_frame=last_frame, + rebin_energy=rebin_energy, + ) else: if spectrum_image is None: spectrum_image = np.zeros( From 634db2f608c5423f0e2e935fc6f62c76703fb8d4 Mon Sep 17 00:00:00 2001 From: Eric Prestat Date: Fri, 28 Jun 2024 17:22:32 +0100 Subject: [PATCH 3/3] Add changelog entry --- upcoming_changes/287.bugfix.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 upcoming_changes/287.bugfix.rst diff --git a/upcoming_changes/287.bugfix.rst b/upcoming_changes/287.bugfix.rst new file mode 100644 index 000000000..3f396ebed --- /dev/null +++ b/upcoming_changes/287.bugfix.rst @@ -0,0 +1,4 @@ +:ref:`EMD Velox ` fixes for reading files containing multiple EDS streams: + +- fix reading multiple EDS streams lazily with ``sum_EDS_detectors=True``, +- fix reading separate EDS stream and individual frames when using ``sum_EDS_detectors=False`` and ``sum_frames=False``. \ No newline at end of file