From facebd1c0a1f017442b247c14ba5c0dbe299574e Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Thu, 11 Jul 2024 23:37:46 +0100 Subject: [PATCH 1/2] metrics: standalone `log()` - fixes #51 --- main_OSEM.py | 2 +- petric.py | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/main_OSEM.py b/main_OSEM.py index 85bcf2e..19a48fa 100644 --- a/main_OSEM.py +++ b/main_OSEM.py @@ -90,7 +90,7 @@ def update(self): def update_objective(self): """ NB: The objective value is not required by OSEM nor by PETRIC, so this returns `0`. - NB: In theory it should be `sum(prompts * log(acq_model.forward(self.x)) - self.x * sensitivity)` across all subsets. + NB: It should be `sum(prompts * log(acq_model.forward(self.x)) - self.x * sensitivity)` across all subsets. """ return 0 diff --git a/petric.py b/petric.py index ebf2139..50efb72 100755 --- a/petric.py +++ b/petric.py @@ -102,11 +102,11 @@ def __call__(self, algorithm): iteration = algorithm.iteration if iteration % algorithm.update_objective_interval != 0 and iteration != algorithm.max_iteration: return + self.log(algorithm.x, iteration) + def log(self, test_im, iteration): assert not any(self.filter.values()), "Filtering not implemented" - test_im_arr = algorithm.x.as_array() - - # (1) global metrics & statistics + test_im_arr = test_im.as_array() self.tb_summary_writer.add_scalar( "RMSE_whole_object", np.sqrt(mse(self.ref_im_arr[self.whole_object_indices], test_im_arr[self.whole_object_indices])) / @@ -115,10 +115,7 @@ def __call__(self, algorithm): "RMSE_background", np.sqrt(mse(self.ref_im_arr[self.background_indices], test_im_arr[self.background_indices])) / self.norm, iteration) - - # (2) local metrics & statistics for voi_name, voi_indices in sorted(self.voi_indices.items()): - # AEM not to be confused with MAE self.tb_summary_writer.add_scalar( f"AEM_VOI_{voi_name}", np.abs(test_im_arr[voi_indices].mean() - self.ref_im_arr[voi_indices].mean()) / self.norm, iteration) From f31d0772e15528e32dbb04c339318d7e1005a22a Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Fri, 12 Jul 2024 13:08:58 +0100 Subject: [PATCH 2/2] QualityMetrics.evaluate --- petric.py | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/petric.py b/petric.py index 50efb72..4a675ed 100755 --- a/petric.py +++ b/petric.py @@ -57,7 +57,7 @@ def __call__(self, algo: Algorithm): algo.x.write(str(self.outdir / 'iter_final.hv')) -class TensorBoard(cbks.Callback): +class StatsLog(cbks.Callback): """Log image slices & objective value""" def __init__(self, verbose=1, transverse_slice=None, coronal_slice=None, vmax=None, logdir=OUTDIR): super().__init__(verbose) @@ -98,27 +98,26 @@ def __init__(self, reference_image, whole_object_mask, background_mask, **kwargs self.ref_im_arr = reference_image.as_array() self.norm = self.ref_im_arr[self.background_indices].mean() - def __call__(self, algorithm): - iteration = algorithm.iteration - if iteration % algorithm.update_objective_interval != 0 and iteration != algorithm.max_iteration: + def __call__(self, algo: Algorithm): + iteration = algo.iteration + if iteration % algo.update_objective_interval != 0 and iteration != algo.max_iteration: return - self.log(algorithm.x, iteration) + for tag, value in self.evaluate(algo.x).items(): + self.tb_summary_writer.add_scalar(tag, value, iteration) - def log(self, test_im, iteration): + def evaluate(self, test_im: STIR.ImageData) -> dict[str, float]: assert not any(self.filter.values()), "Filtering not implemented" test_im_arr = test_im.as_array() - self.tb_summary_writer.add_scalar( - "RMSE_whole_object", - np.sqrt(mse(self.ref_im_arr[self.whole_object_indices], test_im_arr[self.whole_object_indices])) / - self.norm, iteration) - self.tb_summary_writer.add_scalar( - "RMSE_background", - np.sqrt(mse(self.ref_im_arr[self.background_indices], test_im_arr[self.background_indices])) / self.norm, - iteration) - for voi_name, voi_indices in sorted(self.voi_indices.items()): - self.tb_summary_writer.add_scalar( - f"AEM_VOI_{voi_name}", - np.abs(test_im_arr[voi_indices].mean() - self.ref_im_arr[voi_indices].mean()) / self.norm, iteration) + whole = { + "RMSE_whole_object": np.sqrt( + mse(self.ref_im_arr[self.whole_object_indices], test_im_arr[self.whole_object_indices])) / self.norm, + "RMSE_background": np.sqrt( + mse(self.ref_im_arr[self.background_indices], test_im_arr[self.background_indices])) / self.norm} + local = { + f"AEM_VOI_{voi_name}": np.abs(test_im_arr[voi_indices].mean() - self.ref_im_arr[voi_indices].mean()) / + self.norm + for voi_name, voi_indices in sorted(self.voi_indices.items())} + return {**whole, **local} class MetricsWithTimeout(cbks.Callback): @@ -129,20 +128,20 @@ def __init__(self, seconds=300, outdir=OUTDIR, transverse_slice=None, coronal_sl self.callbacks = [ cbks.ProgressCallback(), SaveIters(outdir=outdir), - (tb_cbk := TensorBoard(logdir=outdir, transverse_slice=transverse_slice, coronal_slice=coronal_slice))] + (tb_cbk := StatsLog(logdir=outdir, transverse_slice=transverse_slice, coronal_slice=coronal_slice))] self.tb = tb_cbk.tb self.reset() def reset(self, seconds=None): self.limit = time() + (self._seconds if seconds is None else seconds) - def __call__(self, algorithm: Algorithm): + def __call__(self, algo: Algorithm): if (now := time()) > self.limit: log.warning("Timeout reached. Stopping algorithm.") raise StopIteration if self.callbacks: for c in self.callbacks: - c(algorithm) + c(algo) self.limit += time() - now @staticmethod @@ -221,7 +220,7 @@ def get_image(fname): if SRCDIR.is_dir(): # create list of existing data - # Note: as MetricsWithTimeout initialises Tensorboard, this will currently create directories in OUTDIR accordingly + # NB: `MetricsWithTimeout` initialises `SaveIters` which creates `outdir` data_dirs_metrics = [(SRCDIR / "Siemens_mMR_NEMA_IQ", OUTDIR / "mMR_NEMA", [MetricsWithTimeout(outdir=OUTDIR / "mMR_NEMA", transverse_slice=72, coronal_slice=109)]), (SRCDIR / "NeuroLF_Hoffman_Dataset", OUTDIR / "NeuroLF_Hoffman",