From d3459dd34a5d5656b30e090d330b0c0f728bff90 Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Mon, 7 Oct 2024 11:09:45 -0400 Subject: [PATCH 01/14] added unit test for median memory usage --- tests/outlier_detection/test_median.py | 39 ++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/outlier_detection/test_median.py b/tests/outlier_detection/test_median.py index 5361e44b..95e687cc 100644 --- a/tests/outlier_detection/test_median.py +++ b/tests/outlier_detection/test_median.py @@ -1,5 +1,6 @@ import os from pathlib import Path +import tracemalloc import numpy as np import pytest @@ -194,3 +195,41 @@ def test_nanmedian3D(): assert med.dtype == np.float32 assert np.allclose(med, np.nanmedian(cube, axis=0), equal_nan=True) + + +@pytest.mark.parametrize("in_memory", [True, False]) +def test_memory_computer(in_memory): + """ + Analytically calculate how much memory the median computation + is supposed to take, then ensure that the implementation + stays near that. + + in_memory=True case allocates the following memory: + - one cube size + - median array == one frame size + + in_memory=False case allocates the following memory: + - one buffer size, which by default is the frame size + - median array == one frame size + + add a half-frame-size buffer to the expected memory usage in both cases + """ + shp = (20, 500, 500) + cube_size = np.dtype("float32").itemsize * shp[0] * shp[1] * shp[2] #bytes + frame_size = cube_size / shp[0] + + # compute the median while tracking memory usage + tracemalloc.start() + computer = MedianComputer(shp, in_memory=in_memory) + for i in range(shp[0]): + frame = np.full(shp[1:], i, dtype=np.float32) + computer.append(frame, i) + del frame + computer.evaluate() + _, peak_mem = tracemalloc.get_traced_memory() + tracemalloc.stop() + if in_memory: + expected_mem = cube_size + frame_size*1.5 + assert peak_mem < expected_mem + else: + assert peak_mem < frame_size * 2.5 From 03ec94a2b16b6b5c77fe6e4abbaa3124d0aef336 Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Wed, 9 Oct 2024 10:34:51 -0400 Subject: [PATCH 02/14] memory threshold class and fixture --- tests/conftest.py | 8 ++++++++ tests/helpers.py | 18 ++++++++++++++++++ tests/outlier_detection/test_median.py | 25 ++++++++++++------------- 3 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/helpers.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..5009cf51 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,8 @@ +import pytest +from tests.helpers import MemoryThreshold + + +@pytest.fixture +def memory_threshold(expected_usage): + with MemoryThreshold(expected_usage) as tracker: + yield tracker \ No newline at end of file diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 00000000..703aa27c --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,18 @@ +import tracemalloc + +class MemoryThreshold: + def __init__(self, expected_usage): + self.expected_usage = expected_usage + + def __enter__(self): + tracemalloc.start() + return self + + def __exit__(self, exc_type, exc_value, traceback): + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + assert peak <= self.expected_usage, ( + "Peak memory usage exceeded expected usage: " + f"{peak / 1024:.2f} KB > {self.expected_usage / 1024:.2f} KB" + ) \ No newline at end of file diff --git a/tests/outlier_detection/test_median.py b/tests/outlier_detection/test_median.py index 95e687cc..4d32f70b 100644 --- a/tests/outlier_detection/test_median.py +++ b/tests/outlier_detection/test_median.py @@ -1,6 +1,5 @@ import os from pathlib import Path -import tracemalloc import numpy as np import pytest @@ -11,6 +10,7 @@ _OnDiskMedian, nanmedian3D, ) +from tests.helpers import MemoryThreshold def test_disk_appendable_array(tmp_path): @@ -218,18 +218,17 @@ def test_memory_computer(in_memory): cube_size = np.dtype("float32").itemsize * shp[0] * shp[1] * shp[2] #bytes frame_size = cube_size / shp[0] - # compute the median while tracking memory usage - tracemalloc.start() - computer = MedianComputer(shp, in_memory=in_memory) - for i in range(shp[0]): - frame = np.full(shp[1:], i, dtype=np.float32) - computer.append(frame, i) - del frame - computer.evaluate() - _, peak_mem = tracemalloc.get_traced_memory() - tracemalloc.stop() + # calculate expected memory usage if in_memory: expected_mem = cube_size + frame_size*1.5 - assert peak_mem < expected_mem else: - assert peak_mem < frame_size * 2.5 + expected_mem = frame_size * 2.5 + + # compute the median while tracking memory usage + with MemoryThreshold(expected_mem): + computer = MedianComputer(shp, in_memory=in_memory) + for i in range(shp[0]): + frame = np.full(shp[1:], i, dtype=np.float32) + computer.append(frame, i) + del frame + computer.evaluate() From 6b31d175fbc03285156fca8dd0e05c8d597fdf18 Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Wed, 9 Oct 2024 10:42:31 -0400 Subject: [PATCH 03/14] added docstrings --- tests/conftest.py | 1 + tests/helpers.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 5009cf51..b4e4d58a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,5 +4,6 @@ @pytest.fixture def memory_threshold(expected_usage): + """Fixture to check peak memory usage against an expected threshold.""" with MemoryThreshold(expected_usage) as tracker: yield tracker \ No newline at end of file diff --git a/tests/helpers.py b/tests/helpers.py index 703aa27c..39f65e6d 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -1,7 +1,21 @@ import tracemalloc class MemoryThreshold: + """ + Context manager to check peak memory usage against an expected threshold. + + example usage: + with MemoryThreshold(expected_usage): + # code that should not exceed expected + """ + def __init__(self, expected_usage): + """ + Parameters + ---------- + expected_usage : int + Expected peak memory usage in bytes + """ self.expected_usage = expected_usage def __enter__(self): From c633dbbecbfddf1af47e16ed20194bbd3d736c08 Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Wed, 9 Oct 2024 10:46:22 -0400 Subject: [PATCH 04/14] put file io into tmp_path --- tests/outlier_detection/test_median.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/outlier_detection/test_median.py b/tests/outlier_detection/test_median.py index 4d32f70b..bc90f018 100644 --- a/tests/outlier_detection/test_median.py +++ b/tests/outlier_detection/test_median.py @@ -198,7 +198,7 @@ def test_nanmedian3D(): @pytest.mark.parametrize("in_memory", [True, False]) -def test_memory_computer(in_memory): +def test_memory_computer(in_memory, tmp_path): """ Analytically calculate how much memory the median computation is supposed to take, then ensure that the implementation @@ -226,7 +226,7 @@ def test_memory_computer(in_memory): # compute the median while tracking memory usage with MemoryThreshold(expected_mem): - computer = MedianComputer(shp, in_memory=in_memory) + computer = MedianComputer(shp, in_memory=in_memory, tempdir=tmp_path) for i in range(shp[0]): frame = np.full(shp[1:], i, dtype=np.float32) computer.append(frame, i) From 7d903645dd0b419299238675caf06f684db7253f Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Wed, 9 Oct 2024 12:09:28 -0400 Subject: [PATCH 05/14] moved MemoryThreshold to testing_helpers, added tests for it --- tests/helpers.py => src/stcal/testing_helpers.py | 13 +++++++++---- tests/conftest.py | 9 --------- tests/outlier_detection/test_median.py | 2 +- tests/test_infrastructure.py | 16 ++++++++++++++++ 4 files changed, 26 insertions(+), 14 deletions(-) rename tests/helpers.py => src/stcal/testing_helpers.py (70%) delete mode 100644 tests/conftest.py create mode 100644 tests/test_infrastructure.py diff --git a/tests/helpers.py b/src/stcal/testing_helpers.py similarity index 70% rename from tests/helpers.py rename to src/stcal/testing_helpers.py index 39f65e6d..df9abb09 100644 --- a/tests/helpers.py +++ b/src/stcal/testing_helpers.py @@ -1,5 +1,10 @@ import tracemalloc + +class MemoryThresholdExceeded(Exception): + pass + + class MemoryThreshold: """ Context manager to check peak memory usage against an expected threshold. @@ -26,7 +31,7 @@ def __exit__(self, exc_type, exc_value, traceback): _, peak = tracemalloc.get_traced_memory() tracemalloc.stop() - assert peak <= self.expected_usage, ( - "Peak memory usage exceeded expected usage: " - f"{peak / 1024:.2f} KB > {self.expected_usage / 1024:.2f} KB" - ) \ No newline at end of file + if peak > self.expected_usage: + msg = ("Peak memory usage exceeded expected usage: " + f"{peak / 1024:.2f} KB > {self.expected_usage / 1024:.2f} KB") + raise MemoryThresholdExceeded(msg) diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index b4e4d58a..00000000 --- a/tests/conftest.py +++ /dev/null @@ -1,9 +0,0 @@ -import pytest -from tests.helpers import MemoryThreshold - - -@pytest.fixture -def memory_threshold(expected_usage): - """Fixture to check peak memory usage against an expected threshold.""" - with MemoryThreshold(expected_usage) as tracker: - yield tracker \ No newline at end of file diff --git a/tests/outlier_detection/test_median.py b/tests/outlier_detection/test_median.py index bc90f018..1be1b789 100644 --- a/tests/outlier_detection/test_median.py +++ b/tests/outlier_detection/test_median.py @@ -10,7 +10,7 @@ _OnDiskMedian, nanmedian3D, ) -from tests.helpers import MemoryThreshold +from stcal.testing_helpers import MemoryThreshold def test_disk_appendable_array(tmp_path): diff --git a/tests/test_infrastructure.py b/tests/test_infrastructure.py new file mode 100644 index 00000000..c1a8734a --- /dev/null +++ b/tests/test_infrastructure.py @@ -0,0 +1,16 @@ +"""Tests of custom testing infrastructure""" + +import pytest +import numpy as np +from stcal.testing_helpers import MemoryThreshold, MemoryThresholdExceeded + + +def test_memory_threshold(): + with MemoryThreshold(1000): + buff = np.empty(500, dtype=np.uint8) + + +def test_memory_threshold_raise(): + with pytest.raises(MemoryThresholdExceeded): + with MemoryThreshold(1000): + buff = np.empty(2000, dtype=np.uint8) \ No newline at end of file From f9aa62273e7173d0ada0293fbb28a304218c5505 Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Wed, 9 Oct 2024 12:30:30 -0400 Subject: [PATCH 06/14] added changelog fragment --- changes/299.general.rst | 1 + src/stcal/testing_helpers.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 changes/299.general.rst diff --git a/changes/299.general.rst b/changes/299.general.rst new file mode 100644 index 00000000..c13ddf4b --- /dev/null +++ b/changes/299.general.rst @@ -0,0 +1 @@ +Add infrastructure for testing memory usage diff --git a/src/stcal/testing_helpers.py b/src/stcal/testing_helpers.py index df9abb09..e3da191c 100644 --- a/src/stcal/testing_helpers.py +++ b/src/stcal/testing_helpers.py @@ -1,6 +1,5 @@ import tracemalloc - class MemoryThresholdExceeded(Exception): pass From 5116c338936765a2712acba8b602e99004aa5f95 Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Wed, 9 Oct 2024 12:51:56 -0400 Subject: [PATCH 07/14] attempt fix test for py312 --- tests/test_infrastructure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_infrastructure.py b/tests/test_infrastructure.py index c1a8734a..5be2192c 100644 --- a/tests/test_infrastructure.py +++ b/tests/test_infrastructure.py @@ -7,7 +7,7 @@ def test_memory_threshold(): with MemoryThreshold(1000): - buff = np.empty(500, dtype=np.uint8) + buff = np.empty(200, dtype=np.uint8) def test_memory_threshold_raise(): From 0d5e58b71ebd4e3481295ed2298b4d39308fad23 Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Mon, 14 Oct 2024 09:29:25 -0400 Subject: [PATCH 08/14] change per @penaguerrero comment --- src/stcal/testing_helpers.py | 13 +++++++++++-- tests/test_infrastructure.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/stcal/testing_helpers.py b/src/stcal/testing_helpers.py index e3da191c..390e310b 100644 --- a/src/stcal/testing_helpers.py +++ b/src/stcal/testing_helpers.py @@ -1,5 +1,7 @@ import tracemalloc +MEMORY_UNIT_CONVERSION = {"B": 1, "KB": 1024, "MB": 1024 ** 2, "GB": 1024 ** 3} + class MemoryThresholdExceeded(Exception): pass @@ -13,14 +15,19 @@ class MemoryThreshold: # code that should not exceed expected """ - def __init__(self, expected_usage): + def __init__(self, expected_usage, log_units="KB"): """ Parameters ---------- expected_usage : int Expected peak memory usage in bytes + + log_units : str, optional + Units in which to display memory usage for error message. + Supported are "B", "KB", "MB", "GB". Default is "KB". """ self.expected_usage = expected_usage + self.log_units = log_units def __enter__(self): tracemalloc.start() @@ -31,6 +38,8 @@ def __exit__(self, exc_type, exc_value, traceback): tracemalloc.stop() if peak > self.expected_usage: + scaling = MEMORY_UNIT_CONVERSION[self.log_units] msg = ("Peak memory usage exceeded expected usage: " - f"{peak / 1024:.2f} KB > {self.expected_usage / 1024:.2f} KB") + f"{peak / scaling:.2f} {self.log_units} > " + f"{self.expected_usage / scaling:.2f} {self.log_units} ") raise MemoryThresholdExceeded(msg) diff --git a/tests/test_infrastructure.py b/tests/test_infrastructure.py index 5be2192c..afac9e91 100644 --- a/tests/test_infrastructure.py +++ b/tests/test_infrastructure.py @@ -10,7 +10,7 @@ def test_memory_threshold(): buff = np.empty(200, dtype=np.uint8) -def test_memory_threshold_raise(): +def test_memory_threshold_exceeded(): with pytest.raises(MemoryThresholdExceeded): with MemoryThreshold(1000): buff = np.empty(2000, dtype=np.uint8) \ No newline at end of file From 2a7d03a8164eac5f2a28bb430578a109b45e69ad Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Mon, 14 Oct 2024 09:31:21 -0400 Subject: [PATCH 09/14] added back in a docstring change brett wanted --- src/stcal/testing_helpers.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/stcal/testing_helpers.py b/src/stcal/testing_helpers.py index 390e310b..3c6f0fb7 100644 --- a/src/stcal/testing_helpers.py +++ b/src/stcal/testing_helpers.py @@ -13,6 +13,14 @@ class MemoryThreshold: example usage: with MemoryThreshold(expected_usage): # code that should not exceed expected + + If the code in the with statement uses more than the expected_usage + memory (in bytes) a ``MemoryThresholdExceeded`` exception + will be raised. + + Note that this class does not prevent allocations beyond the threshold + and only checks the actual peak allocations to the threshold at the + end of the with statement. """ def __init__(self, expected_usage, log_units="KB"): From 0c44cd6b75ace8405cac705e052200d5fc4d87af Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Mon, 14 Oct 2024 12:28:18 -0400 Subject: [PATCH 10/14] make expected usage a string with units --- src/stcal/testing_helpers.py | 24 +++++++++++------------- tests/test_infrastructure.py | 6 +++--- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/stcal/testing_helpers.py b/src/stcal/testing_helpers.py index 3c6f0fb7..188cb58c 100644 --- a/src/stcal/testing_helpers.py +++ b/src/stcal/testing_helpers.py @@ -23,19 +23,17 @@ class MemoryThreshold: end of the with statement. """ - def __init__(self, expected_usage, log_units="KB"): + def __init__(self, expected_usage): """ Parameters ---------- - expected_usage : int - Expected peak memory usage in bytes - - log_units : str, optional - Units in which to display memory usage for error message. - Supported are "B", "KB", "MB", "GB". Default is "KB". + expected_usage : str + Expected peak memory usage expressed as a whitespace-separated string + with a number and a memory unit (e.g. "100 KB"). + Supported units are "B", "KB", "MB", "GB". """ - self.expected_usage = expected_usage - self.log_units = log_units + expected, self.units = expected_usage.upper().split() + self.expected_usage_bytes = float(expected) * MEMORY_UNIT_CONVERSION[self.units] def __enter__(self): tracemalloc.start() @@ -45,9 +43,9 @@ def __exit__(self, exc_type, exc_value, traceback): _, peak = tracemalloc.get_traced_memory() tracemalloc.stop() - if peak > self.expected_usage: - scaling = MEMORY_UNIT_CONVERSION[self.log_units] + if peak > self.expected_usage_bytes: + scaling = MEMORY_UNIT_CONVERSION[self.units] msg = ("Peak memory usage exceeded expected usage: " - f"{peak / scaling:.2f} {self.log_units} > " - f"{self.expected_usage / scaling:.2f} {self.log_units} ") + f"{peak / scaling:.2f} {self.units} > " + f"{self.expected_usage_bytes / scaling:.2f} {self.units} ") raise MemoryThresholdExceeded(msg) diff --git a/tests/test_infrastructure.py b/tests/test_infrastructure.py index afac9e91..3998ec7c 100644 --- a/tests/test_infrastructure.py +++ b/tests/test_infrastructure.py @@ -6,11 +6,11 @@ def test_memory_threshold(): - with MemoryThreshold(1000): + with MemoryThreshold("1 KB"): buff = np.empty(200, dtype=np.uint8) def test_memory_threshold_exceeded(): with pytest.raises(MemoryThresholdExceeded): - with MemoryThreshold(1000): - buff = np.empty(2000, dtype=np.uint8) \ No newline at end of file + with MemoryThreshold("500. B"): + buff = np.empty(1000, dtype=np.uint8) \ No newline at end of file From a13f6d3fb12b30a55791457c46044b4ec544bf6b Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Mon, 14 Oct 2024 12:33:54 -0400 Subject: [PATCH 11/14] update unit test to use new syntax --- src/stcal/testing_helpers.py | 6 +++--- tests/outlier_detection/test_median.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/stcal/testing_helpers.py b/src/stcal/testing_helpers.py index 188cb58c..7c322a30 100644 --- a/src/stcal/testing_helpers.py +++ b/src/stcal/testing_helpers.py @@ -1,6 +1,6 @@ import tracemalloc -MEMORY_UNIT_CONVERSION = {"B": 1, "KB": 1024, "MB": 1024 ** 2, "GB": 1024 ** 3} +MEMORY_UNIT_CONVERSION = {"B": 1, "KB": 1024, "MB": 1024 ** 2, "GB": 1024 ** 3, "TB": 1024 ** 4} class MemoryThresholdExceeded(Exception): pass @@ -15,7 +15,7 @@ class MemoryThreshold: # code that should not exceed expected If the code in the with statement uses more than the expected_usage - memory (in bytes) a ``MemoryThresholdExceeded`` exception + memory a ``MemoryThresholdExceeded`` exception will be raised. Note that this class does not prevent allocations beyond the threshold @@ -30,7 +30,7 @@ def __init__(self, expected_usage): expected_usage : str Expected peak memory usage expressed as a whitespace-separated string with a number and a memory unit (e.g. "100 KB"). - Supported units are "B", "KB", "MB", "GB". + Supported units are "B", "KB", "MB", "GB", "TB". """ expected, self.units = expected_usage.upper().split() self.expected_usage_bytes = float(expected) * MEMORY_UNIT_CONVERSION[self.units] diff --git a/tests/outlier_detection/test_median.py b/tests/outlier_detection/test_median.py index 1be1b789..62a30dac 100644 --- a/tests/outlier_detection/test_median.py +++ b/tests/outlier_detection/test_median.py @@ -225,7 +225,7 @@ def test_memory_computer(in_memory, tmp_path): expected_mem = frame_size * 2.5 # compute the median while tracking memory usage - with MemoryThreshold(expected_mem): + with MemoryThreshold(str(expected_mem) + " B"): computer = MedianComputer(shp, in_memory=in_memory, tempdir=tmp_path) for i in range(shp[0]): frame = np.full(shp[1:], i, dtype=np.float32) From e75a29cf718bde3e26d5acea85eb79f7279b1ff5 Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Mon, 14 Oct 2024 12:47:19 -0400 Subject: [PATCH 12/14] fix py312 test --- tests/test_infrastructure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_infrastructure.py b/tests/test_infrastructure.py index 3998ec7c..4b11b053 100644 --- a/tests/test_infrastructure.py +++ b/tests/test_infrastructure.py @@ -7,7 +7,7 @@ def test_memory_threshold(): with MemoryThreshold("1 KB"): - buff = np.empty(200, dtype=np.uint8) + buff = np.empty(100, dtype=np.uint8) def test_memory_threshold_exceeded(): From 5c853dcff2dd9070b35079fd694b2454e2d57513 Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Mon, 14 Oct 2024 15:20:13 -0400 Subject: [PATCH 13/14] attempted fix for test again --- tests/test_infrastructure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_infrastructure.py b/tests/test_infrastructure.py index 4b11b053..fd77cdd1 100644 --- a/tests/test_infrastructure.py +++ b/tests/test_infrastructure.py @@ -6,8 +6,8 @@ def test_memory_threshold(): - with MemoryThreshold("1 KB"): - buff = np.empty(100, dtype=np.uint8) + with MemoryThreshold("10 KB"): + buff = np.empty(1000, dtype=np.uint8) def test_memory_threshold_exceeded(): From 12c58863fdb6b05cdcfe17e2ef05d78f808c8d6f Mon Sep 17 00:00:00 2001 From: Ned Molter Date: Mon, 14 Oct 2024 15:26:14 -0400 Subject: [PATCH 14/14] attempted fix test for oldest deps --- tests/test_infrastructure.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_infrastructure.py b/tests/test_infrastructure.py index fd77cdd1..37e68482 100644 --- a/tests/test_infrastructure.py +++ b/tests/test_infrastructure.py @@ -7,10 +7,10 @@ def test_memory_threshold(): with MemoryThreshold("10 KB"): - buff = np.empty(1000, dtype=np.uint8) + buff = np.ones(1000, dtype=np.uint8) def test_memory_threshold_exceeded(): with pytest.raises(MemoryThresholdExceeded): with MemoryThreshold("500. B"): - buff = np.empty(1000, dtype=np.uint8) \ No newline at end of file + buff = np.ones(10000, dtype=np.uint8) \ No newline at end of file