From 67a5dbf2410556cfd110887ece40592d6782fd30 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 18 Nov 2024 11:37:23 +0000 Subject: [PATCH 1/3] fix capitalization --- spynnaker/pyNN/utilities/neo_buffer_database.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spynnaker/pyNN/utilities/neo_buffer_database.py b/spynnaker/pyNN/utilities/neo_buffer_database.py index 1086cad607..1bb85bb74b 100644 --- a/spynnaker/pyNN/utilities/neo_buffer_database.py +++ b/spynnaker/pyNN/utilities/neo_buffer_database.py @@ -1079,7 +1079,9 @@ def get_spike_counts( self.__get_segment_info() metadata = self.__get_recording_metadata(pop_label, SPIKES) if metadata is None: - return {} + if SpynnakerDataView.is_ran_last(): + raise ConfigurationException( + f"{pop_label} did not record spikes") (rec_id, _, buffered_type, _, _, pop_size, _, n_colour_bits) = metadata @@ -1291,7 +1293,7 @@ def csv_segment( If the recording metadata not setup correctly """ if not os.path.isfile(csv_file): - raise SpynnakerException("PLease call csv_block_metadata first") + raise SpynnakerException("Please call csv_block_metadata first") with open(csv_file, 'a', newline='', encoding="utf-8") as csvfile: csv_writer = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) From acab4dc693031eefc083147c648dc0c744f2660a Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 18 Nov 2024 13:24:14 +0000 Subject: [PATCH 2/3] fixes for get data with recording started later --- spynnaker/pyNN/models/recorder.py | 24 +++++-- .../pyNN/utilities/neo_buffer_database.py | 14 ++-- .../test_recording_later_additions.py | 69 ++++++++++++++++--- unittests/test_pop_views_assembly/test_csv.py | 19 ++--- 4 files changed, 99 insertions(+), 27 deletions(-) diff --git a/spynnaker/pyNN/models/recorder.py b/spynnaker/pyNN/models/recorder.py index 94c0b4f9f2..65f422d685 100644 --- a/spynnaker/pyNN/models/recorder.py +++ b/spynnaker/pyNN/models/recorder.py @@ -271,18 +271,30 @@ def csv_neo_block( wrote_metadata = db.csv_block_metadata( csv_file, pop_label, annotations) if wrote_metadata: - db.csv_segment( - csv_file, pop_label, variables, view_indexes) + db.csv_segment(csv_file, pop_label, variables, + view_indexes, allow_missing=True) - with NeoBufferDatabase() as db: - if SpynnakerDataView.is_reset_last(): + if SpynnakerDataView.is_reset_last(): + if wrote_metadata: logger.warning( "Due to the call directly after reset, " "the data will only contain {} segments", SpynnakerDataView.get_segment_counter() - 1) + return else: - db.csv_segment( - csv_file, pop_label, variables, view_indexes) + raise ConfigurationException( + f"Unable to write data for {pop_label}") + + with NeoBufferDatabase() as db: + if not wrote_metadata: + wrote_metadata = db.csv_block_metadata( + csv_file, pop_label, annotations) + if wrote_metadata: + db.csv_segment(csv_file, pop_label, variables, + view_indexes, allow_missing=False) + else: + raise ConfigurationException( + f"Unable to write data for {pop_label}") def __append_current_segment( self, block: neo.Block, variables: Names, diff --git a/spynnaker/pyNN/utilities/neo_buffer_database.py b/spynnaker/pyNN/utilities/neo_buffer_database.py index 1bb85bb74b..37264a12dc 100644 --- a/spynnaker/pyNN/utilities/neo_buffer_database.py +++ b/spynnaker/pyNN/utilities/neo_buffer_database.py @@ -1159,7 +1159,7 @@ def __add_data( def __read_and_csv_data( self, pop_label: str, variable: str, csv_writer: CSVWriter, - view_indexes: ViewIndices, t_stop: float): + view_indexes: ViewIndices, t_stop: float, allow_missing=False): """ Reads the data for one variable and adds it to the CSV file. @@ -1178,7 +1178,11 @@ def __read_and_csv_data( """ metadata = self.__get_recording_metadata(pop_label, variable) if metadata is None: - return + if allow_missing: + return + else: + raise ConfigurationException( + f"No data for {pop_label=} {variable=}") (rec_id, data_type, buffer_type, t_start, sampling_interval_ms, pop_size, units, n_colour_bits) = metadata @@ -1271,7 +1275,7 @@ def get_full_block( def csv_segment( self, csv_file: str, pop_label: str, variables: Names, - view_indexes: ViewIndices = None): + view_indexes: ViewIndices, allow_missing: bool): """ Writes the data including metadata to a CSV file. @@ -1304,8 +1308,8 @@ def csv_segment( csv_writer, segment_number, rec_datetime) for variable in self.__clean_variables(variables, pop_label): - self.__read_and_csv_data( - pop_label, variable, csv_writer, view_indexes, t_stop) + self.__read_and_csv_data(pop_label, variable, csv_writer, + view_indexes, t_stop, allow_missing) def csv_block_metadata( self, csv_file: str, pop_label: str, diff --git a/spynnaker_integration_tests/test_various/test_recording_later_additions.py b/spynnaker_integration_tests/test_various/test_recording_later_additions.py index a52882d76f..9d0f0536a4 100644 --- a/spynnaker_integration_tests/test_various/test_recording_later_additions.py +++ b/spynnaker_integration_tests/test_various/test_recording_later_additions.py @@ -12,14 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. - +import os +from spinn_front_end_common.utilities.exceptions import ConfigurationException import pyNN.spiNNaker as sim from spinnaker_testbase import BaseTestCase class TestRecordingLaterAdditions(BaseTestCase): - def do_simple(self): + def do_later_additions(self): sim.setup(timestep=1.0) sim.set_number_of_neurons_per_core(sim.IF_curr_exp, 100) @@ -31,19 +32,58 @@ def do_simple(self): synapse_type=sim.StaticSynapse(weight=5, delay=1)) pop_a.record(["spikes", "v"]) + pop_b = sim.Population(1, sim.IF_curr_exp(), label="pop_b") + sim.Projection(input_pop, pop_b, sim.OneToOneConnector(), + synapse_type=sim.StaticSynapse(weight=5, delay=1)) + pop_c = sim.Population(1, sim.IF_curr_exp(), label="pop_c") sim.Projection(input_pop, pop_c, sim.OneToOneConnector(), synapse_type=sim.StaticSynapse(weight=5, delay=1)) pop_c.record(["spikes"]) + sim.run(10) + + # Did not record all these should fail + with self.assertRaises(ConfigurationException): + pop_b.get_data(variables=["spikes", "v"]) + with self.assertRaises(ConfigurationException): + pop_b.get_spike_counts() + with self.assertRaises(ConfigurationException): + pop_b.write_data("test_b_bad.csv", ["spikes", "v"]) + with self.assertRaises(ConfigurationException): + pop_c.get_data(variables=["spikes", "v"]) + with self.assertRaises(ConfigurationException): + pop_c.write_data("test_c_bad.csv", ["spikes", "v"]) + sim.reset() - pop_b = sim.Population(1, sim.IF_curr_exp(), label="pop_b") - sim.Projection(input_pop, pop_b, sim.OneToOneConnector(), - synapse_type=sim.StaticSynapse(weight=5, delay=1)) - pop_b.record(["spikes", "v"]) + # No recording from previous segment so should fail + with self.assertRaises(ConfigurationException): + pop_b.get_data(variables=["spikes", "v"]) + with self.assertRaises(ConfigurationException): + pop_b.write_data("test_b_bad.csv", ["spikes", "v"]) + # Only includes this segment so should fail + with self.assertRaises(ConfigurationException): + pop_b.get_spike_counts() + # Some recording from previous will wok + pop_c.get_data(variables=["spikes", "v"]) + pop_c.write_data("test_c_1.csv", ["spikes", "v"]) + pop_b.record(["spikes", "v"]) pop_c.record(["v"]) + + # No recording from previous segment so should fail + with self.assertRaises(ConfigurationException): + pop_b.get_data(variables=["spikes", "v"]) + with self.assertRaises(ConfigurationException): + pop_b.write_data("test_b_bad.csv", ["spikes", "v"]) + # Only includes this segment so should fail + with self.assertRaises(ConfigurationException): + pop_b.get_spike_counts() + # Some recording from previous will wok + pop_c.get_data(variables=["spikes", "v"]) + pop_c.write_data("test_c_1.csv", ["spikes", "v"]) + sim.run(20) neo_a = pop_a.get_data(variables=["spikes", "v"]) @@ -116,5 +156,18 @@ def do_simple(self): pop_c.write_data("test_c.csv", ["spikes", "v"]) sim.end() - def test_simple(self): - self.runsafe(self.do_simple) + def cleanup(self, file): + try: + if os.path.exists(file): + os.remove(file) + except Exception: + pass + + def test_later_additions(self): + self.cleanup("test_a.csv") + self.cleanup("test_b.csv") + self.cleanup("test_b_bad.csv") + self.cleanup("test_c.csv") + self.cleanup("test_c_1.csv") + self.cleanup("test_c_bad.csv") + self.runsafe(self.do_later_additions) diff --git a/unittests/test_pop_views_assembly/test_csv.py b/unittests/test_pop_views_assembly/test_csv.py index a8285702a1..c4d9fd080b 100644 --- a/unittests/test_pop_views_assembly/test_csv.py +++ b/unittests/test_pop_views_assembly/test_csv.py @@ -62,7 +62,8 @@ def test_write(self): with NeoBufferDatabase(my_buffer) as db: db.csv_block_metadata( my_csv, "pop_1", annotations={"foo": 12, "bar": 34}) - db.csv_segment(my_csv, "pop_1", variables="all") + db.csv_segment(my_csv, "pop_1", variables="all", + view_indexes=None, allow_missing=False) neo = NeoCsv().read_csv(my_csv) # All annotations converted to String and not back @@ -88,7 +89,7 @@ def test_view(self): db.csv_block_metadata(my_csv, "pop_1", annotations=None) db.csv_segment( my_csv, "pop_1", variables=["spikes", "v"], - view_indexes=[2, 4, 7, 8]) + view_indexes=[2, 4, 7, 8], allow_missing=False) neo = NeoCsv().read_csv(my_csv) @@ -109,7 +110,8 @@ def test_over_view(self): my_csv = os.path.join(my_dir, "test_over_view.csv") with NeoBufferDatabase(my_buffer) as db: db.csv_block_metadata(my_csv, "pop_1", annotations=None) - db.csv_segment(my_csv, "pop_1", variables="all") + db.csv_segment(my_csv, "pop_1", variables="all", + view_indexes=None, allow_missing=False) neo = NeoCsv().read_csv(my_csv) spikes = neo_convertor.convert_spikes(neo) @@ -129,8 +131,8 @@ def test_over_sub_view(self): my_csv = os.path.join(my_dir, "test_over_sub_view.csv") with NeoBufferDatabase(my_buffer) as db: db.csv_block_metadata(my_csv, "pop_1", annotations=None) - db.csv_segment( - my_csv, "pop_1", variables="all", view_indexes=[2, 4]) + db.csv_segment(my_csv, "pop_1", variables="all", + view_indexes=[2, 4], allow_missing=False) neo = NeoCsv().read_csv(my_csv) spikes = neo_convertor.convert_spikes(neo) @@ -150,8 +152,8 @@ def test_no_intersection(self): my_csv = os.path.join(my_dir, "test_no_intersection.csv") with NeoBufferDatabase(my_buffer) as db: db.csv_block_metadata(my_csv, "pop_1", annotations=None) - db.csv_segment( - my_csv, "pop_1", variables="all", view_indexes=[4, 6]) + db.csv_segment(my_csv, "pop_1", variables="all", + view_indexes=[4, 6], allow_missing=False) neo = NeoCsv().read_csv(my_csv) spikes = neo_convertor.convert_spikes(neo) @@ -170,7 +172,8 @@ def test_rewiring(self): my_csv = os.path.join(my_dir, "test_rewiring.csv") with NeoBufferDatabase(my_buffer) as db: db.csv_block_metadata(my_csv, "pop_1", annotations=None) - db.csv_segment(my_csv, "pop_1", variables="all") + db.csv_segment(my_csv, "pop_1", variables="all", + view_indexes=None, allow_missing=False) neo = NeoCsv().read_csv(my_csv) formation_events = neo.segments[0].events[0] elimination_events = neo.segments[0].events[1] From 5378eb864053ddf65337f2c81820775c5cc7701b Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 18 Nov 2024 13:37:30 +0000 Subject: [PATCH 3/3] get spike fails after a reset --- spynnaker/pyNN/utilities/neo_buffer_database.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/spynnaker/pyNN/utilities/neo_buffer_database.py b/spynnaker/pyNN/utilities/neo_buffer_database.py index 37264a12dc..a21a423b43 100644 --- a/spynnaker/pyNN/utilities/neo_buffer_database.py +++ b/spynnaker/pyNN/utilities/neo_buffer_database.py @@ -1079,9 +1079,8 @@ def get_spike_counts( self.__get_segment_info() metadata = self.__get_recording_metadata(pop_label, SPIKES) if metadata is None: - if SpynnakerDataView.is_ran_last(): - raise ConfigurationException( - f"{pop_label} did not record spikes") + raise ConfigurationException( + f"{pop_label} did not record spikes") (rec_id, _, buffered_type, _, _, pop_size, _, n_colour_bits) = metadata @@ -1159,7 +1158,7 @@ def __add_data( def __read_and_csv_data( self, pop_label: str, variable: str, csv_writer: CSVWriter, - view_indexes: ViewIndices, t_stop: float, allow_missing=False): + view_indexes: ViewIndices, t_stop: float, allow_missing: bool): """ Reads the data for one variable and adds it to the CSV file. @@ -1175,6 +1174,8 @@ def __read_and_csv_data( :param view_indexes: :type view_indexes: None, ~numpy.array or list(int) :param float t_stop: + :param allow_missing: Flag to say if data for missing variable + should raise an exception """ metadata = self.__get_recording_metadata(pop_label, variable) if metadata is None: