From a7839e4d46a5996c16e2d1a3e3e43603ab3954c7 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Fri, 1 Nov 2024 07:56:03 +0000 Subject: [PATCH 1/4] handle spikes /tstop when running forever --- .../pyNN/utilities/neo_buffer_database.py | 31 ++++++++++++------- spynnaker/pyNN/utilities/neo_csv.py | 10 ++++-- .../test_spike_run_forever_again.py | 9 ++++-- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/spynnaker/pyNN/utilities/neo_buffer_database.py b/spynnaker/pyNN/utilities/neo_buffer_database.py index 4c9fb377dc..1a2ae28b04 100644 --- a/spynnaker/pyNN/utilities/neo_buffer_database.py +++ b/spynnaker/pyNN/utilities/neo_buffer_database.py @@ -30,6 +30,8 @@ import neo # type: ignore[import] from spinn_utilities.log import FormatAdapter +from spinn_utilities.logger_utils import warn_once + from spinnman.messages.eieio.data_messages import EIEIODataHeader @@ -155,19 +157,22 @@ def write_t_stop(self) -> None: The database must be writable for this to work! """ t_stop = SpynnakerDataView.get_current_run_time_ms() + if t_stop == 0: + if SpynnakerDataView.get_current_run_timesteps() is None: + return self.execute( """ UPDATE segment SET t_stop = ? """, (t_stop,)) - def __get_segment_info(self) -> Tuple[int, datetime, float, float, str]: + def __get_segment_info( + self) -> Tuple[int, datetime, Optional[float], float, str]: """ Gets the metadata for the segment. :return: segment number, record time, last run time recorded, simulator timestep in ms, simulator name - :rtype: tuple(int, ~datetime.datetime, float, float, str) :raises \ ~spinn_front_end_common.utilities.exceptions.ConfigurationException: If the recording metadata not setup correctly @@ -180,12 +185,7 @@ def __get_segment_info(self) -> Tuple[int, datetime, float, float, str]: """): t_str = self._string(row[self._REC_DATETIME]) time = datetime.strptime(t_str, "%Y-%m-%d %H:%M:%S.%f") - if row[self._T_STOP] is None: - t_stop = 0.0 - logger.warning("Data from a virtual run will be empty") - else: - t_stop = row[self._T_STOP] - return (row[self._SEGMENT_NUMBER], time, t_stop, row[self._DT], + return (row[self._SEGMENT_NUMBER], time, row[self._T_STOP], row[self._DT], self._string(row[self._SIMULATOR])) raise ConfigurationException( "No recorded data. Did the simulation run?") @@ -1092,7 +1092,8 @@ def get_spike_counts( def __add_data( self, pop_label: str, variable: str, - segment: neo.Segment, view_indexes: ViewIndices, t_stop: float): + segment: neo.Segment, view_indexes: ViewIndices, + t_stop: Optional[float]): """ Gets the data as a Numpy array for one population and variable. @@ -1105,7 +1106,6 @@ def __add_data( :param str variable: :param ~neo.core.Segment segment: Segment to add data to - :param float t_stop: :raises \ ~spinn_front_end_common.utilities.exceptions.ConfigurationException: If the recording metadata not setup correctly @@ -1137,13 +1137,21 @@ def __add_data( rec_id, view_indexes, buffer_type, n_colour_bits, variable) sampling_rate = 1000 / sampling_interval_ms * quantities.Hz + if t_stop is None: + if len(spikes) == 0: + t_stop = 0 + warn_once(logger, "No spike data. Setting end time to 0") + else: + t_stop = numpy.amax(spikes, 0)[1] + warn_once(logger, "Unknown how long the simulation ran. " + "So using max spike as stop time") self._insert_spike_data( view_indexes, segment, spikes, t_start, t_stop, sampling_rate) def __read_and_csv_data( self, pop_label: str, variable: str, csv_writer: CSVWriter, - view_indexes: ViewIndices, t_stop: float): + view_indexes: ViewIndices, t_stop: Optional[float]): """ Reads the data for one variable and adds it to the CSV file. @@ -1158,7 +1166,6 @@ def __read_and_csv_data( :param ~csv.writer csv_writer: Open CSV writer to write to :param view_indexes: :type view_indexes: None, ~numpy.array or list(int) - :param float t_stop: """ (rec_id, data_type, buffer_type, t_start, sampling_interval_ms, pop_size, units, n_colour_bits) = \ diff --git a/spynnaker/pyNN/utilities/neo_csv.py b/spynnaker/pyNN/utilities/neo_csv.py index f30a750f3d..845d0bf667 100644 --- a/spynnaker/pyNN/utilities/neo_csv.py +++ b/spynnaker/pyNN/utilities/neo_csv.py @@ -71,7 +71,7 @@ class NeoCsv(object): def _csv_variable_metdata( self, csv_writer: CSVWriter, variable_type: str, variable: str, - t_start: float, t_stop: float, sampling_interval_ms: float, + t_start: float, t_stop: Optional[float], sampling_interval_ms: float, units: Optional[str]): """ Writes the metadata for a variable to CSV @@ -85,8 +85,12 @@ def _csv_variable_metdata( :param str units: """ csv_writer.writerow([variable_type, variable]) - csv_writer.writerow([self._T_START, t_start * ms]) - csv_writer.writerow([self._T_STOP, t_stop * ms]) + if t_stop is None: + csv_writer.writerow([self._T_START, "Unknown"]) + csv_writer.writerow([self._T_STOP, "Unknown"]) + else: + csv_writer.writerow([self._T_START, t_start * ms]) + csv_writer.writerow([self._T_STOP, t_stop * ms]) sampling_period = sampling_interval_ms * ms csv_writer.writerow([self._SAMPLING_PERIOD, sampling_period]) if units is None: diff --git a/spynnaker_integration_tests/test_external_devices/test_spike_run_forever_again.py b/spynnaker_integration_tests/test_external_devices/test_spike_run_forever_again.py index 35c5aefce4..027f421837 100644 --- a/spynnaker_integration_tests/test_external_devices/test_spike_run_forever_again.py +++ b/spynnaker_integration_tests/test_external_devices/test_spike_run_forever_again.py @@ -14,7 +14,7 @@ from time import sleep import pyNN.spiNNaker as sim from spinnaker_testbase import BaseTestCase - +from spynnaker.pyNN.utilities.neo_convertor import count_spikes spike_receive_count = 0 spike_send_count = 0 @@ -52,6 +52,7 @@ def do_run(): label="sender") pop = sim.Population( 1, sim.IF_curr_exp(), label="pop_1") + pop.record("spikes") sim.Projection(ssa, pop, sim.OneToOneConnector(), sim.StaticSynapse(weight=5, delay=1)) sim.external_devices.activate_live_output_for( @@ -59,8 +60,12 @@ def do_run(): for _ in range(5): sim.external_devices.run_forever() + neo = pop.get_data("spikes") + spikes = count_spikes(neo) + pop.write_data("test.csv", "spikes") + sim.end() - print(spike_send_count, spike_receive_count) + print(spike_send_count, spike_receive_count, spikes) class TestSpikeRunForeverAgain(BaseTestCase): From 00b40de4ea76cd11df491751d25f4056c2c3aaa5 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Fri, 1 Nov 2024 08:19:23 +0000 Subject: [PATCH 2/4] flake8 --- spynnaker/pyNN/utilities/neo_buffer_database.py | 4 ++-- spynnaker/pyNN/utilities/neo_csv.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/spynnaker/pyNN/utilities/neo_buffer_database.py b/spynnaker/pyNN/utilities/neo_buffer_database.py index 1a2ae28b04..7051696ccd 100644 --- a/spynnaker/pyNN/utilities/neo_buffer_database.py +++ b/spynnaker/pyNN/utilities/neo_buffer_database.py @@ -185,8 +185,8 @@ def __get_segment_info( """): t_str = self._string(row[self._REC_DATETIME]) time = datetime.strptime(t_str, "%Y-%m-%d %H:%M:%S.%f") - return (row[self._SEGMENT_NUMBER], time, row[self._T_STOP], row[self._DT], - self._string(row[self._SIMULATOR])) + return (row[self._SEGMENT_NUMBER], time, row[self._T_STOP], + row[self._DT], self._string(row[self._SIMULATOR])) raise ConfigurationException( "No recorded data. Did the simulation run?") diff --git a/spynnaker/pyNN/utilities/neo_csv.py b/spynnaker/pyNN/utilities/neo_csv.py index 845d0bf667..3e773c3c40 100644 --- a/spynnaker/pyNN/utilities/neo_csv.py +++ b/spynnaker/pyNN/utilities/neo_csv.py @@ -71,8 +71,8 @@ class NeoCsv(object): def _csv_variable_metdata( self, csv_writer: CSVWriter, variable_type: str, variable: str, - t_start: float, t_stop: Optional[float], sampling_interval_ms: float, - units: Optional[str]): + t_start: float, t_stop: Optional[float], + sampling_interval_ms: float, units: Optional[str]) -> None: """ Writes the metadata for a variable to CSV @@ -80,7 +80,7 @@ def _csv_variable_metdata( :param str variable_type: :param str variable: :param float t_start: - :param float t_stop: + :param t_stop: :param float sampling_interval_ms: :param str units: """ From 4714bc162801a2c84aa0a54fea7c2924a5e6d16c Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Mon, 4 Nov 2024 07:02:56 +0000 Subject: [PATCH 3/4] refill send buffer on new run forever --- .../test_external_devices/test_spike_run_forever_again.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/spynnaker_integration_tests/test_external_devices/test_spike_run_forever_again.py b/spynnaker_integration_tests/test_external_devices/test_spike_run_forever_again.py index 027f421837..5d88e93c10 100644 --- a/spynnaker_integration_tests/test_external_devices/test_spike_run_forever_again.py +++ b/spynnaker_integration_tests/test_external_devices/test_spike_run_forever_again.py @@ -52,7 +52,11 @@ def do_run(): label="sender") pop = sim.Population( 1, sim.IF_curr_exp(), label="pop_1") - pop.record("spikes") + input_pop = sim.Population( + 1, sim.SpikeSourceArray( + spike_times=[0, 2, 4, 6, 8, 10, 12, 14, 16, 18]), + label="input") + input_pop.record("spikes") sim.Projection(ssa, pop, sim.OneToOneConnector(), sim.StaticSynapse(weight=5, delay=1)) sim.external_devices.activate_live_output_for( @@ -62,7 +66,6 @@ def do_run(): sim.external_devices.run_forever() neo = pop.get_data("spikes") spikes = count_spikes(neo) - pop.write_data("test.csv", "spikes") sim.end() print(spike_send_count, spike_receive_count, spikes) From 77d971c7d912b85a40ada0216a3b91f5cc2f0744 Mon Sep 17 00:00:00 2001 From: "Christian Y. Brenninkmeijer" Date: Tue, 5 Nov 2024 06:30:01 +0000 Subject: [PATCH 4/4] fix test --- .../test_spike_run_forever_again.py | 86 ++++++++++--------- 1 file changed, 47 insertions(+), 39 deletions(-) diff --git a/spynnaker_integration_tests/test_external_devices/test_spike_run_forever_again.py b/spynnaker_integration_tests/test_external_devices/test_spike_run_forever_again.py index 5d88e93c10..5be166fc50 100644 --- a/spynnaker_integration_tests/test_external_devices/test_spike_run_forever_again.py +++ b/spynnaker_integration_tests/test_external_devices/test_spike_run_forever_again.py @@ -33,49 +33,57 @@ def sim_control(label, sender): def receive_spikes(label, time, neuron_ids): global spike_receive_count spike_receive_count += len(neuron_ids) - for neuron_id in neuron_ids: - print("Received spike at time", time, "from", label, "-", neuron_id) - - -def do_run(): - - conn = sim.external_devices.SpynnakerLiveSpikesConnection( - receive_labels=["pop_1"], send_labels=["sender"], local_port=None) - conn.add_receive_callback("pop_1", receive_spikes) - conn.add_start_resume_callback("sender", sim_control) - - # initial call to set up the front end (pynn requirement) - sim.setup(timestep=1.0, min_delay=1.0) - ssa = sim.Population( - 1, sim.external_devices.SpikeInjector( - database_notify_port_num=conn.local_port), - label="sender") - pop = sim.Population( - 1, sim.IF_curr_exp(), label="pop_1") - input_pop = sim.Population( - 1, sim.SpikeSourceArray( - spike_times=[0, 2, 4, 6, 8, 10, 12, 14, 16, 18]), - label="input") - input_pop.record("spikes") - sim.Projection(ssa, pop, sim.OneToOneConnector(), - sim.StaticSynapse(weight=5, delay=1)) - sim.external_devices.activate_live_output_for( - pop, database_notify_port_num=conn.local_port) - - for _ in range(5): - sim.external_devices.run_forever() - neo = pop.get_data("spikes") - spikes = count_spikes(neo) - - sim.end() - print(spike_send_count, spike_receive_count, spikes) + # for neuron_id in neuron_ids: + # print("Received spike at time", time, "from", label, "-", neuron_id) + class TestSpikeRunForeverAgain(BaseTestCase): + def do_run(self): + conn = sim.external_devices.SpynnakerLiveSpikesConnection( + receive_labels=["pop_1"], send_labels=["sender"], local_port=None) + conn.add_receive_callback("pop_1", receive_spikes) + conn.add_start_resume_callback("sender", sim_control) + + # initial call to set up the front end (pynn requirement) + sim.setup(timestep=1.0, min_delay=1.0) + ssa = sim.Population( + 1, sim.external_devices.SpikeInjector( + database_notify_port_num=conn.local_port), + label="sender") + pop = sim.Population( + 1, sim.IF_curr_exp(), label="pop_1") + pop.record("spikes") + spike_times = [0, 2, 4, 8, 16, 32, 64, 128, 256] + input_pop = sim.Population( + 1, sim.SpikeSourceArray( + spike_times=spike_times), + label="input") + input_pop.record("spikes") + sim.Projection(ssa, pop, sim.OneToOneConnector(), + sim.StaticSynapse(weight=5, delay=1)) + sim.external_devices.activate_live_output_for( + pop, database_notify_port_num=conn.local_port) + + n_loops = 5 + for _ in range(n_loops): + sim.external_devices.run_forever() + + neo = pop.get_data("spikes") + pop_spikes = count_spikes(neo) + + neo = input_pop.get_data("spikes") + input_spikes = count_spikes(neo) + + sim.end() + self.assertEqual(spike_send_count, spike_receive_count) + self.assertEqual(spike_send_count, pop_spikes) + print(input_spikes, len(spike_times) * n_loops) + def test_run(self): - self.runsafe(do_run) + self.runsafe(self.do_run) -if __name__ == "__main__": - do_run() +if __name__ == '__main__': + TestSpikeRunForeverAgain().do_run()