Merge branch 'master' into fix_one_to_one_performance

spynnaker/pyNN/models/recorder.py

@@ -271,18 +271,30 @@ def csv_neo_block(
                     wrote_metadata = db.csv_block_metadata(
                         csv_file, pop_label, annotations)
                 if wrote_metadata:
-                    db.csv_segment(
-                        csv_file, pop_label, variables, view_indexes)
+                    db.csv_segment(csv_file, pop_label, variables,
+                                   view_indexes, allow_missing=True)
 
-        with NeoBufferDatabase() as db:
-            if SpynnakerDataView.is_reset_last():
+        if SpynnakerDataView.is_reset_last():
+            if wrote_metadata:
                 logger.warning(
                     "Due to the call directly after reset, "
                     "the data will only contain {} segments",
                     SpynnakerDataView.get_segment_counter() - 1)
+                return
             else:
-                db.csv_segment(
-                    csv_file, pop_label, variables, view_indexes)
+                raise ConfigurationException(
+                    f"Unable to write data for {pop_label}")
+
+        with NeoBufferDatabase() as db:
+            if not wrote_metadata:
+                wrote_metadata = db.csv_block_metadata(
+                    csv_file, pop_label, annotations)
+            if wrote_metadata:
+                db.csv_segment(csv_file, pop_label, variables,
+                               view_indexes, allow_missing=False)
+            else:
+                raise ConfigurationException(
+                    f"Unable to write data for {pop_label}")
 
     def __append_current_segment(
             self, block: neo.Block, variables: Names,
@@ -323,7 +335,8 @@ def __append_previous_segment(
             ~spinn_front_end_common.utilities.exceptions.ConfigurationException:
             If the recording not setup correctly
         """
-        with NeoBufferDatabase.segement_db(segment_number) as db:
+        with NeoBufferDatabase.segement_db(
+                segment_number, read_only=not clear) as db:
             if block is None:
                 block = db.get_empty_block(
                     self.__population.label, annotations)

spynnaker/pyNN/utilities/neo_buffer_database.py

@@ -128,12 +128,13 @@ def __init__(self, database_file: Optional[str] = None,
             segment_cache[segment] = database_file
 
     @classmethod
-    def segement_db(cls, segment_number: int) -> NeoBufferDatabase:
+    def segement_db(cls, segment_number: int,
+                    read_only: Optional[bool] = None) -> NeoBufferDatabase:
         """
         Retrieves a NeoBufferDatabase for this segment.
         """
         database_file = segment_cache[segment_number]
-        return NeoBufferDatabase(database_file)
+        return NeoBufferDatabase(database_file, read_only)
 
     def write_segment_metadata(self) -> None:
         """
@@ -1078,7 +1079,8 @@ def get_spike_counts(
         self.__get_segment_info()
         metadata = self.__get_recording_metadata(pop_label, SPIKES)
         if metadata is None:
-            return {}
+            raise ConfigurationException(
+                f"{pop_label} did not record spikes")
 
         (rec_id, _, buffered_type, _, _, pop_size, _,
          n_colour_bits) = metadata
@@ -1156,7 +1158,7 @@ def __add_data(
 
     def __read_and_csv_data(
             self, pop_label: str, variable: str, csv_writer: CSVWriter,
-            view_indexes: ViewIndices, t_stop: float):
+            view_indexes: ViewIndices, t_stop: float, allow_missing: bool):
         """
         Reads the data for one variable and adds it to the CSV file.
 
@@ -1172,10 +1174,16 @@ def __read_and_csv_data(
         :param view_indexes:
         :type view_indexes: None, ~numpy.array or list(int)
         :param float t_stop:
+        :param allow_missing: Flag to say if data for missing variable
+            should raise an exception
         """
         metadata = self.__get_recording_metadata(pop_label, variable)
         if metadata is None:
-            return
+            if allow_missing:
+                return
+            else:
+                raise ConfigurationException(
+                    f"No data for {pop_label=} {variable=}")
 
         (rec_id, data_type, buffer_type, t_start, sampling_interval_ms,
          pop_size, units, n_colour_bits) = metadata
@@ -1268,7 +1276,7 @@ def get_full_block(
 
     def csv_segment(
             self, csv_file: str, pop_label: str, variables: Names,
-            view_indexes: ViewIndices = None):
+            view_indexes: ViewIndices, allow_missing: bool):
         """
         Writes the data including metadata to a CSV file.
 
@@ -1290,7 +1298,7 @@ def csv_segment(
             If the recording metadata not setup correctly
         """
         if not os.path.isfile(csv_file):
-            raise SpynnakerException("PLease call csv_block_metadata first")
+            raise SpynnakerException("Please call csv_block_metadata first")
         with open(csv_file, 'a', newline='', encoding="utf-8") as csvfile:
             csv_writer = csv.writer(csvfile, delimiter=',', quotechar='"',
                                     quoting=csv.QUOTE_MINIMAL)
@@ -1301,8 +1309,8 @@ def csv_segment(
                 csv_writer, segment_number, rec_datetime)
 
             for variable in self.__clean_variables(variables, pop_label):
-                self.__read_and_csv_data(
-                    pop_label, variable, csv_writer, view_indexes, t_stop)
+                self.__read_and_csv_data(pop_label, variable, csv_writer,
+                                         view_indexes, t_stop, allow_missing)
 
     def csv_block_metadata(
             self, csv_file: str, pop_label: str,

spynnaker_integration_tests/test_various/test_recording_later_additions.py

@@ -12,14 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+import os
+from spinn_front_end_common.utilities.exceptions import ConfigurationException
 import pyNN.spiNNaker as sim
 from spinnaker_testbase import BaseTestCase
 
 
 class TestRecordingLaterAdditions(BaseTestCase):
 
-    def do_simple(self):
+    def do_later_additions(self):
         sim.setup(timestep=1.0)
         sim.set_number_of_neurons_per_core(sim.IF_curr_exp, 100)
 
@@ -31,19 +32,58 @@ def do_simple(self):
                        synapse_type=sim.StaticSynapse(weight=5, delay=1))
         pop_a.record(["spikes", "v"])
 
+        pop_b = sim.Population(1, sim.IF_curr_exp(), label="pop_b")
+        sim.Projection(input_pop, pop_b, sim.OneToOneConnector(),
+                       synapse_type=sim.StaticSynapse(weight=5, delay=1))
+
         pop_c = sim.Population(1, sim.IF_curr_exp(), label="pop_c")
         sim.Projection(input_pop, pop_c, sim.OneToOneConnector(),
                        synapse_type=sim.StaticSynapse(weight=5, delay=1))
         pop_c.record(["spikes"])
+
         sim.run(10)
+
+        # Did not record all these should fail
+        with self.assertRaises(ConfigurationException):
+            pop_b.get_data(variables=["spikes", "v"])
+        with self.assertRaises(ConfigurationException):
+            pop_b.get_spike_counts()
+        with self.assertRaises(ConfigurationException):
+            pop_b.write_data("test_b_bad.csv", ["spikes", "v"])
+        with self.assertRaises(ConfigurationException):
+            pop_c.get_data(variables=["spikes", "v"])
+        with self.assertRaises(ConfigurationException):
+            pop_c.write_data("test_c_bad.csv", ["spikes", "v"])
+
         sim.reset()
 
-        pop_b = sim.Population(1, sim.IF_curr_exp(), label="pop_b")
-        sim.Projection(input_pop, pop_b, sim.OneToOneConnector(),
-                       synapse_type=sim.StaticSynapse(weight=5, delay=1))
-        pop_b.record(["spikes", "v"])
+        # No recording from previous segment so should fail
+        with self.assertRaises(ConfigurationException):
+            pop_b.get_data(variables=["spikes", "v"])
+        with self.assertRaises(ConfigurationException):
+            pop_b.write_data("test_b_bad.csv", ["spikes", "v"])
+        # Only includes this segment so should fail
+        with self.assertRaises(ConfigurationException):
+            pop_b.get_spike_counts()
+        # Some recording from previous will wok
+        pop_c.get_data(variables=["spikes", "v"])
+        pop_c.write_data("test_c_1.csv", ["spikes", "v"])
 
+        pop_b.record(["spikes", "v"])
         pop_c.record(["v"])
+
+        # No recording from previous segment so should fail
+        with self.assertRaises(ConfigurationException):
+            pop_b.get_data(variables=["spikes", "v"])
+        with self.assertRaises(ConfigurationException):
+            pop_b.write_data("test_b_bad.csv", ["spikes", "v"])
+        # Only includes this segment so should fail
+        with self.assertRaises(ConfigurationException):
+            pop_b.get_spike_counts()
+        # Some recording from previous will wok
+        pop_c.get_data(variables=["spikes", "v"])
+        pop_c.write_data("test_c_1.csv", ["spikes", "v"])
+
         sim.run(20)
 
         neo_a = pop_a.get_data(variables=["spikes", "v"])
@@ -116,5 +156,18 @@ def do_simple(self):
         pop_c.write_data("test_c.csv", ["spikes", "v"])
         sim.end()
 
-    def test_simple(self):
-        self.runsafe(self.do_simple)
+    def cleanup(self, file):
+        try:
+            if os.path.exists(file):
+                os.remove(file)
+        except Exception:
+            pass
+
+    def test_later_additions(self):
+        self.cleanup("test_a.csv")
+        self.cleanup("test_b.csv")
+        self.cleanup("test_b_bad.csv")
+        self.cleanup("test_c.csv")
+        self.cleanup("test_c_1.csv")
+        self.cleanup("test_c_bad.csv")
+        self.runsafe(self.do_later_additions)

unittests/test_pop_views_assembly/test_csv.py

@@ -62,7 +62,8 @@ def test_write(self):
         with NeoBufferDatabase(my_buffer) as db:
             db.csv_block_metadata(
                 my_csv, "pop_1", annotations={"foo": 12, "bar": 34})
-            db.csv_segment(my_csv, "pop_1", variables="all")
+            db.csv_segment(my_csv, "pop_1", variables="all",
+                           view_indexes=None, allow_missing=False)
 
         neo = NeoCsv().read_csv(my_csv)
         # All annotations converted to String and not back
@@ -88,7 +89,7 @@ def test_view(self):
             db.csv_block_metadata(my_csv, "pop_1", annotations=None)
             db.csv_segment(
                 my_csv, "pop_1", variables=["spikes", "v"],
-                view_indexes=[2, 4, 7, 8])
+                view_indexes=[2, 4, 7, 8], allow_missing=False)
 
         neo = NeoCsv().read_csv(my_csv)
 
@@ -109,7 +110,8 @@ def test_over_view(self):
         my_csv = os.path.join(my_dir, "test_over_view.csv")
         with NeoBufferDatabase(my_buffer) as db:
             db.csv_block_metadata(my_csv, "pop_1", annotations=None)
-            db.csv_segment(my_csv, "pop_1", variables="all")
+            db.csv_segment(my_csv, "pop_1", variables="all",
+                           view_indexes=None, allow_missing=False)
 
         neo = NeoCsv().read_csv(my_csv)
         spikes = neo_convertor.convert_spikes(neo)
@@ -129,8 +131,8 @@ def test_over_sub_view(self):
         my_csv = os.path.join(my_dir, "test_over_sub_view.csv")
         with NeoBufferDatabase(my_buffer) as db:
             db.csv_block_metadata(my_csv, "pop_1", annotations=None)
-            db.csv_segment(
-                my_csv, "pop_1", variables="all", view_indexes=[2, 4])
+            db.csv_segment(my_csv, "pop_1", variables="all",
+                           view_indexes=[2, 4], allow_missing=False)
 
         neo = NeoCsv().read_csv(my_csv)
         spikes = neo_convertor.convert_spikes(neo)
@@ -150,8 +152,8 @@ def test_no_intersection(self):
         my_csv = os.path.join(my_dir, "test_no_intersection.csv")
         with NeoBufferDatabase(my_buffer) as db:
             db.csv_block_metadata(my_csv, "pop_1", annotations=None)
-            db.csv_segment(
-                my_csv, "pop_1", variables="all", view_indexes=[4, 6])
+            db.csv_segment(my_csv, "pop_1", variables="all",
+                           view_indexes=[4, 6], allow_missing=False)
 
         neo = NeoCsv().read_csv(my_csv)
         spikes = neo_convertor.convert_spikes(neo)
@@ -170,7 +172,8 @@ def test_rewiring(self):
         my_csv = os.path.join(my_dir, "test_rewiring.csv")
         with NeoBufferDatabase(my_buffer) as db:
             db.csv_block_metadata(my_csv, "pop_1", annotations=None)
-            db.csv_segment(my_csv, "pop_1", variables="all")
+            db.csv_segment(my_csv, "pop_1", variables="all",
+                           view_indexes=None, allow_missing=False)
         neo = NeoCsv().read_csv(my_csv)
         formation_events = neo.segments[0].events[0]
         elimination_events = neo.segments[0].events[1]