From 148a9eb25b53651f9d9c887a529898745d2bdc52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nadir=20Rom=C3=A1n=20Guerrero?= Date: Tue, 30 Nov 2021 14:06:33 +0100 Subject: [PATCH] Initial simulation config implementation (#158) --- include/bbp/sonata/config.h | 107 +++++++++++++++ python/bindings.cpp | 45 +++++++ python/generated/docstrings.h | 71 ++++++++++ python/libsonata/__init__.py | 1 + python/tests/test.py | 30 ++++- src/config.cpp | 145 +++++++++++++++++++-- tests/data/config/simulation_config.json | 63 +++++++++ tests/test_config.cpp | 157 +++++++++++++++++++++++ 8 files changed, 608 insertions(+), 11 deletions(-) create mode 100644 tests/data/config/simulation_config.json diff --git a/include/bbp/sonata/config.h b/include/bbp/sonata/config.h index bd8d7a1c..c6bae27d 100644 --- a/include/bbp/sonata/config.h +++ b/include/bbp/sonata/config.h @@ -175,5 +175,112 @@ class SONATA_API CircuitConfig std::unordered_map _edgePopulationProperties; }; +/** + * Read access to a SONATA simulation config file. + */ +class SONATA_API SimulationConfig +{ + public: + /** + * Parameters defining global simulation settings for spike reports + */ + struct Run { + /// Biological simulation end time in milliseconds + float tstop{}; + /// Integration step duration in milliseconds + float dt{}; + }; + /** + * Parameters to override simulator output for spike reports + */ + struct Output { + /// Spike report file output directory. Default is "output" + std::string outputDir; + /// Spike report file name. Default is "out.h5" + std::string spikesFile; + }; + /** + * List of report parameters collected during the simulation + */ + struct Report { + /// Node sets on which to report + std::string cells; + /// Report type. Possible values: "compartment", "summation", "synapse" + std::string type; + /// Interval between reporting steps in milliseconds + float dt{}; + /// Time to step reporting in milliseconds + float startTime{}; + /// Time to stop reporting in milliseconds + float endTime{}; + /// Report filename. Default is "_SONATA.h5" + std::string fileName; + }; + + /** + * Parses a SONATA JSON simulation configuration file. + * + * \throws SonataError on: + * - Ill-formed JSON + * - Missing mandatory entries (in any depth) + */ + SimulationConfig(const std::string& content, const std::string& basePath); + + /** + * Loads a SONATA JSON simulation config file from disk and returns a CircuitConfig object + * which parses it. + * + * \throws SonataError on: + * - Non accesible file (does not exists / does not have read access) + * - Ill-formed JSON + * - Missing mandatory entries (in any depth) + */ + static SimulationConfig fromFile(const std::string& path); + + /** + * Returns the base path of the simulation config file + */ + const std::string& getBasePath() const noexcept; + + /** + * Returns the JSON content of the simulation config file + */ + const std::string& getJSON() const noexcept; + + /** + * Returns the Run section of the simulation configuration. + */ + const Run& getRun() const noexcept; + + /** + * Returns the Output section of the simulation configuration. + */ + const Output& getOutput() const noexcept; + + /** + * Returns the given report parameters. + * + * \throws SonataError if the given report name does not correspond with any existing + * report. + */ + const Report& getReport(const std::string& name) const; + + private: + // JSON string + const std::string _jsonContent; + // Base path of the simulation config file + const std::string _basePath; + + // Run section + Run _run; + // Output section + Output _output; + // List of reports + std::unordered_map _reports; + + class Parser; + friend class Parser; +}; + } // namespace sonata } // namespace bbp diff --git a/python/bindings.cpp b/python/bindings.cpp index c30e603d..25b5d163 100644 --- a/python/bindings.cpp +++ b/python/bindings.cpp @@ -508,6 +508,51 @@ PYBIND11_MODULE(_libsonata, m) { .def("edge_population_properties", &CircuitConfig::getEdgePopulationProperties, "name"_a) .def_property_readonly("expanded_json", &CircuitConfig::getExpandedJSON); + py::class_(m, + "Run", + "Stores parameters defining global simulation settings") + .def_readonly("tstop", + &SimulationConfig::Run::tstop, + "Biological simulation end time in milliseconds") + .def_readonly("dt", + &SimulationConfig::Run::dt, + "Simulation integration step in milliseconds"); + + py::class_(m, + "Output", + "Stores overriden parameters of simulation output") + .def_readonly("output_dir", + &SimulationConfig::Output::outputDir, + "Simulation output directory") + .def_readonly("spikes_file", + &SimulationConfig::Output::spikesFile, + "Spike report filename"); + + py::class_(m, "Report", "List of parameters of a report") + .def_readonly("cells", &SimulationConfig::Report::cells, "Node sets on which to report") + .def_readonly("type", + &SimulationConfig::Report::type, + "Report type. Possible values are 'compartment', 'summation', 'synapse") + .def_readonly("dt", + &SimulationConfig::Report::dt, + "Interval between reporting steps in milliseconds") + .def_readonly("start_time", + &SimulationConfig::Report::startTime, + "Time to start reporting, in milliseconds") + .def_readonly("end_time", + &SimulationConfig::Report::endTime, + "Time to stop reporting in milliseconds") + .def_readonly("file_name", &SimulationConfig::Report::fileName, "Report file name"); + + py::class_(m, "SimulationConfig", "") + .def(py::init()) + .def_static("from_file", + [](py::object path) { return SimulationConfig::fromFile(py::str(path)); }) + .def_property_readonly("base_path", &SimulationConfig::getBasePath) + .def_property_readonly("json", &SimulationConfig::getJSON) + .def_property_readonly("run", &SimulationConfig::getRun) + .def_property_readonly("output", &SimulationConfig::getOutput) + .def("report", &SimulationConfig::getReport, "name"_a); bindPopulationClass( m, "EdgePopulation", "Collection of edges with attributes and connectivity index") diff --git a/python/generated/docstrings.h b/python/generated/docstrings.h index a721d850..b94ada20 100644 --- a/python/generated/docstrings.h +++ b/python/generated/docstrings.h @@ -487,6 +487,77 @@ static const char *__doc_bbp_sonata_Selection_ranges = R"doc(Get a list of range static const char *__doc_bbp_sonata_Selection_ranges_2 = R"doc()doc"; +static const char *__doc_bbp_sonata_SimulationConfig = R"doc(Read access to a SONATA simulation config file.)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_Output = R"doc(Parameters to override simulator output for spike reports)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_Output_outputDir = R"doc(Spike report file output directory. Default is "output")doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_Output_spikesFile = R"doc(Spike report file name. Default is "out.h5")doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_Parser = R"doc()doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_Report = R"doc(List of report parameters collected during the simulation)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_Report_cells = R"doc(Node sets on which to report)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_Report_dt = R"doc(Interval between reporting steps in milliseconds)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_Report_endTime = R"doc(Time to stop reporting in milliseconds)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_Report_fileName = R"doc(Report filename. Default is "_SONATA.h5")doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_Report_startTime = R"doc(Time to step reporting in milliseconds)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_Report_type = R"doc(Report type. Possible values: "compartment", "summation", "synapse")doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_Run = R"doc(Parameters defining global simulation settings for spike reports)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_Run_dt = R"doc(Integration step duration in milliseconds)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_Run_tstop = R"doc(Biological simulation end time in milliseconds)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_SimulationConfig = +R"doc(Parses a SONATA JSON simulation configuration file. + +Throws: + s SonataError on: - Ill-formed JSON - Missing mandatory entries + (in any depth))doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_basePath = R"doc()doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_fromFile = +R"doc(Loads a SONATA JSON simulation config file from disk and returns a +CircuitConfig object which parses it. + +Throws: + s SonataError on: - Non accesible file (does not exists / does not + have read access) - Ill-formed JSON - Missing mandatory entries + (in any depth))doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_getBasePath = R"doc(Returns the base path of the simulation config file)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_getJSON = R"doc(Returns the JSON content of the simulation config file)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_getOutput = R"doc(Returns the Output section of the simulation configuration.)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_getReport = +R"doc(Returns the given report parameters. + +Throws: + s SonataError if the given report name does not correspond with + any existing report.)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_getRun = R"doc(Returns the Run section of the simulation configuration.)doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_jsonContent = R"doc()doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_output = R"doc()doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_reports = R"doc()doc"; + +static const char *__doc_bbp_sonata_SimulationConfig_run = R"doc()doc"; + static const char *__doc_bbp_sonata_SonataError = R"doc()doc"; static const char *__doc_bbp_sonata_SonataError_SonataError = R"doc()doc"; diff --git a/python/libsonata/__init__.py b/python/libsonata/__init__.py index bece7f11..7740b24e 100644 --- a/python/libsonata/__init__.py +++ b/python/libsonata/__init__.py @@ -5,6 +5,7 @@ from libsonata._libsonata import ( CircuitConfig, + SimulationConfig, EdgePopulation, EdgeStorage, ElementDataFrame, diff --git a/python/tests/test.py b/python/tests/test.py index 10a40a47..4b9f480c 100644 --- a/python/tests/test.py +++ b/python/tests/test.py @@ -11,7 +11,7 @@ SomaReportReader, SomaReportPopulation, ElementReportReader, ElementReportPopulation, NodeSets, - CircuitConfig + CircuitConfig, SimulationConfig ) @@ -540,5 +540,33 @@ def test_get_population_properties(self): self.assertTrue(edge_prop.biophysical_neuron_models_dir.endswith('biophysical_neuron_models')) self.assertEqual(edge_prop.alternate_morphology_formats, {}) + +class TestSimulationConfig(unittest.TestCase): + def setUp(self): + self.config = SimulationConfig.from_file( + os.path.join(PATH, 'config/simulation_config.json')) + + def test_basic(self): + self.assertEqual(self.config.base_path, os.path.abspath(os.path.join(PATH, 'config'))) + + self.assertEqual(self.config.run.tstop, 1000) + self.assertTrue(abs(self.config.run.dt - 0.025) < 0.01) + + self.assertEqual(self.config.output.output_dir, + os.path.abspath(os.path.join(PATH, 'config/output'))) + self.assertEqual(self.config.output.spikes_file, 'out.h5') + + self.assertEqual(self.config.report('soma').cells, 'Mosaic') + self.assertEqual(self.config.report('soma').type, 'compartment') + self.assertTrue(abs(self.config.report('compartment').dt - 0.1) < 0.01) + self.assertEqual(self.config.report('axonal_comp_centers').start_time, 0) + self.assertEqual(self.config.report('axonal_comp_centers').file_name, + os.path.abspath(os.path.join(PATH, 'config/axon_centers.h5'))) + self.assertEqual(self.config.report('cell_imembrane').end_time, 500) + + def test_json(self): + temp_config = json.loads(self.config.json) + self.assertEqual(temp_config['run']['tstop'], 1000) + if __name__ == '__main__': unittest.main() diff --git a/src/config.cpp b/src/config.cpp index 0725d8ef..e950e013 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -124,6 +124,11 @@ Variables readVariables(const nlohmann::json& json) { return variables; } +std::string toAbsolute(const fs::path& base, const fs::path& path) { + const auto absolute = path.is_absolute() ? path : fs::absolute(base / path); + return absolute.lexically_normal().string(); +} + } // namespace class CircuitConfig::Parser @@ -158,18 +163,12 @@ class CircuitConfig::Parser const std::string& defaultValue = std::string()) const { auto value = getJSONValue(json, key); if (!value.empty()) { - return toAbsolute(value); + return toAbsolute(_basePath, value); } return defaultValue; } - std::string toAbsolute(const std::string& pathStr) const { - const fs::path path(pathStr); - const auto absolute = path.is_absolute() ? path : fs::absolute(_basePath / path); - return absolute.lexically_normal().string(); - } - const nlohmann::json& getSubNetworkJson(const std::string& prefix) const { // Fail if no network entry is defined if (_json.find("networks") == _json.end()) { @@ -190,7 +189,7 @@ class CircuitConfig::Parser std::string getNodeSetsPath() const { // Retrieve node sets file, if any if (_json.find("node_sets_file") != _json.end()) { - return toAbsolute(_json["node_sets_file"]); + return toAbsolute(_basePath, _json["node_sets_file"]); } return std::string(); @@ -208,7 +207,7 @@ class CircuitConfig::Parser const auto alternateMorphoDir = components.find("alternate_morphologies"); if (alternateMorphoDir != components.end()) { for (auto it = alternateMorphoDir->begin(); it != alternateMorphoDir->end(); ++it) { - result.alternateMorphologiesDir[it.key()] = toAbsolute(it.value()); + result.alternateMorphologiesDir[it.key()] = toAbsolute(_basePath, it.value()); } } @@ -283,7 +282,8 @@ class CircuitConfig::Parser const auto altMorphoDir = popData.find("alternate_morphologies"); if (altMorphoDir != popData.end()) { for (auto it = altMorphoDir->begin(); it != altMorphoDir->end(); ++it) { - popProperties.alternateMorphologyFormats[it.key()] = toAbsolute(it.value()); + popProperties.alternateMorphologyFormats[it.key()] = toAbsolute(_basePath, + it.value()); } } } @@ -478,5 +478,130 @@ const std::string& CircuitConfig::getExpandedJSON() const { } +class SimulationConfig::Parser +{ + public: + Parser(const std::string& content, const std::string& basePath) + : _basePath(fs::absolute(fs::path(basePath)).lexically_normal()) + , _json(nlohmann::json::parse(content)) {} + + template + void parseMandatory(const Iterator& it, + const char* name, + const SectionName& sn, + Type& buf) const { + const auto element = it.find(name); + if (element == it.end()) + throw SonataError(fmt::format("Could not find '{}' in '{}'", name, sn)); + buf = element->template get(); + } + + template + void parseOptional(const Iterator& it, const char* name, Type& buf) const { + const auto element = it.find(name); + if (element != it.end()) + buf = element->template get(); + } + + SimulationConfig::Run parseRun() const { + const auto runIt = _json.find("run"); + if (runIt == _json.end()) + throw SonataError("Could not find 'run' section"); + + SimulationConfig::Run result{}; + parseMandatory(*runIt, "tstop", "run", result.tstop); + parseMandatory(*runIt, "dt", "run", result.dt); + return result; + } + + SimulationConfig::Output parseOutput() const { + SimulationConfig::Output result{}; + result.outputDir = "output"; + result.spikesFile = "out.h5"; + + const auto outputIt = _json.find("output"); + if (outputIt != _json.end()) { + parseOptional(*outputIt, "output_dir", result.outputDir); + parseOptional(*outputIt, "spikes_file", result.spikesFile); + } + + result.outputDir = toAbsolute(_basePath, result.outputDir); + + return result; + } + + std::unordered_map parseReports() const { + std::unordered_map result; + + const auto reportsIt = _json.find("reports"); + if (reportsIt == _json.end()) + return result; + + for (auto it = reportsIt->begin(); it != reportsIt->end(); ++it) { + auto& report = result[it.key()]; + auto& valueIt = it.value(); + const auto debugStr = fmt::format("report {}", it.key()); + parseMandatory(valueIt, "cells", debugStr, report.cells); + parseMandatory(valueIt, "type", debugStr, report.type); + parseMandatory(valueIt, "dt", debugStr, report.dt); + parseMandatory(valueIt, "start_time", debugStr, report.startTime); + parseMandatory(valueIt, "end_time", debugStr, report.endTime); + parseOptional(valueIt, "file_name", report.fileName); + if (report.fileName.empty()) + report.fileName = it.key() + "_SONATA.h5"; + else { + const auto extension = fs::path(report.fileName).extension().string(); + if (extension.empty() || extension != ".h5") + report.fileName += ".h5"; + } + report.fileName = toAbsolute(_basePath, report.fileName); + } + + return result; + } + + private: + const fs::path _basePath; + const nlohmann::json _json; +}; + +SimulationConfig::SimulationConfig(const std::string& content, const std::string& basePath) + : _jsonContent(content) + , _basePath(fs::absolute(basePath).lexically_normal().string()) { + const Parser parser(content, basePath); + _run = parser.parseRun(); + _output = parser.parseOutput(); + _reports = parser.parseReports(); +} + +SimulationConfig SimulationConfig::fromFile(const std::string& path) { + return SimulationConfig(readFile(path), fs::path(path).parent_path()); +} + +const std::string& SimulationConfig::getBasePath() const noexcept { + return _basePath; +} + +const std::string& SimulationConfig::getJSON() const noexcept { + return _jsonContent; +} + +const SimulationConfig::Run& SimulationConfig::getRun() const noexcept { + return _run; +} + +const SimulationConfig::Output& SimulationConfig::getOutput() const noexcept { + return _output; +} + +const SimulationConfig::Report& SimulationConfig::getReport(const std::string& name) const { + const auto it = _reports.find(name); + if (it == _reports.end()) + throw SonataError( + fmt::format("The report '{}' is not present in the simulation config file", name)); + + return it->second; +} + } // namespace sonata } // namespace bbp diff --git a/tests/data/config/simulation_config.json b/tests/data/config/simulation_config.json new file mode 100644 index 00000000..e32efc5e --- /dev/null +++ b/tests/data/config/simulation_config.json @@ -0,0 +1,63 @@ +{ + "run": { + "tstop": 1000, + "dt": 0.025, + "random_seed": 201506, + "spike_location": "ais", + "integration_method" : 2, + "forward_skip": 500 + }, + "output": { + "output_dir": "output", + "spikes_file": "out.h5" + }, + "reports": { + "soma": { + "cells": "Mosaic", + "sections": "soma", + "type": "compartment", + "variable_name": "v", + "unit": "mV", + "dt": 0.1, + "start_time" : 0, + "end_time" : 500, + "file_name": "soma", + "enabled" : true + }, + "compartment": { + "cells": "Mosaic", + "sections": "all", + "type": "compartment", + "variable_name": "v", + "unit": "mV", + "dt": 0.1, + "start_time" : 0, + "end_time" : 500, + "file_name": "voltage", + "enabled" : true + }, + "axonal_comp_centers": { + "cells": "Mosaic", + "sections": "axon", + "type": "compartment", + "variable_name": "v", + "unit": "mV", + "compartments": "center", + "dt": 0.1, + "start_time" : 0, + "end_time" : 500, + "file_name": "axon_centers", + "enabled" : true + }, + "cell_imembrane": { + "cells": "Column", + "sections": "soma", + "type": "summation", + "unit": "nA", + "dt": 0.05, + "start_time": 0, + "end_time": 500, + "enabled": true + } + } +} diff --git a/tests/test_config.cpp b/tests/test_config.cpp index f0b3a3b0..b032066c 100644 --- a/tests/test_config.cpp +++ b/tests/test_config.cpp @@ -2,6 +2,8 @@ #include +#include "../extlib/filesystem.hpp" + #include #include @@ -288,3 +290,158 @@ TEST_CASE("CircuitConfig") { } } } + +TEST_CASE("SimulationConfig") { + SECTION("Simple") { + const auto config = SimulationConfig::fromFile("./data/config/simulation_config.json"); + CHECK_NOTHROW(config.getRun()); + using Catch::Matchers::WithinULP; + REQUIRE_THAT(config.getRun().tstop, WithinULP(1000.f, 1)); + REQUIRE_THAT(config.getRun().dt, WithinULP(0.025f, 1)); + + namespace fs = ghc::filesystem; + const auto basePath = fs::absolute( + fs::path("./data/config/simulation_config.json").parent_path()); + + CHECK_NOTHROW(config.getOutput()); + const auto outputPath = fs::absolute(basePath / fs::path("output")); + CHECK(config.getOutput().outputDir == outputPath.lexically_normal()); + CHECK(config.getOutput().spikesFile == "out.h5"); + + CHECK_THROWS_AS(config.getReport("DoesNotExist"), SonataError); + + CHECK(config.getReport("soma").cells == "Mosaic"); + CHECK(config.getReport("soma").type == "compartment"); + CHECK(config.getReport("compartment").dt == 0.1f); + CHECK(config.getReport("axonal_comp_centers").startTime == 0.f); + const auto axonalFilePath = fs::absolute(basePath / fs::path("axon_centers.h5")); + CHECK(config.getReport("axonal_comp_centers").fileName == + axonalFilePath.lexically_normal()); + CHECK(config.getReport("cell_imembrane").endTime == 500.f); + + CHECK_NOTHROW(nlohmann::json::parse(config.getJSON())); + CHECK(config.getBasePath() == basePath.lexically_normal()); + } + + SECTION("Exception") { + { // No run section + auto contents = R"({})"; + CHECK_THROWS_AS(SimulationConfig(contents, "./"), SonataError); + } + { // No tstop in run section + auto contents = R"({ + "run": { + "dt": 0.05 + } + })"; + CHECK_THROWS_AS(SimulationConfig(contents, "./"), SonataError); + } + { // No dt in run section + auto contents = R"({ + "run": { + "tstop": 1000 + } + })"; + CHECK_THROWS_AS(SimulationConfig(contents, "./"), SonataError); + } + { // No reports section + auto contents = R"({ + "run": { + "dt": 0.05, + "tstop": 1000 + } + })"; + CHECK_NOTHROW(SimulationConfig(contents, "./")); + } + { // No cells in a report object + auto contents = R"({ + "run": { + "dt": 0.05, + "tstop": 1000 + }, + "reports": { + "test": { + "type": "typestring", + "variable_name": "variablestring", + "dt": 0.05, + "start_time": 0, + "end_time": 500 + } + } + })"; + CHECK_THROWS_AS(SimulationConfig(contents, "./"), SonataError); + } + { // No type in a report object + auto contents = R"({ + "run": { + "dt": 0.05, + "tstop": 1000 + }, + "reports": { + "test": { + "cells": "nodesetstring", + "variable_name": "variablestring", + "dt": 0.05, + "start_time": 0, + "end_time": 500 + } + } + })"; + CHECK_THROWS_AS(SimulationConfig(contents, "./"), SonataError); + } + { // No dt in a report object + auto contents = R"({ + "run": { + "dt": 0.05, + "tstop": 1000 + }, + "reports": { + "test": { + "cells": "nodesetstring", + "type": "typestring", + "variable_name": "variablestring", + "start_time": 0, + "end_time": 500 + } + } + })"; + CHECK_THROWS_AS(SimulationConfig(contents, "./"), SonataError); + } + { // No start_time in a report object + auto contents = R"({ + "run": { + "dt": 0.05, + "tstop": 1000 + }, + "reports": { + "test": { + "cells": "nodesetstring", + "type": "typestring", + "variable_name": "variablestring", + "dt": 0.05, + "end_time": 500 + } + } + })"; + CHECK_THROWS_AS(SimulationConfig(contents, "./"), SonataError); + } + { // No end_time in a report object + auto contents = R"({ + "run": { + "dt": 0.05, + "tstop": 1000 + }, + "reports": { + "test": { + "cells": "nodesetstring", + "type": "typestring", + "variable_name": "variablestring", + "dt": 0.05, + "start_time": 0 + } + } + })"; + CHECK_THROWS_AS(SimulationConfig(contents, "./"), SonataError); + } + } +}