From 794555a2c40e7c264f4d261920c32a0d419ee119 Mon Sep 17 00:00:00 2001 From: MikeG Date: Fri, 7 Oct 2022 17:54:44 +0200 Subject: [PATCH] make h5 files available through PopulationProperties (#230) * circuit_config.json must now conform to the BBP standard for populations to be found * the circuit_config.json now is the ground truth for available properties: if multiple nodes files contain the same population name, only the one referenced in the circuit_config.json is considered --- include/bbp/sonata/config.h | 14 ++ python/bindings.cpp | 8 +- python/generated/docstrings.h | 14 +- python/tests/test.py | 6 + src/config.cpp | 180 +++++++++++--------------- tests/data/config/circuit_config.json | 12 +- tests/test_config.cpp | 5 + 7 files changed, 124 insertions(+), 115 deletions(-) diff --git a/include/bbp/sonata/config.h b/include/bbp/sonata/config.h index 2e746d6a..e5fc2241 100644 --- a/include/bbp/sonata/config.h +++ b/include/bbp/sonata/config.h @@ -54,6 +54,20 @@ struct SONATA_API PopulationProperties { * Dictionary for alternate directory paths. */ std::unordered_map alternateMorphologyFormats; + + /** + * Path to underlying elements H5 file. + * It is discouraged to directly access the contents of the file. + * Instead use 'libsonata' to read this file. + */ + std::string elementsPath; + + /** + * Path to underlying types csv file. + * It is discouraged to directly access the contents of the file. + * Instead use 'libsonata' to read this file. + */ + std::string typesPath; }; /** diff --git a/python/bindings.cpp b/python/bindings.cpp index 2ad88149..c9c2dce9 100644 --- a/python/bindings.cpp +++ b/python/bindings.cpp @@ -509,7 +509,13 @@ PYBIND11_MODULE(_libsonata, m) { DOC_POPULATION_PROPERTIES(morphologiesDir)) .def_readonly("alternate_morphology_formats", &PopulationProperties::alternateMorphologyFormats, - DOC_POPULATION_PROPERTIES(alternateMorphologyFormats)); + DOC_POPULATION_PROPERTIES(alternateMorphologyFormats)) + .def_readonly("elements_path", + &PopulationProperties::elementsPath, + DOC_POPULATION_PROPERTIES(elementsPath)) + .def_readonly("types_path", + &PopulationProperties::typesPath, + DOC_POPULATION_PROPERTIES(typesPath)); py::class_(m, "CircuitConfig", "") .def(py::init()) diff --git a/python/generated/docstrings.h b/python/generated/docstrings.h index 0aeeae93..9061cb9d 100644 --- a/python/generated/docstrings.h +++ b/python/generated/docstrings.h @@ -230,10 +230,20 @@ static const char *__doc_bbp_sonata_PopulationProperties_alternateMorphologyForm static const char *__doc_bbp_sonata_PopulationProperties_biophysicalNeuronModelsDir = R"doc(Path to the template HOC files defining the E-Mode)doc"; +static const char *__doc_bbp_sonata_PopulationProperties_elementsPath = +R"doc(Path to underlying elements H5 file. It is discouraged to directly +access the contents of the file. Instead use 'libsonata' to read this +file.)doc"; + static const char *__doc_bbp_sonata_PopulationProperties_morphologiesDir = R"doc(Path to the directory containing the morphologies)doc"; static const char *__doc_bbp_sonata_PopulationProperties_type = R"doc(Population type)doc"; +static const char *__doc_bbp_sonata_PopulationProperties_typesPath = +R"doc(Path to underlying types csv file. It is discouraged to directly +access the contents of the file. Instead use 'libsonata' to read this +file.)doc"; + static const char *__doc_bbp_sonata_PopulationStorage = R"doc(Collection of {PopulationClass}s stored in a H5 file and optional CSV.)doc"; static const char *__doc_bbp_sonata_PopulationStorage_Impl = R"doc()doc"; @@ -558,10 +568,6 @@ R"doc(Properties to assign values to variables in synapse MOD files. The format is a dictionary with keys being the SUFFIX names and values being dictionaries of variables' names and values.)doc"; -static const char *__doc_bbp_sonata_SimulationConfig_Conditions_minisSingleVesicle = -R"doc(Limit spontaneous release to single vesicle when true. Default is -false)doc"; - static const char *__doc_bbp_sonata_SimulationConfig_Conditions_modifications = R"doc(Collection of dictionaries with each member decribing a modification that mimics experimental manipulations to the circuit.)doc"; diff --git a/python/tests/test.py b/python/tests/test.py index 872c282d..885a3d0f 100644 --- a/python/tests/test.py +++ b/python/tests/test.py @@ -571,12 +571,18 @@ def test_get_population_properties(self): self.assertTrue(node_prop.biophysical_neuron_models_dir.endswith('biophysical_neuron_models')) self.assertEqual(node_prop.alternate_morphology_formats, {}) + self.assertEqual(node_prop.types_path, '') + self.assertTrue(node_prop.elements_path.endswith('tests/data/nodes1.h5')) + edge_prop = self.config.edge_population_properties('edges-AC') self.assertEqual(edge_prop.type, 'chemical_synapse') self.assertTrue(edge_prop.morphologies_dir.endswith('morphologies')) self.assertTrue(edge_prop.biophysical_neuron_models_dir.endswith('biophysical_neuron_models')) self.assertEqual(edge_prop.alternate_morphology_formats, {}) + self.assertEqual(edge_prop.types_path, '') + self.assertTrue(edge_prop.elements_path.endswith('tests/data/edges1.h5')) + def test_biophysical_properties_raises(self): contents = { diff --git a/src/config.cpp b/src/config.cpp index a754bff1..e707e4e0 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -134,6 +134,47 @@ namespace { // to be replaced by std::filesystem once C++17 is used namespace fs = ghc::filesystem; +void checkBiophysicalPopulations( + const std::unordered_map& populations) { + for (const auto& it : populations) { + const auto& population = it.first; + const auto& properties = it.second; + if (properties.type == "biophysical") { + if (properties.morphologiesDir.empty() && + properties.alternateMorphologyFormats.empty()) { + throw SonataError( + fmt::format("Node population '{}' is defined as 'biophysical' " + "but does not define 'morphologies_dir' or " + "'alternateMorphologyFormats'", + population)); + } else if (properties.biophysicalNeuronModelsDir.empty()) { + throw SonataError( + fmt::format("Node population '{}' is defined as 'biophysical' " + "but does not define 'biophysical_neuron_models_dir'", + population)); + } + } + } +} + +PopulationProperties getPopulationProperties( + const std::string& populationName, + const std::unordered_map& populations) { + auto it = populations.find(populationName); + if (it == populations.end()) { + throw SonataError(fmt::format("Could not find population '{}'", populationName)); + } + + return it->second; +} + +template +PopulationType getPopulation(const std::string& populationName, + const std::unordered_map& src) { + const auto properties = getPopulationProperties(populationName, src); + return PopulationType(properties.elementsPath, properties.typesPath, populationName); +} + std::map replaceVariables(std::map variables) { constexpr size_t maxIterations = 10; @@ -589,6 +630,23 @@ class CircuitConfig::Parser return result; } + template + std::tuple parseSubNetworksMG(const std::string& prefix, + const JSON& value) const { + const std::string elementsFile = prefix + "s_file"; + auto h5File = getJSONPath(value, elementsFile); + + if (h5File.empty()) { + throw SonataError( + fmt::format("'{}' network do not define '{}' entry", prefix, elementsFile)); + } + + const std::string typesFile = prefix + "_types_file"; + auto csvFile = getJSONPath(value, typesFile); + + return {h5File, csvFile}; + } + template Subnetworks parseSubNetworks(const std::string& prefix) const { using PopulationStorage = bbp::sonata::PopulationStorage; @@ -631,6 +689,10 @@ class CircuitConfig::Parser // Iterate over all defined subnetworks for (const auto& node : network) { + std::string elementsPath; + std::string typesPath; + std::tie(elementsPath, typesPath) = parseSubNetworksMG(prefix, node); + const auto populationsIt = node.find("populations"); if (populationsIt == node.end()) { continue; @@ -642,6 +704,9 @@ class CircuitConfig::Parser PopulationProperties& popProperties = output[it.key()]; + popProperties.elementsPath = elementsPath; + popProperties.typesPath = typesPath; + popProperties.type = getJSONValue(popData, "type"); popProperties.morphologiesDir = getJSONPath(popData, "morphologies_dir"); popProperties.biophysicalNeuronModelsDir = @@ -678,73 +743,6 @@ class CircuitConfig::Parser nlohmann::json _json; }; -class CircuitConfig::PopulationResolver -{ - public: - static std::set listPopulations(const std::vector& src) { - std::set result; - for (const auto& subNetwork : src) { - result.insert(subNetwork.populations.begin(), subNetwork.populations.end()); - } - return result; - } - - template - static PopulationType getPopulation(const std::string& populationName, - const std::vector& src) { - for (const auto& subNetwork : src) { - for (const auto& population : subNetwork.populations) { - if (population == populationName) { - return PopulationType(subNetwork.elements, subNetwork.types, populationName); - } - } - } - - throw SonataError(fmt::format("Could not find population '{}'", populationName)); - } - - static void checkDuplicatePopulations(const std::vector& src) { - std::set check; - for (const auto& subNetwork : src) { - for (const auto& population : subNetwork.populations) { - if (check.find(population) != check.end()) { - throw SonataError(fmt::format("Duplicate population name '{}'", population)); - } - check.insert(population); - } - } - } - - static void checkBiophysicalPopulations( - const std::vector& src, - const std::unordered_map& populations) { - for (const auto& subNetwork : src) { - for (const auto& population : subNetwork.populations) { - const auto it = populations.find(population); - if (it == populations.end()) { - continue; - } - - if (it->second.type == "biophysical") { - if (it->second.morphologiesDir.empty() && - it->second.alternateMorphologyFormats.empty()) { - throw SonataError( - fmt::format("Node population '{}' is defined as 'biophysical' " - "but does not define 'morphologies_dir' or " - "'alternateMorphologyFormats'", - population)); - } else if (it->second.biophysicalNeuronModelsDir.empty()) { - throw SonataError( - fmt::format("Node population '{}' is defined as 'biophysical' " - "but does not define 'biophysical_neuron_models_dir'", - population)); - } - } - } - } - } -}; - CircuitConfig::CircuitConfig(const std::string& contents, const std::string& basePath) { Parser parser(contents, basePath); @@ -754,17 +752,9 @@ CircuitConfig::CircuitConfig(const std::string& contents, const std::string& bas _nodeSetsFile = parser.getNodeSetsPath(); - // Load node subnetwork and check for duplicate populations - _networkNodes = parser.parseNodeNetwork(); - PopulationResolver::checkDuplicatePopulations(_networkNodes); - // Load node population overrides and check biophysical types _nodePopulationProperties = parser.parseNodePopulations(); - // Load edge subnetowrk and check for duplicate populations - _networkEdges = parser.parseEdgeNetwork(); - PopulationResolver::checkDuplicatePopulations(_networkEdges); - // Load edge population overrides _edgePopulationProperties = parser.parseEdgePopulations(); @@ -790,7 +780,7 @@ CircuitConfig::CircuitConfig(const std::string& contents, const std::string& bas updateDefaultProperties(_nodePopulationProperties, "biophysical"); updateDefaultProperties(_edgePopulationProperties, "chemical_synapse"); - PopulationResolver::checkBiophysicalPopulations(_networkNodes, _nodePopulationProperties); + checkBiophysicalPopulations(_nodePopulationProperties); } CircuitConfig CircuitConfig::fromFile(const std::string& path) { @@ -802,53 +792,27 @@ const std::string& CircuitConfig::getNodeSetsPath() const { } std::set CircuitConfig::listNodePopulations() const { - return PopulationResolver::listPopulations(_networkNodes); + return getMapKeys(_nodePopulationProperties); } NodePopulation CircuitConfig::getNodePopulation(const std::string& name) const { - return PopulationResolver::getPopulation(name, _networkNodes); + return getPopulation(name, _nodePopulationProperties); } std::set CircuitConfig::listEdgePopulations() const { - return PopulationResolver::listPopulations(_networkEdges); + return getMapKeys(_edgePopulationProperties); } EdgePopulation CircuitConfig::getEdgePopulation(const std::string& name) const { - return PopulationResolver::getPopulation(name, _networkEdges); + return getPopulation(name, _edgePopulationProperties); } PopulationProperties CircuitConfig::getNodePopulationProperties(const std::string& name) const { - auto populations = listNodePopulations(); - if (populations.find(name) == populations.end()) { - throw SonataError(fmt::format("Could not find node population '{}'", name)); - } - - auto popPropertiesIt = _nodePopulationProperties.find(name); - if (popPropertiesIt != _nodePopulationProperties.end()) { - return popPropertiesIt->second; - } - - return {"biophysical", - _components.biophysicalNeuronModelsDir, - _components.morphologiesDir, - _components.alternateMorphologiesDir}; + return getPopulationProperties(name, _nodePopulationProperties); } PopulationProperties CircuitConfig::getEdgePopulationProperties(const std::string& name) const { - auto populations = listEdgePopulations(); - if (populations.find(name) == populations.end()) { - throw SonataError(fmt::format("Could not find edge population '{}'", name)); - } - - auto popPropertiesIt = _edgePopulationProperties.find(name); - if (popPropertiesIt != _edgePopulationProperties.end()) { - return popPropertiesIt->second; - } - - return {"chemical_synapse", - _components.biophysicalNeuronModelsDir, - _components.morphologiesDir, - _components.alternateMorphologiesDir}; + return getPopulationProperties(name, _edgePopulationProperties); } const std::string& CircuitConfig::getExpandedJSON() const { diff --git a/tests/data/config/circuit_config.json b/tests/data/config/circuit_config.json index 9cc3902d..61f883cf 100644 --- a/tests/data/config/circuit_config.json +++ b/tests/data/config/circuit_config.json @@ -14,13 +14,21 @@ "nodes": [ { "nodes_file": "$NETWORK_DIR/nodes1.h5", - "node_types_file": null + "node_types_file": null, + "populations": { + "nodes-A": {}, + "nodes-B": {} + } } ], "edges": [ { "edges_file": "$NETWORK_DIR/edges1.h5", - "edge_types_file": null + "edge_types_file": null, + "populations": { + "edges-AB": {}, + "edges-AC": {} + } } ] } diff --git a/tests/test_config.cpp b/tests/test_config.cpp index 80e4f6eb..c001367f 100644 --- a/tests/test_config.cpp +++ b/tests/test_config.cpp @@ -41,7 +41,12 @@ TEST_CASE("CircuitConfig") { CHECK_THROWS_AS(config.getEdgePopulationProperties("DoesNotExist"), SonataError); CHECK(config.getNodePopulationProperties("nodes-A").type == "biophysical"); + CHECK(endswith(config.getNodePopulationProperties("nodes-A").typesPath, "")); + CHECK(endswith(config.getNodePopulationProperties("nodes-A").elementsPath, "tests/data/nodes1.h5")); + CHECK(config.getEdgePopulationProperties("edges-AB").type == "chemical_synapse"); + CHECK(endswith(config.getEdgePopulationProperties("edges-AB").typesPath, "")); + CHECK(endswith(config.getEdgePopulationProperties("edges-AB").elementsPath, "tests/data/edges1.h5")); CHECK_NOTHROW(nlohmann::json::parse(config.getExpandedJSON())); CHECK(nlohmann::json::parse(config.getExpandedJSON())