Skip to content

Commit

Permalink
make h5 files available through PopulationProperties (#230)
Browse files Browse the repository at this point in the history 
* circuit_config.json must now conform to the BBP standard for
  populations to be found
* the circuit_config.json now is the ground truth for available
  properties: if multiple nodes files contain the same population name,
  only the one referenced in the circuit_config.json is considered
  • Loading branch information
@mgeplf
mgeplf authored Oct 7, 2022
1 parent 5f08cd4 commit 794555a
Show file tree
Hide file tree
Showing 7 changed files with 124 additions and 115 deletions.
14 changes: 14 additions & 0 deletions include/bbp/sonata/config.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,20 @@ struct SONATA_API PopulationProperties {
* Dictionary for alternate directory paths.
*/
std::unordered_map<std::string, std::string> alternateMorphologyFormats;

/**
* Path to underlying elements H5 file.
* It is discouraged to directly access the contents of the file.
* Instead use 'libsonata' to read this file.
*/
std::string elementsPath;

/**
* Path to underlying types csv file.
* It is discouraged to directly access the contents of the file.
* Instead use 'libsonata' to read this file.
*/
std::string typesPath;
};

/**
Expand Down
8 changes: 7 additions & 1 deletion python/bindings.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -509,7 +509,13 @@ PYBIND11_MODULE(_libsonata, m) {
DOC_POPULATION_PROPERTIES(morphologiesDir))
.def_readonly("alternate_morphology_formats",
&PopulationProperties::alternateMorphologyFormats,
DOC_POPULATION_PROPERTIES(alternateMorphologyFormats));
DOC_POPULATION_PROPERTIES(alternateMorphologyFormats))
.def_readonly("elements_path",
&PopulationProperties::elementsPath,
DOC_POPULATION_PROPERTIES(elementsPath))
.def_readonly("types_path",
&PopulationProperties::typesPath,
DOC_POPULATION_PROPERTIES(typesPath));

py::class_<CircuitConfig>(m, "CircuitConfig", "")
.def(py::init<const std::string&, const std::string&>())
Expand Down
14 changes: 10 additions & 4 deletions python/generated/docstrings.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,10 +230,20 @@ static const char *__doc_bbp_sonata_PopulationProperties_alternateMorphologyForm

static const char *__doc_bbp_sonata_PopulationProperties_biophysicalNeuronModelsDir = R"doc(Path to the template HOC files defining the E-Mode)doc";

static const char *__doc_bbp_sonata_PopulationProperties_elementsPath =
R"doc(Path to underlying elements H5 file. It is discouraged to directly
access the contents of the file. Instead use 'libsonata' to read this
file.)doc";

static const char *__doc_bbp_sonata_PopulationProperties_morphologiesDir = R"doc(Path to the directory containing the morphologies)doc";

static const char *__doc_bbp_sonata_PopulationProperties_type = R"doc(Population type)doc";

static const char *__doc_bbp_sonata_PopulationProperties_typesPath =
R"doc(Path to underlying types csv file. It is discouraged to directly
access the contents of the file. Instead use 'libsonata' to read this
file.)doc";

static const char *__doc_bbp_sonata_PopulationStorage = R"doc(Collection of {PopulationClass}s stored in a H5 file and optional CSV.)doc";

static const char *__doc_bbp_sonata_PopulationStorage_Impl = R"doc()doc";
Expand Down Expand Up @@ -558,10 +568,6 @@ R"doc(Properties to assign values to variables in synapse MOD files. The
format is a dictionary with keys being the SUFFIX names and values
being dictionaries of variables' names and values.)doc";

static const char *__doc_bbp_sonata_SimulationConfig_Conditions_minisSingleVesicle =
R"doc(Limit spontaneous release to single vesicle when true. Default is
false)doc";

static const char *__doc_bbp_sonata_SimulationConfig_Conditions_modifications =
R"doc(Collection of dictionaries with each member decribing a modification
that mimics experimental manipulations to the circuit.)doc";
Expand Down
6 changes: 6 additions & 0 deletions python/tests/test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -571,12 +571,18 @@ def test_get_population_properties(self):
self.assertTrue(node_prop.biophysical_neuron_models_dir.endswith('biophysical_neuron_models'))
self.assertEqual(node_prop.alternate_morphology_formats, {})

self.assertEqual(node_prop.types_path, '')
self.assertTrue(node_prop.elements_path.endswith('tests/data/nodes1.h5'))

edge_prop = self.config.edge_population_properties('edges-AC')
self.assertEqual(edge_prop.type, 'chemical_synapse')
self.assertTrue(edge_prop.morphologies_dir.endswith('morphologies'))
self.assertTrue(edge_prop.biophysical_neuron_models_dir.endswith('biophysical_neuron_models'))
self.assertEqual(edge_prop.alternate_morphology_formats, {})

self.assertEqual(edge_prop.types_path, '')
self.assertTrue(edge_prop.elements_path.endswith('tests/data/edges1.h5'))


def test_biophysical_properties_raises(self):
contents = {
Expand Down
180 changes: 72 additions & 108 deletions src/config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,47 @@ namespace {
// to be replaced by std::filesystem once C++17 is used
namespace fs = ghc::filesystem;

void checkBiophysicalPopulations(
const std::unordered_map<std::string, PopulationProperties>& populations) {
for (const auto& it : populations) {
const auto& population = it.first;
const auto& properties = it.second;
if (properties.type == "biophysical") {
if (properties.morphologiesDir.empty() &&
properties.alternateMorphologyFormats.empty()) {
throw SonataError(
fmt::format("Node population '{}' is defined as 'biophysical' "
"but does not define 'morphologies_dir' or "
"'alternateMorphologyFormats'",
population));
} else if (properties.biophysicalNeuronModelsDir.empty()) {
throw SonataError(
fmt::format("Node population '{}' is defined as 'biophysical' "
"but does not define 'biophysical_neuron_models_dir'",
population));
}
}
}
}

PopulationProperties getPopulationProperties(
const std::string& populationName,
const std::unordered_map<std::string, PopulationProperties>& populations) {
auto it = populations.find(populationName);
if (it == populations.end()) {
throw SonataError(fmt::format("Could not find population '{}'", populationName));
}

return it->second;
}

template <typename PopulationType>
PopulationType getPopulation(const std::string& populationName,
const std::unordered_map<std::string, PopulationProperties>& src) {
const auto properties = getPopulationProperties(populationName, src);
return PopulationType(properties.elementsPath, properties.typesPath, populationName);
}

std::map<std::string, std::string> replaceVariables(std::map<std::string, std::string> variables) {
constexpr size_t maxIterations = 10;

Expand Down Expand Up @@ -589,6 +630,23 @@ class CircuitConfig::Parser
return result;
}

template <typename JSON>
std::tuple<std::string, std::string> parseSubNetworksMG(const std::string& prefix,
const JSON& value) const {
const std::string elementsFile = prefix + "s_file";
auto h5File = getJSONPath(value, elementsFile);

if (h5File.empty()) {
throw SonataError(
fmt::format("'{}' network do not define '{}' entry", prefix, elementsFile));
}

const std::string typesFile = prefix + "_types_file";
auto csvFile = getJSONPath(value, typesFile);

return {h5File, csvFile};
}

template <typename PopulationType>
Subnetworks parseSubNetworks(const std::string& prefix) const {
using PopulationStorage = bbp::sonata::PopulationStorage<PopulationType>;
Expand Down Expand Up @@ -631,6 +689,10 @@ class CircuitConfig::Parser

// Iterate over all defined subnetworks
for (const auto& node : network) {
std::string elementsPath;
std::string typesPath;
std::tie(elementsPath, typesPath) = parseSubNetworksMG(prefix, node);

const auto populationsIt = node.find("populations");
if (populationsIt == node.end()) {
continue;
Expand All @@ -642,6 +704,9 @@ class CircuitConfig::Parser

PopulationProperties& popProperties = output[it.key()];

popProperties.elementsPath = elementsPath;
popProperties.typesPath = typesPath;

popProperties.type = getJSONValue<std::string>(popData, "type");
popProperties.morphologiesDir = getJSONPath(popData, "morphologies_dir");
popProperties.biophysicalNeuronModelsDir =
Expand Down Expand Up @@ -678,73 +743,6 @@ class CircuitConfig::Parser
nlohmann::json _json;
};

class CircuitConfig::PopulationResolver
{
public:
static std::set<std::string> listPopulations(const std::vector<SubnetworkFiles>& src) {
std::set<std::string> result;
for (const auto& subNetwork : src) {
result.insert(subNetwork.populations.begin(), subNetwork.populations.end());
}
return result;
}

template <typename PopulationType>
static PopulationType getPopulation(const std::string& populationName,
const std::vector<SubnetworkFiles>& src) {
for (const auto& subNetwork : src) {
for (const auto& population : subNetwork.populations) {
if (population == populationName) {
return PopulationType(subNetwork.elements, subNetwork.types, populationName);
}
}
}

throw SonataError(fmt::format("Could not find population '{}'", populationName));
}

static void checkDuplicatePopulations(const std::vector<SubnetworkFiles>& src) {
std::set<std::string> check;
for (const auto& subNetwork : src) {
for (const auto& population : subNetwork.populations) {
if (check.find(population) != check.end()) {
throw SonataError(fmt::format("Duplicate population name '{}'", population));
}
check.insert(population);
}
}
}

static void checkBiophysicalPopulations(
const std::vector<SubnetworkFiles>& src,
const std::unordered_map<std::string, PopulationProperties>& populations) {
for (const auto& subNetwork : src) {
for (const auto& population : subNetwork.populations) {
const auto it = populations.find(population);
if (it == populations.end()) {
continue;
}

if (it->second.type == "biophysical") {
if (it->second.morphologiesDir.empty() &&
it->second.alternateMorphologyFormats.empty()) {
throw SonataError(
fmt::format("Node population '{}' is defined as 'biophysical' "
"but does not define 'morphologies_dir' or "
"'alternateMorphologyFormats'",
population));
} else if (it->second.biophysicalNeuronModelsDir.empty()) {
throw SonataError(
fmt::format("Node population '{}' is defined as 'biophysical' "
"but does not define 'biophysical_neuron_models_dir'",
population));
}
}
}
}
}
};

CircuitConfig::CircuitConfig(const std::string& contents, const std::string& basePath) {
Parser parser(contents, basePath);

Expand All @@ -754,17 +752,9 @@ CircuitConfig::CircuitConfig(const std::string& contents, const std::string& bas

_nodeSetsFile = parser.getNodeSetsPath();

// Load node subnetwork and check for duplicate populations
_networkNodes = parser.parseNodeNetwork();
PopulationResolver::checkDuplicatePopulations(_networkNodes);

// Load node population overrides and check biophysical types
_nodePopulationProperties = parser.parseNodePopulations();

// Load edge subnetowrk and check for duplicate populations
_networkEdges = parser.parseEdgeNetwork();
PopulationResolver::checkDuplicatePopulations(_networkEdges);

// Load edge population overrides
_edgePopulationProperties = parser.parseEdgePopulations();

Expand All @@ -790,7 +780,7 @@ CircuitConfig::CircuitConfig(const std::string& contents, const std::string& bas
updateDefaultProperties(_nodePopulationProperties, "biophysical");
updateDefaultProperties(_edgePopulationProperties, "chemical_synapse");

PopulationResolver::checkBiophysicalPopulations(_networkNodes, _nodePopulationProperties);
checkBiophysicalPopulations(_nodePopulationProperties);
}

CircuitConfig CircuitConfig::fromFile(const std::string& path) {
Expand All @@ -802,53 +792,27 @@ const std::string& CircuitConfig::getNodeSetsPath() const {
}

std::set<std::string> CircuitConfig::listNodePopulations() const {
return PopulationResolver::listPopulations(_networkNodes);
return getMapKeys(_nodePopulationProperties);
}

NodePopulation CircuitConfig::getNodePopulation(const std::string& name) const {
return PopulationResolver::getPopulation<NodePopulation>(name, _networkNodes);
return getPopulation<NodePopulation>(name, _nodePopulationProperties);
}

std::set<std::string> CircuitConfig::listEdgePopulations() const {
return PopulationResolver::listPopulations(_networkEdges);
return getMapKeys(_edgePopulationProperties);
}

EdgePopulation CircuitConfig::getEdgePopulation(const std::string& name) const {
return PopulationResolver::getPopulation<EdgePopulation>(name, _networkEdges);
return getPopulation<EdgePopulation>(name, _edgePopulationProperties);
}

PopulationProperties CircuitConfig::getNodePopulationProperties(const std::string& name) const {
auto populations = listNodePopulations();
if (populations.find(name) == populations.end()) {
throw SonataError(fmt::format("Could not find node population '{}'", name));
}

auto popPropertiesIt = _nodePopulationProperties.find(name);
if (popPropertiesIt != _nodePopulationProperties.end()) {
return popPropertiesIt->second;
}

return {"biophysical",
_components.biophysicalNeuronModelsDir,
_components.morphologiesDir,
_components.alternateMorphologiesDir};
return getPopulationProperties(name, _nodePopulationProperties);
}

PopulationProperties CircuitConfig::getEdgePopulationProperties(const std::string& name) const {
auto populations = listEdgePopulations();
if (populations.find(name) == populations.end()) {
throw SonataError(fmt::format("Could not find edge population '{}'", name));
}

auto popPropertiesIt = _edgePopulationProperties.find(name);
if (popPropertiesIt != _edgePopulationProperties.end()) {
return popPropertiesIt->second;
}

return {"chemical_synapse",
_components.biophysicalNeuronModelsDir,
_components.morphologiesDir,
_components.alternateMorphologiesDir};
return getPopulationProperties(name, _edgePopulationProperties);
}

const std::string& CircuitConfig::getExpandedJSON() const {
Expand Down
12 changes: 10 additions & 2 deletions tests/data/config/circuit_config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,21 @@
"nodes": [
{
"nodes_file": "$NETWORK_DIR/nodes1.h5",
"node_types_file": null
"node_types_file": null,
"populations": {
"nodes-A": {},
"nodes-B": {}
}
}
],
"edges": [
{
"edges_file": "$NETWORK_DIR/edges1.h5",
"edge_types_file": null
"edge_types_file": null,
"populations": {
"edges-AB": {},
"edges-AC": {}
}
}
]
}
Expand Down
5 changes: 5 additions & 0 deletions tests/test_config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,12 @@ TEST_CASE("CircuitConfig") {
CHECK_THROWS_AS(config.getEdgePopulationProperties("DoesNotExist"), SonataError);

CHECK(config.getNodePopulationProperties("nodes-A").type == "biophysical");
CHECK(endswith(config.getNodePopulationProperties("nodes-A").typesPath, ""));
CHECK(endswith(config.getNodePopulationProperties("nodes-A").elementsPath, "tests/data/nodes1.h5"));

CHECK(config.getEdgePopulationProperties("edges-AB").type == "chemical_synapse");
CHECK(endswith(config.getEdgePopulationProperties("edges-AB").typesPath, ""));
CHECK(endswith(config.getEdgePopulationProperties("edges-AB").elementsPath, "tests/data/edges1.h5"));

CHECK_NOTHROW(nlohmann::json::parse(config.getExpandedJSON()));
CHECK(nlohmann::json::parse(config.getExpandedJSON())
Expand Down

0 comments on commit 794555a

Please sign in to comment.