From ea81bd97d35accc65548ab75861a48ca9fd5ea7f Mon Sep 17 00:00:00 2001 From: WeinaJi Date: Wed, 7 Feb 2024 16:13:30 +0100 Subject: [PATCH] [BBPBGLIB-1120] Remove node loaders for MVD3 and NCS (#117) ## Context After deprecating BlueConfig files, we no longer support legacy node format, MVD3 or NCS. Therefore, this PR removes the corresponding node loaders. ## Scope In `cell_reader.py`: keeping only `load_sonata` and removing the other readers. In `metype.py`: removing `Cell_V5`. ## Testing Adapt the current unit tests to use SONATA nodes. ## Review * [x] PR description is complete * [x] Coding style (imports, function length, New functions, classes or files) are good * [x] Unit/Scientific test added * [x] Updated Readme, in-code, developer documentation --- docs/api/subpackages/io.rst | 5 +- docs/architecture.rst | 5 +- docs/examples.rst | 2 +- neurodamus/cell_distributor.py | 64 ++---- neurodamus/core/configuration.py | 3 - neurodamus/io/cell_readers.py | 246 +--------------------- neurodamus/metype.py | 81 ------- neurodamus/ngv.py | 4 +- tests/integration-e2e/test_loadbalance.py | 70 +++--- tests/integration-e2e/test_plugin.py | 3 +- tests/scientific/conftest.py | 2 - 11 files changed, 59 insertions(+), 426 deletions(-) diff --git a/docs/api/subpackages/io.rst b/docs/api/subpackages/io.rst index 381da353..0a0d32f6 100644 --- a/docs/api/subpackages/io.rst +++ b/docs/api/subpackages/io.rst @@ -14,10 +14,7 @@ neurodamus.io.cell\_readers .. autosummary:: - load_mvd3 - load_ncs - load_nodes_mvd3 - load_combo_metypes + load_sonata split_round_robin .. rubric:: Exceptions diff --git a/docs/architecture.rst b/docs/architecture.rst index 1c675afb..18539c58 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -103,7 +103,7 @@ BBP v6 circuits would define its own cell hoc templates (/classes, one for each based on Cell.hoc. Morphologically detailed neurons are the heart of a Neuron simulation. Neurodamus will read the -cells metadata from a node file (start.ncs, circuit.mvd3 or nodes.h5 sonata) and instantiate each +cells metadata from a node file in SONATA format and instantiate each individual cell by: - loading the morphology and instantiate the respective sections @@ -138,7 +138,7 @@ Due to extensively different logic, both these cell types inherit directly from must be handled by their own cell manager classes. To these modules implementing a new `Cell`, `CellManagers` and eventually `ConnectionManager` types we call **Engine** - more on this later. -For the record, `Cell_V5/V6` and the corresponding `CellDistributor` and `SynapseRuleManager` +For the record, `Cell_V6` and the corresponding `CellDistributor` and `SynapseRuleManager` were also made components of a special engine: the `METypeEngine`, obviously built-in (find it in node.py). @@ -180,7 +180,6 @@ considered a cell manager, but by far lighter than a full cell manager. * `CellType`: The default cell class this manager handles. * `_node_loader`: The default loader function for nodes (cell metadata) - * `_node_format`: The default file format of nodes, .e.g. Sonata - Instance properties: diff --git a/docs/examples.rst b/docs/examples.rst index 5fe5ba50..7a0365d9 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -85,7 +85,7 @@ Dry run mode ~~~~~~~~~~~~ In order to obtain a more accurate estimation of the resources needed for a simulation, users can also run Neurodamus in dry run mode. This functionality is only available -for libsonata circuits. MVD3 circuits are not supported. +for SONATA circuits. This mode will partially instantiate cells and synapses to get a statistical overview of the memory used but won't run the actual simulation. diff --git a/neurodamus/cell_distributor.py b/neurodamus/cell_distributor.py index cb8b53da..8ce49bee 100644 --- a/neurodamus/cell_distributor.py +++ b/neurodamus/cell_distributor.py @@ -8,7 +8,6 @@ import os import weakref from contextlib import contextmanager -from enum import Enum from io import StringIO from os import path as ospath from pathlib import Path @@ -24,19 +23,13 @@ from .core.nodeset import NodeSet from .io import cell_readers from .lfp_manager import LFPManager -from .metype import Cell_V5, Cell_V6, EmptyCell +from .metype import Cell_V6, EmptyCell from .target_manager import TargetSpec from .utils import compat from .utils.logging import log_verbose, log_all from .utils.memory import DryRunStats, get_mem_usage_kb -class NodeFormat(Enum): - NCS = 1 - MVD3 = 2 - SONATA = 3 - - class VirtualCellPopulation: """ A virtual cell population offers a compatible interface with Cell Manager, @@ -110,15 +103,11 @@ class CellManagerBase(_CellManager): """ _node_loader = None - """Default function implementing the loading of nodes data, a.k.a. MVD + """Default function implementing the loading of nodes data signature: load(circuit_conf, gidvec, stride=1, stride_offset=0) """ - - _node_format = NodeFormat.SONATA # NCS, Mvd, Sonata... - """Default Node file format""" - def __init__(self, circuit_conf, target_manager, _run_conf=None, **_kw): """Initializes CellDistributor @@ -179,12 +168,11 @@ def get_final_gids(self): return numpy.array(self.local_nodes.final_gids()) def _init_config(self, circuit_conf, pop): - if self._node_format == NodeFormat.SONATA: - if not ospath.isabs(circuit_conf.CellLibraryFile): - circuit_conf.CellLibraryFile = find_input_file(circuit_conf.CellLibraryFile) - if not pop: # Last attempt to get pop name - pop = self._get_sonata_population_name(circuit_conf.CellLibraryFile) - logging.info(" -> Discovered node population name: %s", pop) + if not ospath.isabs(circuit_conf.CellLibraryFile): + circuit_conf.CellLibraryFile = find_input_file(circuit_conf.CellLibraryFile) + if not pop: # Last attempt to get pop name + pop = self._get_sonata_population_name(circuit_conf.CellLibraryFile) + logging.info(" -> Discovered node population name: %s", pop) if not pop and circuit_conf._name: pop = circuit_conf._name logging.warning("(Compat) Assuming population name from Circuit: %s", pop) @@ -520,28 +508,11 @@ class CellDistributor(CellManagerBase): Instantiated cells are stored locally (.cells property) """ - _cell_loaders = { - "start.ncs": cell_readers.load_ncs, - "circuit.mvd3": cell_readers.load_mvd3, - } - _sonata_with_extra_attrs = True # Enable search extra node attributes def _init_config(self, circuit_conf, _pop): if not circuit_conf.CellLibraryFile: - logging.warning("CellLibraryFile not set. Assuming legacy 'start.ncs'") - circuit_conf.CellLibraryFile = "start.ncs" - if circuit_conf.CellLibraryFile.endswith(".ncs"): - self._node_format = NodeFormat.NCS - elif circuit_conf.CellLibraryFile.endswith(".mvd3"): - self._node_format = NodeFormat.MVD3 - - self._is_v5_circuit = circuit_conf.CellLibraryFile == "start.ncs" or ( - circuit_conf.nrnPath and ospath.isfile(ospath.join(circuit_conf.nrnPath, "start.ncs")) - and not ospath.isfile(ospath.join(circuit_conf.CircuitPath, "circuit.mvd3")) - ) - if self._is_v5_circuit and self._circuit_conf.DetailedAxon: - raise ConfigurationError("V5 circuits don't support keeping detailed axons") + raise ConfigurationError("CellLibraryFile not set") super()._init_config(circuit_conf, _pop) @@ -553,19 +524,12 @@ def load_nodes(self, load_balancer=None, **kw): cell_requirements = all_cell_requirements.get(self._population_name) or ( self.is_default and all_cell_requirements.get(None) ) - if self._node_format == NodeFormat.SONATA: - loader = cell_readers.load_sonata - loader_opts["node_population"] = self._population_name # mandatory in Sonata - loader_opts["load_dynamic_props"] = cell_requirements - loader_opts["has_extra_data"] = self._sonata_with_extra_attrs - else: - if cell_requirements: - raise Exception('Additional cell properties only available with SONATA') - nodes_filename = self._circuit_conf.CellLibraryFile - loader = self._cell_loaders.get(nodes_filename, cell_readers.load_mvd3) - loader_opts = {} + loader = cell_readers.load_sonata + loader_opts["node_population"] = self._population_name # mandatory in Sonata + loader_opts["load_dynamic_props"] = cell_requirements + loader_opts["has_extra_data"] = self._sonata_with_extra_attrs - log_verbose("Nodes Format: %s, Loader: %s", self._node_format, loader.__name__) + log_verbose("Nodes Format: SONATA , Loader: %s", loader.__name__) return super().load_nodes(load_balancer, _loader=loader, loader_opts=loader_opts) def _instantiate_cells(self, dry_run_stats_obj: DryRunStats = None, **opts): @@ -578,7 +542,7 @@ def _instantiate_cells(self, dry_run_stats_obj: DryRunStats = None, **opts): if self.CellType is not NotImplemented: return super()._instantiate_cells(self.CellType) conf = self._circuit_conf - CellType = Cell_V5 if self._is_v5_circuit else Cell_V6 + CellType = Cell_V6 if conf.MorphologyType: CellType.morpho_extension = conf.MorphologyType diff --git a/neurodamus/core/configuration.py b/neurodamus/core/configuration.py index 9d035342..1a139248 100644 --- a/neurodamus/core/configuration.py +++ b/neurodamus/core/configuration.py @@ -545,9 +545,6 @@ def _make_circuit_config(config_dict, req_morphology=True): config_dict["MorphologyPath"] = False elif config_dict.get("nrnPath") == "": config_dict["nrnPath"] = False - if config_dict.get("CellLibraryFile", "start.ncs") == "start.ncs": - raise ConfigurationError( - "ncs circuits don't support disabling connectivity with nrnPath=''") _validate_circuit_morphology(config_dict, req_morphology) _validate_file_extension(config_dict.get("CellLibraryFile")) _validate_file_extension(config_dict.get("nrnPath")) diff --git a/neurodamus/io/cell_readers.py b/neurodamus/io/cell_readers.py index 269309b8..a3e2e768 100644 --- a/neurodamus/io/cell_readers.py +++ b/neurodamus/io/cell_readers.py @@ -9,10 +9,9 @@ from os import path as ospath from ..core import NeurodamusCore as Nd -from ..core.configuration import ConfigurationError, SimConfig +from ..core.configuration import SimConfig from ..core import run_only_rank0 -from ..metype import METypeManager, METypeItem -from ..utils import compat +from ..metype import METypeManager from ..utils.logging import log_verbose EMPTY_GIDVEC = np.empty(0, dtype="uint32") @@ -22,33 +21,6 @@ class CellReaderError(Exception): pass -def _ncs_get_total(ncs_f): - # first lines might be comments. Skip '#' - for tstr in ncs_f: - if not tstr.strip().startswith("#"): - break - # should have "Cells x" - try: - total_circuit_cells = int(tstr.strip().split()[1]) - except IndexError: - raise CellReaderError("NCS file contains invalid entry: " + tstr) - return total_circuit_cells - - -def _ncs_get_cells(ncs_f): - ncs_f.readline() # skip the '{' - - for cell_i, line in enumerate(ncs_f): - line = line.strip() - if line == "}": - break - parts = line.split() - assert len(parts) >= 5, "Error in ncs line " + line - _gid = int(parts[0][1:]) - metype = parts[4] - yield cell_i, _gid, metype - - def split_round_robin(all_gids, stride=1, stride_offset=0, total_cells=None): """ Splits a numpy ndarray[uint32] round-robin. If the array is None generates new arrays based on the nr of total cells @@ -106,168 +78,6 @@ def dry_run_distribution(gid_metype_bundle, stride=1, stride_offset=0, total_cel return np.concatenate(groups) if groups else EMPTY_GIDVEC -def load_ncs(circuit_conf, all_gids, stride=1, stride_offset=0): - """ Obtain the gids and the metypes for cells in the base circuit. - - Args: - circuit_conf: the Run secgion from the configuration - all_gids: The cells ids to be loaded. If it's None then all the cells shall be loaded - stride: If distribution is desired stride can be set to a value > 1 - stride_offset: When using distribution, the offset to be read within the stride - Returns: - A tuple of (gids, metypes, total_ncs_cells) - """ - gids = compat.Vector("I") - gid2mefile = {} - # NCS is historically under nrnPath which nowadays is a file path - ncs_dir = circuit_conf.nrnPath - if ospath.isfile(ncs_dir): - ncs_dir = ospath.dirname(ncs_dir) - ncs_path = ospath.join(ncs_dir, "start.ncs") - - with open(ncs_path, "r") as ncs_f: - total_ncs_cells = _ncs_get_total(ncs_f) - if all_gids is None: - for cellIndex, gid, metype in _ncs_get_cells(ncs_f): - if cellIndex % stride == stride_offset: - gids.append(gid) - gid2mefile[gid] = metype - else: - # Index desired cells - gid2mefile = {int(gid): None for i, gid in enumerate(all_gids) - if i % stride == stride_offset} - gids.extend(gid2mefile.keys()) - assigned_metypes = 0 - for cellIndex, gid, metype in _ncs_get_cells(ncs_f): - if gid in gid2mefile: - gid2mefile[gid] = metype - assigned_metypes += 1 - if assigned_metypes < len(gid2mefile): - logging.error("start.ncs: found info only for %d cells (out of %d)", - assigned_metypes, len(all_gids)) - raise CellReaderError("Target contains invalid circuit cells") - return gids, gid2mefile, total_ncs_cells - - -def load_mvd3(circuit_conf, all_gids, stride=1, stride_offset=0): - """Load cells from MVD3, required for v6 circuits - reuse load_nodes with mvdtool by default - if py-mvdtool not installed, use the old h5py loader - """ - try: - import mvdtool # noqa: F401 - except ImportError: - logging.warning("Cannot import mvdtool to load mvd3, will load with h5py") - return _load_mvd3_h5py(circuit_conf, all_gids, stride, stride_offset) - - return load_nodes_mvd3(circuit_conf, all_gids, stride, stride_offset) - - -def _load_mvd3_h5py(circuit_conf, all_gids, stride=1, stride_offset=0): - """Load cells from MVD3 using h5py - """ - import h5py # Can be heavy so loaded on demand - pth = ospath.join(circuit_conf.CircuitPath, "circuit.mvd3") - mvd = h5py.File(pth, 'r') - - mecombo_ds = mvd["/cells/properties/me_combo"] - total_mvd_cells = len(mecombo_ds) - - gidvec = split_round_robin(all_gids, stride, stride_offset, total_mvd_cells) - - if not len(gidvec): - # Not enough cells to give this rank a few - return compat.Vector('I'), METypeManager(), total_mvd_cells - - # Indexes are 0-based, and cant be numpy - indexes = compat.Vector("I", gidvec - 1) - - morph_ids = mvd["/cells/properties/morphology"][indexes] - combo_ids = mvd["/cells/properties/me_combo"][indexes] - morpho_ds = mvd["/library/morphology"] - morpho_names = [str(morpho_ds[i]) for i in morph_ids] - combo_ds = mvd["/library/me_combo"] - combo_names = [str(combo_ds[i]) for i in combo_ids] - - # We require gidvec as compat.Vector - gidvec = compat.Vector("I", gidvec) - - # now we can take the combo file and get the emodel + additional info - combo_file = circuit_conf.MEComboInfoFile - me_manager = load_combo_metypes(combo_file, gidvec, combo_names, morpho_names) - - return gidvec, me_manager, total_mvd_cells - - -def load_nodes_mvd3(circuit_conf, all_gids, stride=1, stride_offset=0): - """Load cells from MVD3 file. - node_population can be provided by load_sonata() and can be None. False to auto-detect - """ - try: - import mvdtool - except ImportError: - raise ConfigurationError("load_nodes: mvdtool is not available. Please install") - pth = circuit_conf.CellLibraryFile - assert pth.endswith('.mvd3'), "CellLibraryFile must be a mvd3 file" - if not ospath.isfile(pth): - pth = ospath.join(circuit_conf.CircuitPath, pth) - if not ospath.isfile(pth): - raise ConfigurationError("Could not find Nodes: " + circuit_conf.CellLibraryFile) - - node_reader = mvdtool.open(pth) - combo_file = circuit_conf.MEComboInfoFile - - if not combo_file: - logging.warning("Missing BlueConfig field 'MEComboInfoFile' which has gid:mtype:emodel") - else: - node_reader.open_combo_tsv(combo_file) - - total_cells = len(node_reader) - - meinfo = METypeManager() - - if SimConfig.dry_run: - raise Exception("Dry run is not supported for MVD3") - else: - gidvec = split_round_robin(all_gids, stride, stride_offset, total_cells) - - if not len(gidvec): - # Not enough cells to give this rank a few - return gidvec, METypeManager(), total_cells - - fetch_MEinfo(node_reader, gidvec, combo_file, meinfo) - - return gidvec, meinfo, total_cells - - -def fetch_MEinfo(node_reader, gidvec, combo_file, meinfo): - - indexes = np.sort(gidvec - 1) # MVDtool requires 0-indexed ids. - gidvec = indexes + 1 # Return 1-indexed ids - if len(indexes) < 10: # Ensure array is not too small (pybind11 #1392) - indexes = indexes.tolist() - - morpho_names = node_reader.morphologies(indexes) - mtypes = node_reader.mtypes(indexes) - emodels = node_reader.emodels(indexes) \ - if combo_file else None # Rare but we may not need emodels (ngv) - etypes = node_reader.etypes(indexes) \ - if combo_file else None - exc_mini_freqs = node_reader.exc_mini_frequencies(indexes) \ - if node_reader.hasMiniFrequencies() else None - inh_mini_freqs = node_reader.inh_mini_frequencies(indexes) \ - if node_reader.hasMiniFrequencies() else None - threshold_currents = node_reader.threshold_currents(indexes) \ - if node_reader.hasCurrents() else None - holding_currents = node_reader.holding_currents(indexes) \ - if node_reader.hasCurrents() else None - positions = node_reader.positions(indexes) - rotations = node_reader.rotations(indexes) if node_reader.rotated else None - - meinfo.load_infoNP(gidvec, morpho_names, emodels, mtypes, etypes, threshold_currents, - holding_currents, exc_mini_freqs, inh_mini_freqs, positions, rotations) - - def load_sonata(circuit_conf, all_gids, stride=1, stride_offset=0, *, node_population, load_dynamic_props=(), has_extra_data=False, dry_run_stats=None): """ @@ -388,58 +198,6 @@ def validate_property(prop_name): return gidvec, meinfos, fullsize -def load_combo_metypes(combo_file, gidvec, combo_list, morph_list): - """ Read file with mecombo info, retaining only those that are local to this node - Args: - combo_file: Path to the combo file to read metype info from - gidvec: local gids to load info about - combo_list: comboList Combos corresponding to local gids - morph_list: morphList Morpholgies corresponding to local gids - """ - if not combo_file: - logging.error("Missing BlueConfig field 'MEComboInfoFile' which has gid:mtype:emodel.") - raise ConfigurationError("MEComboInfoFile not specified") - - # Optimization: index combos - combo_ids = defaultdict(list) - for i, c in enumerate(combo_list): - combo_ids[c].append(i) - - log_verbose("Loading emodel+additional info from Combo f %s", combo_file) - f = open(combo_file) - next(f) # Skip Header - - me_manager = METypeManager() - for tstr in f: - vals = tstr.strip().split() - if len(vals) not in (6, 8): - wmsg = ("Could not parse line %s from MEComboInfoFile %s." - "Expecting 6 (hippocampus) or 8 (somatosensory) fields") - logging.warning(wmsg, tstr, combo_file) - - # metypes may be reused by several cells - # We create a single item and later assign to each matching gid - meitem = METypeItem(*vals) - for i in combo_ids[meitem.combo_name]: - if morph_list[i] == meitem.morph_name: - me_manager[int(gidvec[i])] = meitem - - # confirm that all gids have been matched. - # Otherwise, print combo + morph info to help finding issues - nerr = 0 - for gid in gidvec: - gid = int(gid) - if gid not in me_manager: - logging.error("MEComboInfoFile: No MEInfo for gid %d", gid) - nerr += 1 - if nerr > 0: - logging.error("gidvec: " + str(gidvec)) - logging.error("Memap: " + str(me_manager.gids)) - raise CellReaderError("Errors found during processing of mecombo file. See log") - - return me_manager - - def _getNeededAttributes(node_reader, etype_path, emodels, gidvec): """ Read additional attributes required by emodel templates global var __NeededAttributes diff --git a/neurodamus/metype.py b/neurodamus/metype.py index 99448949..46f2b764 100644 --- a/neurodamus/metype.py +++ b/neurodamus/metype.py @@ -195,87 +195,6 @@ def __getattr__(self, item): return prop -class Cell_V5(METype): - __slots__ = ('_rng_list',) - - def __init__(self, gid, meinfo, circuit_conf): - # In NCS, meinfo is simply the metype filename (string) - mepath = circuit_conf.METypePath - morpho_path = circuit_conf.MorphologyPath - if isinstance(meinfo, METypeItem): - meinfo = meinfo.emodel_tpl # Compat with loading V5 cells from Sonata Nodes - melabel = self._load_template(meinfo, mepath) - super().__init__(gid, mepath, melabel, morpho_path) - - def _instantiate_cell(self, gid, _etype_path, emodel, morpho_path, _meinfos, _axon): - """Instantiates a cell v5 or older. Assumes emodel hoc templates are loaded - """ - EModel = getattr(Nd, emodel) - logging.debug("Loading Gid %d: emodel: %s", gid, emodel) - try: - # For this step, do not call mpi_abort in neuron and let neurodamus handle and abort, - # NB: Do not re-raise as ConfigurationError, neurodamus doesn't call mpi_abort so hangs - old_flag = Nd.pc.mpiabort_on_error(0) - self._ccell = ccell = EModel(gid, morpho_path) - Nd.pc.mpiabort_on_error(old_flag) - except Exception as e: - raise Exception("Error when loading Gid %d: emodel: %s, morpho_path: %s" - % (gid, emodel, morpho_path)) from e - self._cellref = ccell.CellRef - self._synapses = ccell.CellRef.synlist - self._syn_helper_list = ccell.CellRef.synHelperList - self._threshold_current = ccell.getThreshold() - try: - self._hypAmp_current = ccell.getHypAmp() - except Exception: - pass - - def re_init_rng(self, ion_seed): - if not hasattr(self._ccell, "re_init_rng"): - return # dont apply on cells without re_init_rng func - rng = SimConfig.rng_info - rng_mode = rng.getRNGMode() - - if rng_mode == rng.COMPATIBILITY: - return super().re_init_rng(ion_seed) - if rng_mode == rng.RANDOM123: - Nd.rng123ForStochKvInit(self._ccell) - return - # otherwise rng_mode is mcellran4 - self._rng_list = Nd.rngForStochKvInit(self._ccell) - gid = self._cellref.gid - if gid > 400000: - logging.warning("mcellran4 cannot initialize properly with large gids: %d", gid) - - @staticmethod - def _load_template(tpl_filename, tpl_location=None): - """Helper function which loads the template into NEURON and returns its name. - The actual template name will have any hyphens (e.g.: R-C261296A-P1_repaired) - replaced with underscores as hyphens must not appear in template names. - - Args: - tpl_filename: the template file to load - tpl_location: (Optional) path for the templates - Returns: - The name of the template as it appears inside the file (sans hyphens) - """ - # start.ncs gives metype names with hyphens, but the templates themselves - # have those hyphens replaced with underscores. - tpl_path = ospath.join(tpl_location, tpl_filename) \ - if tpl_location else tpl_filename - - # first open the file manually to get the hoc template name - tpl_name = None - with open(tpl_path + ".hoc", "r") as templateReader: - for line in templateReader: - line = line.strip() - if line.startswith("begintemplate"): - tpl_name = line.split()[1] - break - Nd.load_hoc(tpl_path) - return tpl_name - - class EmptyCell(BaseCell): """ Class representing an empty cell, e.g. an artificial cell diff --git a/neurodamus/ngv.py b/neurodamus/ngv.py index e2bc84ec..74fa42f0 100644 --- a/neurodamus/ngv.py +++ b/neurodamus/ngv.py @@ -24,7 +24,7 @@ class Astrocyte(BaseCell): __slots__ = ('_glut_list', '_secidx2names', '_nseg_warning') def __init__(self, gid, meinfos, circuit_conf): - """Instantiate a new Cell from mvd/node info.""" + """Instantiate a new Cell from node info.""" super().__init__(gid, meinfos, None) morpho_path = circuit_conf.MorphologyPath morph_filename = meinfos.morph_name + "." + circuit_conf.MorphologyType @@ -207,7 +207,7 @@ def getVersion(): class AstrocyteManager(CellDistributor): # Cell Manager is the same as CellDistributor, so it's able to handle - # the same Node formats (mvd, ...) and Cell morphologies. + # the same Node formats and Cell morphologies. # The difference lies only in the Cell Type CellType = Astrocyte _sonata_with_extra_attrs = False diff --git a/tests/integration-e2e/test_loadbalance.py b/tests/integration-e2e/test_loadbalance.py index 9ec6c8b0..7dbed753 100644 --- a/tests/integration-e2e/test_loadbalance.py +++ b/tests/integration-e2e/test_loadbalance.py @@ -11,16 +11,19 @@ @pytest.fixture -def target_manager_hoc(): - from neurodamus.target_manager import _HocTarget - t1 = _HocTarget("Small", None, _raw_gids=[1, 11, 21, 31, 41, 51, 61, 71, 81, 91, 101, 111]) - t2 = _HocTarget("VerySmall", None, _raw_gids=[1, 11, 21, 31, 41, 51]) +def target_manager(): + from neurodamus.target_manager import NodesetTarget + from neurodamus.core.nodeset import NodeSet + nodes_t1 = NodeSet([1, 11, 21, 31, 41, 51, 61, 71, 81, 91, 101, 111]).register_global("All") + nodes_t2 = NodeSet([1, 11, 21, 31, 41, 51]).register_global("All") + t1 = NodesetTarget("Small", [nodes_t1], [nodes_t1]) + t2 = NodesetTarget("VerySmall", [nodes_t2], [nodes_t2]) return MockedTargetManager(t1, t2) -def test_loadbal_no_cx(target_manager_hoc, caplog): +def test_loadbal_no_cx(target_manager, caplog): from neurodamus.cell_distributor import LoadBalance, TargetSpec - lbal = LoadBalance(1, "/gpfs/fake_path_to_nodes_1", "pop", target_manager_hoc, 4) + lbal = LoadBalance(1, "/gpfs/fake_path_to_nodes_1", "pop", target_manager, 4) assert not lbal._cx_targets assert not lbal._valid_loadbalance with caplog.at_level(logging.INFO): @@ -28,7 +31,7 @@ def test_loadbal_no_cx(target_manager_hoc, caplog): assert " => No complexity files for current circuit yet" in caplog.records[-1].message -def test_loadbal_subtarget(target_manager_hoc, caplog): +def test_loadbal_subtarget(target_manager, caplog): """Ensure given the right files are in the lbal dir, the correct situation is detected """ from neurodamus.cell_distributor import LoadBalance, TargetSpec @@ -38,7 +41,7 @@ def test_loadbal_subtarget(target_manager_hoc, caplog): lbdir, _ = LoadBalance._get_circuit_loadbal_dir(nodes_file, "pop") shutil.copyfile(SIM_DIR / "1k_v5_balance" / "cx_Small.dat", lbdir / "cx_Small#.dat") - lbal = LoadBalance(1, nodes_file, "pop", target_manager_hoc, 4) + lbal = LoadBalance(1, nodes_file, "pop", target_manager, 4) assert "Small" in lbal._cx_targets assert not lbal._valid_loadbalance with caplog.at_level(logging.INFO): @@ -60,45 +63,44 @@ def circuit_conf(): PROJ = "/gpfs/bbp.cscs.ch/project" return CircuitConfig( CircuitPath=PROJ + "/proj12/jenkins/cellular/circuit-2k", - CellLibraryFile="circuit.mvd3", - MEComboInfoFile=PROJ + "/proj64/entities/emodels/2017.11.03/mecombo_emodel.tsv", + CellLibraryFile=PROJ + "/proj12/jenkins/cellular/circuit-2k/nodes_v2.h5", METypePath=PROJ + "/proj64/entities/emodels/2017.11.03/hoc", MorphologyPath=PROJ + "/proj12/jenkins/cellular/circuit-2k/morphologies/ascii", nrnPath="", # no connectivity - CircuitTarget="Small" + CircuitTarget="All:Small" ) -def test_load_balance_integrated(target_manager_hoc, circuit_conf): +def test_load_balance_integrated(target_manager, circuit_conf): """Comprehensive test using real cells and deriving cx for a sub-target """ from neurodamus.cell_distributor import CellDistributor, LoadBalance, TargetSpec tmp_path = tempfile.TemporaryDirectory("test_load_balance_integrated") os.chdir(tmp_path.name) - cell_manager = CellDistributor(circuit_conf, target_manager_hoc) + cell_manager = CellDistributor(circuit_conf, target_manager) cell_manager.load_nodes() - lbal = LoadBalance(1, circuit_conf.CircuitPath, "", target_manager_hoc, 4) - t1 = TargetSpec("Small") + lbal = LoadBalance(1, circuit_conf.CircuitPath, "All", target_manager, 4) + t1 = TargetSpec("All:Small") assert not lbal._cx_valid(t1) with lbal.generate_load_balance(t1, cell_manager): cell_manager.finalize() - assert "Small" in lbal._cx_targets - assert "Small" in lbal._valid_loadbalance + assert "All_Small" in lbal._cx_targets + assert "All_Small" in lbal._valid_loadbalance assert lbal._cx_valid(t1) # Check subtarget - assert "VerySmall" not in lbal._cx_targets - assert "VerySmall" not in lbal._valid_loadbalance - assert lbal._reuse_cell_complexity(TargetSpec("VerySmall")) + assert "All_VerySmall" not in lbal._cx_targets + assert "All_VerySmall" not in lbal._valid_loadbalance + assert lbal._reuse_cell_complexity(TargetSpec("All:VerySmall")) # Check not super-targets assert not lbal._reuse_cell_complexity(TargetSpec(None)) -def test_multisplit(target_manager_hoc, circuit_conf, capsys): +def test_multisplit(target_manager, circuit_conf, capsys): """Comprehensive test using real cells, multi-split and complexity derivation """ from neurodamus.cell_distributor import CellDistributor, LoadBalance, TargetSpec @@ -106,10 +108,10 @@ def test_multisplit(target_manager_hoc, circuit_conf, capsys): tmp_path = tempfile.TemporaryDirectory("test_multisplit") os.chdir(tmp_path.name) - cell_manager = CellDistributor(circuit_conf, target_manager_hoc) + cell_manager = CellDistributor(circuit_conf, target_manager) cell_manager.load_nodes() - lbal = LoadBalance(MULTI_SPLIT, circuit_conf.CircuitPath, "", target_manager_hoc, 4) - t1 = TargetSpec("Small") + lbal = LoadBalance(MULTI_SPLIT, circuit_conf.CircuitPath, "All", target_manager, 4) + t1 = TargetSpec("All:Small") assert not lbal._cx_valid(t1) with lbal.generate_load_balance(t1, cell_manager): @@ -118,24 +120,24 @@ def test_multisplit(target_manager_hoc, circuit_conf, capsys): captured = capsys.readouterr() assert "13 pieces" in captured.out assert "at least one cell is broken into 2 pieces" in captured.out - assert "Small" in lbal._cx_targets - assert "Small" in lbal._valid_loadbalance + assert "All_Small" in lbal._cx_targets + assert "All_Small" in lbal._valid_loadbalance assert lbal._cx_valid(t1) # Convert balance for 1 CPU so we can import lbal.target_cpu_count = 1 - lbal._cpu_assign("Small") + lbal._cpu_assign("All_Small") binfo = lbal.load_balance_info(t1) assert binfo.npiece() == 13 # Ensure load-bal is reused for smaller targets in multisplit too - assert "VerySmall" not in lbal._cx_targets - assert "VerySmall" not in lbal._valid_loadbalance - assert lbal.valid_load_distribution(TargetSpec("VerySmall")) - assert "VerySmall" in lbal._cx_targets - assert "VerySmall" in lbal._valid_loadbalance + assert "All_VerySmall" not in lbal._cx_targets + assert "All_VerySmall" not in lbal._valid_loadbalance + assert lbal.valid_load_distribution(TargetSpec("All:VerySmall")) + assert "All_VerySmall" in lbal._cx_targets + assert "All_VerySmall" in lbal._valid_loadbalance captured = capsys.readouterr() - assert "Target VerySmall is a subset of the target Small" in captured.out + assert "Target VerySmall is a subset of the target All_Small" in captured.out def test_loadbal_integration(): @@ -157,7 +159,7 @@ class MockedTargetManager: """ def __init__(self, *targets) -> None: - self.targets = {t.name: t for t in targets} + self.targets = {t.name.split(":")[-1]: t for t in targets} def get_target(self, target_spec, target_pop=None): from neurodamus.target_manager import TargetSpec diff --git a/tests/integration-e2e/test_plugin.py b/tests/integration-e2e/test_plugin.py index 93fc5cc9..f734aee4 100644 --- a/tests/integration-e2e/test_plugin.py +++ b/tests/integration-e2e/test_plugin.py @@ -32,7 +32,7 @@ class CellName(str): pass def __init__(self, gid, cell_info, circuit_conf): - """Instantiate a new Cell from mvd/node info""" + """Instantiate a new Cell from node info""" from neurodamus.core import NeurodamusCore as Nd # dont load top-level because of pytest super().__init__(gid, cell_info, circuit_conf) self.gid = gid @@ -47,7 +47,6 @@ def connect2target(self, target_pp=None): class ACellManager(CellManagerBase): CellType = ACellType - _node_format = "fake" @staticmethod def _node_loader(circuit_conf, gidvec, stride=1, stride_offset=0, **_kw): diff --git a/tests/scientific/conftest.py b/tests/scientific/conftest.py index eb991fd6..82255319 100644 --- a/tests/scientific/conftest.py +++ b/tests/scientific/conftest.py @@ -3,8 +3,6 @@ assert os.environ.get("NEURODAMUS_NEOCORTEX_ROOT"), \ "Test requires loading a neocortex model to run" -assert os.path.isfile("/gpfs/bbp.cscs.ch/project/proj83/circuits/Bio_M/20200805/circuit.mvd3"), \ - "Circuit file not available" pytestmark = [ pytest.mark.forked,