Skip to content

Commit

Permalink
fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@arnaudon
arnaudon committed Dec 5, 2024
1 parent 9b8952a commit 9b290ef
Show file tree
Hide file tree
Showing 3 changed files with 2 additions and 336 deletions.
2 changes: 1 addition & 1 deletion region_grower/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ def generate_distributions(
)
@click.option(
"--synthesize-axons",
help="Display the versions of all the accessible modules in a logger entry",
help="Use to synthesize axons or not",
is_flag=True,
default=False,
)
Expand Down
335 changes: 1 addition & 334 deletions tests/test_synthesize_morphologies.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,31 +11,22 @@
#

# pylint: disable=missing-function-docstring
import json
import logging
import os
import shutil
import sys
from copy import deepcopy
from itertools import combinations
from pathlib import Path
from uuid import uuid4

import attr
import dask
import jsonschema
import neurots
import pandas as pd
import pytest
import yaml
from morph_tool.utils import iter_morphology_files
from morphio import Morphology
from numpy.testing import assert_allclose
from voxcell import CellCollection
from voxcell import RegionMap

from region_grower import RegionGrowerError
from region_grower.synthesize_morphologies import RegionMapper
from region_grower.synthesize_morphologies import SynthesizeMorphologies

DATA = Path(__file__).parent / "data"
Expand Down Expand Up @@ -476,329 +467,5 @@ def run_with_mpi():
shutil.rmtree(tmp_folder)


def run_with_mpi_boundary():
"""Test morphology synthesis with MPI."""
# pylint: disable=import-outside-toplevel, too-many-locals, import-error
from data_factories import generate_axon_morph_tsv
from data_factories import generate_cell_collection
from data_factories import generate_cells_df
from data_factories import generate_input_cells
from data_factories import generate_mesh
from data_factories import generate_region_structure_boundary
from data_factories import generate_small_O1
from data_factories import input_cells_path
from mpi4py import MPI

from region_grower.utils import setup_logger

COMM = MPI.COMM_WORLD # pylint: disable=c-extension-no-member
rank = COMM.Get_rank()
MASTER_RANK = 0
is_master = rank == MASTER_RANK

tmp_folder = Path("/tmp/test-run-synthesis_" + str(uuid4()))
tmp_folder = COMM.bcast(tmp_folder, root=MASTER_RANK)
input_cells = input_cells_path(tmp_folder)
small_O1_path = str(tmp_folder / "atlas")
region_structure_path = str(tmp_folder / "region_structure_boundary.yaml")
args = create_args(
True,
tmp_folder,
input_cells,
small_O1_path,
tmp_folder / "axon_morphs.tsv",
"apical_NRN_sections.yaml",
min_depth=25,
region_structure=region_structure_path,
)

setup_logger("info")
logging.getLogger("distributed").setLevel(logging.ERROR)

if is_master:
tmp_folder.mkdir(exist_ok=True)
print(f"============= #{rank}: Create data")
cells_raw_data = generate_cells_df()
cell_collection = generate_cell_collection(cells_raw_data)
generate_input_cells(cell_collection, tmp_folder)
generate_small_O1(small_O1_path)
atlas = {"atlas": small_O1_path, "structure": DATA / "region_structure.yaml"}
generate_mesh(atlas, tmp_folder / "mesh.obj")
region_structure_path = generate_region_structure_boundary(
DATA / "region_structure.yaml", region_structure_path, str(tmp_folder / "mesh.obj")
)

generate_axon_morph_tsv(tmp_folder)
for dest in range(1, COMM.Get_size()):
req = COMM.isend("done", dest=dest)
else:
req = COMM.irecv(source=0)
req.wait()

synthesizer = SynthesizeMorphologies(**args)
try:
print(f"============= #{rank}: Start synthesize")
synthesizer.synthesize()

# Check results
print(f"============= #{rank}: Checking results")
expected_size = 12
assert len(list(iter_morphology_files(tmp_folder))) == expected_size
check_yaml(DATA / "apical_boundary.yaml", args["out_apical"])
finally:
# Clean the directory
print(f"============= #{rank}: Cleaning")
shutil.rmtree(tmp_folder)


def test_verify(cell_collection, tmd_distributions, tmd_parameters, small_O1_path):
"""Test the `verify` step."""
mtype = "L2_TPC:A"

@attr.s(auto_attribs=True)
class Data:
"""Container to mimic SynthesizeMorphologies class."""

tmd_distributions: dict
tmd_parameters: dict
cells: CellCollection
region_mapper: dict

region_mapper = RegionMapper(
["test"], RegionMap.load_json(Path(small_O1_path) / "hierarchy.json")
)
data = Data(
tmd_distributions=tmd_distributions,
tmd_parameters=tmd_parameters,
cells=cell_collection,
region_mapper=region_mapper,
)
SynthesizeMorphologies.verify(data)

cell_collection.properties.loc[0, "mtype"] = "UNKNOWN_MTYPE"
data = Data(
tmd_distributions=tmd_distributions,
tmd_parameters=tmd_parameters,
cells=cell_collection,
region_mapper=region_mapper,
)
with pytest.raises(
RegionGrowerError,
match="Missing distributions for mtype 'UNKNOWN_MTYPE' in region 'default'",
):
SynthesizeMorphologies.verify(data)

cell_collection.properties.loc[0, "mtype"] = "L2_TPC:A"

failing_params = deepcopy(tmd_parameters)
del failing_params["default"][mtype]
data = Data(
tmd_distributions=tmd_distributions,
tmd_parameters=failing_params,
cells=cell_collection,
region_mapper=region_mapper,
)
with pytest.raises(
RegionGrowerError, match="Missing parameters for mtype 'L2_TPC:A' in region 'default'"
):
SynthesizeMorphologies.verify(data)

failing_params = deepcopy(tmd_parameters)
del failing_params["default"][mtype]["origin"]
data = Data(
tmd_distributions=tmd_distributions,
tmd_parameters=failing_params,
cells=cell_collection,
region_mapper=region_mapper,
)
with pytest.raises(neurots.validator.ValidationError, match=r"'origin' is a required property"):
SynthesizeMorphologies.verify(data)

# Fail when missing attributes
attributes = ["layer", "fraction", "slope", "intercept"]
good_params = deepcopy(tmd_parameters)
good_params["default"][mtype]["context_constraints"] = {
"apical_dendrite": {
"hard_limit_min": {
"layer": 1,
"fraction": 0.1,
},
"extent_to_target": {
"slope": 0.5,
"intercept": 1,
"layer": 1,
"fraction": 0.5,
},
"hard_limit_max": {
"layer": 1,
"fraction": 0.9,
},
}
}
data = Data(
tmd_distributions=tmd_distributions,
tmd_parameters=good_params,
cells=cell_collection,
region_mapper=region_mapper,
)
SynthesizeMorphologies.verify(data)
for i in range(1, 5):
for missing_attributes in combinations(attributes, i):
failing_params = deepcopy(good_params["default"][mtype])
for att in missing_attributes:
del failing_params["context_constraints"]["apical_dendrite"]["extent_to_target"][
att
]
tmd_parameters["default"][mtype] = failing_params
data = Data(
tmd_distributions=tmd_distributions,
tmd_parameters=tmd_parameters,
cells=cell_collection,
region_mapper=region_mapper,
)
with pytest.raises(
jsonschema.exceptions.ValidationError, match="is a required property"
):
SynthesizeMorphologies.verify(data)


def test_check_axon_morphology(caplog):
"""Test the _check_axon_morphology() function."""
# pylint: disable=protected-access
# Test with no missing name
caplog.set_level(logging.WARNING)
caplog.clear()
df = pd.DataFrame({"axon_name": ["a", "b", "c"], "other_col": [1, 2, 3]})

assert SynthesizeMorphologies._check_axon_morphology(df) is None
assert caplog.record_tuples == []

# Test with no missing names
caplog.clear()
df = pd.DataFrame({"axon_name": ["a", None, "c"], "other_col": [1, 2, 3]})

assert SynthesizeMorphologies._check_axon_morphology(df).tolist() == [0, 2]
assert caplog.record_tuples == [
(
"region_grower.synthesize_morphologies",
30,
"The following gids were not found in the axon morphology list: [1]",
),
]


def test_RegionMapper(small_O1_path):
region_mapper = RegionMapper(
["O0", "UNKNOWN"], RegionMap.load_json(Path(small_O1_path) / "hierarchy.json")
)
# pylint: disable=protected-access
assert region_mapper.mapper == {
"mc0_Column": "O0",
"mc0;6": "O0",
"mc0;5": "O0",
"mc0;4": "O0",
"mc0;3": "O0",
"mc0;2": "O0",
"mc0;1": "O0",
"O0": "O0",
}
assert region_mapper.inverse_mapper == {
"O0": set(["mc0_Column", "mc0;1", "mc0;2", "mc0;3", "mc0;4", "mc0;5", "mc0;6", "O0"]),
"default": set(),
}

assert region_mapper["O0"] == "O0"
assert region_mapper["mc0;1"] == "O0"
assert region_mapper["OTHER"] == "default"


def test_inconsistent_params(
tmpdir,
small_O1_path,
input_cells,
axon_morph_tsv,
): # pylint: disable=unused-argument
"""Test morphology synthesis but skip write step."""
min_depth = 25
tmp_folder = Path(tmpdir)

args = create_args(
False,
tmp_folder,
input_cells,
small_O1_path,
axon_morph_tsv,
"apical_NRN_sections.yaml",
min_depth,
DATA / "region_structure.yaml",
)
args["out_morph_ext"] = ["h5"]

with pytest.raises(
ValueError,
match=(
r"""The 'out_morph_ext' parameter must contain one of \["asc", "swc"\] when """
r"""'with_NRN_sections' is set to True \(current value is \['h5'\]\)\."""
),
):
SynthesizeMorphologies(**args)


def test_inconsistent_context(
tmpdir,
small_O1_path,
input_cells,
axon_morph_tsv,
caplog,
): # pylint: disable=unused-argument
"""Test morphology synthesis with inconsistent context constraints."""
min_depth = 25
tmp_folder = Path(tmpdir)

args = create_args(
False,
tmp_folder,
input_cells,
small_O1_path,
axon_morph_tsv,
"apical_NRN_sections.yaml",
min_depth,
DATA / "region_structure.yaml",
)
args["nb_processes"] = 0

with args["tmd_parameters"].open("r", encoding="utf-8") as f:
tmd_parameters = json.load(f)

tmd_parameters["default"]["L2_TPC:A"]["context_constraints"] = {
"apical_dendrite": {
"extent_to_target": {
"slope": 0.5,
"intercept": 1,
"layer": 1,
"fraction": 0.5,
}
}
}

args["tmd_parameters"] = tmp_folder / "tmd_parameters.json"
with args["tmd_parameters"].open("w", encoding="utf-8") as f:
json.dump(tmd_parameters, f)

synthesizer = SynthesizeMorphologies(**args)
caplog.set_level(logging.WARNING)
caplog.clear()
synthesizer.synthesize()
assert caplog.record_tuples[0] == (
"region_grower.synthesize_morphologies",
30,
"The morphologies with the following region/mtype couples have inconsistent context and "
"constraints: [('default', 'L2_TPC:A')]",
)


if __name__ == "__main__": # pragma: no cover
if sys.argv[-1] == "boundary":
run_with_mpi_boundary()
else:
run_with_mpi()
run_with_mpi()
1 change: 0 additions & 1 deletion tox.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ allowlist_externals =
mpiexec
commands =
mpiexec -n 4 {envbindir}/python tests/test_synthesize_morphologies.py
mpiexec -n 4 {envbindir}/python tests/test_synthesize_morphologies.py boundary

[testenv:coverage]
skip_install = true
Expand Down

0 comments on commit 9b290ef

Please sign in to comment.