From 67e02e3b1e6753a695f1a5b673819e6dd90091e4 Mon Sep 17 00:00:00 2001 From: Anna Grim <108307071+anna-grim@users.noreply.github.com> Date: Thu, 18 Jan 2024 10:16:02 -0800 Subject: [PATCH] build: combine whole brain graphs (#34) Co-authored-by: anna-grim --- src/deep_neurographs/graph_utils.py | 46 ++++++++++++++--------------- src/deep_neurographs/intake.py | 31 +++---------------- src/deep_neurographs/neurograph.py | 34 ++++++++------------- src/deep_neurographs/swc_utils.py | 17 +++++------ src/deep_neurographs/utils.py | 28 +++++++++++++++++- 5 files changed, 74 insertions(+), 82 deletions(-) diff --git a/src/deep_neurographs/graph_utils.py b/src/deep_neurographs/graph_utils.py index 6a69c29..f4cfe5a 100644 --- a/src/deep_neurographs/graph_utils.py +++ b/src/deep_neurographs/graph_utils.py @@ -7,17 +7,19 @@ Routines that extract the irreducible components of a graph. ---define what an irreducible is - leafs : set - Nodes with degreee 1. - junctions : set - Nodes with degree > 2. - edges : dict - Set of edges connecting nodes in leafs and junctions. The keys are - pairs of nodes connected by an edge and values are a dictionary of - attributes. ---define what a branch is +Terminology +------------ + +Leaf: a node with degree 1. + +Junction: a node with degree > 2. + +Irreducibles: the irreducibles of a graph G=(V,E) consists of (1) leaf nodes +V_l, (2) junction nodes, and (3) +junction nodes along + +Branch: the sequence of nodes between two """ @@ -58,17 +60,20 @@ def get_irreducibles(swc_dict, swc_id=None, prune=True, depth=16, smooth=True): all irreducibles from the graph of that type. """ - # Initializations + # Build dense graph dense_graph = swc_utils.to_graph(swc_dict) if prune: dense_graph = prune_short_branches(dense_graph, depth) - # Extract irreducibles + # Extract nodes leafs, junctions = get_irreducible_nodes(dense_graph, swc_dict) - assert len(leafs) > 0, "Error: swc with no leaf nodes!" - root = None + if len(leafs) == 0: + return False, None + + # Extract edges edges = dict() nbs = dict() + root = None for (i, j) in nx.dfs_edges(dense_graph, source=sample(leafs, 1)[0]): # Check if start of path is valid if root is None: @@ -84,8 +89,8 @@ def get_irreducibles(swc_dict, swc_id=None, prune=True, depth=16, smooth=True): ) else: edges[(root, j)] = attrs - nbs = append_value(nbs, root, j) - nbs = append_value(nbs, j, root) + nbs = utils.append_dict_value(nbs, root, j) + nbs = utils.append_dict_value(nbs, j, root) root = None # Output @@ -196,6 +201,7 @@ def get_leafs(graph): def __smooth_branch(swc_dict, attrs, edges, nbs, root, j): attrs["xyz"] = geometry_utils.smooth_branch(np.array(attrs["xyz"]), s=10) + attrs["radius"] = np.array(attrs["radius"]) swc_dict, edges = upd_xyz(swc_dict, attrs, edges, nbs, root, 0) swc_dict, edges = upd_xyz(swc_dict, attrs, edges, nbs, j, -1) edges[(root, j)] = attrs @@ -213,14 +219,6 @@ def upd_xyz(swc_dict, attrs, edges, nbs, i, start_or_end): return swc_dict, edges -def append_value(my_dict, key, value): - if key in my_dict.keys(): - my_dict[key].append(value) - else: - my_dict[key] = [value] - return my_dict - - def upd_branch_endpoint(edges, key, old_xyz, new_xyz): if all(edges[key]["xyz"][0] == old_xyz): edges[key]["xyz"][0] = new_xyz diff --git a/src/deep_neurographs/intake.py b/src/deep_neurographs/intake.py index 55adffc..ac3ca16 100644 --- a/src/deep_neurographs/intake.py +++ b/src/deep_neurographs/intake.py @@ -245,18 +245,6 @@ def build_neurograph( cnt, t1 = report_progress(i, n_components, chunk_size, cnt, t0, t1) i += 1 print("\n" + f"add_irreducibles(): {time() - t0} seconds") - - """ - t0 = time() - start_ids = get_start_ids(swc_dicts) - with ThreadPoolExecutor() as executor: - futures = { - executor.submit( - neurograph.add_immutables, irreducibles[key], swc_dicts[key], key, start_ids[key]): key for key in swc_dicts.keys() - } - wait(futures) - print(f" --> asynchronous - add_irreducibles(): {time() - t0} seconds") - """ return neurograph @@ -290,9 +278,10 @@ def get_irreducibles( progress_cnt = 1 for i, process in enumerate(as_completed(processes)): process_id, result = process.result() - irreducibles[process_id] = result - n_nodes += len(result["leafs"]) + len(result["junctions"]) - n_edges += len(result["edges"]) + if process_id: + irreducibles[process_id] = result + n_nodes += len(result["leafs"]) + len(result["junctions"]) + n_edges += len(result["edges"]) if i > progress_cnt * chunk_size: progress_cnt, t1 = report_progress( i, n_components, chunk_size, progress_cnt, t0, t1 @@ -302,15 +291,6 @@ def get_irreducibles( return irreducibles, n_nodes, n_edges -def get_start_ids(swc_dicts): - start_id = 0 - start_ids = dict() - for key in swc_dicts.keys(): - start_ids[key] = start_id - start_id += len(swc_dicts[key]["id"]) - return start_ids - - # -- Utils -- def get_paths(swc_dir): paths = [] @@ -320,11 +300,8 @@ def get_paths(swc_dir): def report_progress(current, total, chunk_size, cnt, t0, t1): - # Compute eta = get_eta(current, total, chunk_size, t1) runtime = get_runtime(current, total, chunk_size, t0, t1) - - # Write results utils.progress_bar(current, total, eta=eta, runtime=runtime) return cnt + 1, time() diff --git a/src/deep_neurographs/neurograph.py b/src/deep_neurographs/neurograph.py index 032fc1b..0ec0a32 100644 --- a/src/deep_neurographs/neurograph.py +++ b/src/deep_neurographs/neurograph.py @@ -80,10 +80,10 @@ def init_densegraph(self): self.densegraph = DenseGraph(self.swc_paths) # --- Add nodes or edges --- - def add_immutables(self, irreducibles, swc_id, start_id=None): + def add_immutables(self, irreducibles, swc_id): # Nodes node_ids = dict() - cur_id = start_id if start_id else len(self.nodes) + cur_id = len(self.nodes) node_ids, cur_id = self.__add_nodes( irreducibles, "leafs", node_ids, cur_id, swc_id ) @@ -92,27 +92,19 @@ def add_immutables(self, irreducibles, swc_id, start_id=None): ) # Add edges - """ - edges = irreducibles["edges"] - for i, j in edges.keys(): - # Get edge - edge = (node_ids[i], node_ids[j]) - xyz = np.array(edges[(i, j)]["xyz"]) - radii = np.array(edges[(i, j)]["radius"]) - - # Add edge - self.immutable_edges.add(frozenset(edge)) + for edge, values in irreducibles["edges"].items(): + i, j = edge + self.immutable_edges.add(frozenset((node_ids[i], node_ids[j]))) self.add_edge( - node_ids[i], node_ids[j], xyz=xyz, radius=radii, swc_id=swc_id + node_ids[i], + node_ids[j], + radius=values["radius"], + xyz=values["xyz"], + swc_id=swc_id ) - xyz_to_edge = dict((tuple(xyz), edge) for xyz in xyz) - check_xyz = set(xyz_to_edge.keys()) - collisions = check_xyz.intersection(set(self.xyz_to_edge.keys())) - if len(collisions) > 0: - for xyz in collisions: - del xyz_to_edge[xyz] - self.xyz_to_edge.update(xyz_to_edge) - """ + for xyz in values["xyz"][::2]: + self.xyz_to_edge[tuple(xyz)] = (i, j) + self.xyz_to_edge[tuple(values["xyz"][-1])] = (i, j) def __add_nodes(self, nodes, key, node_ids, cur_id, swc_id): for i in nodes[key].keys(): diff --git a/src/deep_neurographs/swc_utils.py b/src/deep_neurographs/swc_utils.py index fd62b44..8a5a0df 100644 --- a/src/deep_neurographs/swc_utils.py +++ b/src/deep_neurographs/swc_utils.py @@ -127,18 +127,17 @@ def fast_parse(contents): contents, offset = get_contents(contents) min_id = np.inf swc_dict = { - "id": np.zeros((len(contents)), dtype=int), - "radius": np.zeros((len(contents)), dtype=float), - "pid": np.zeros((len(contents)), dtype=int), - "xyz": [], + "id": np.zeros((len(contents)), dtype=np.int32), + "radius": np.zeros((len(contents)), dtype=np.float32), + "pid": np.zeros((len(contents)), dtype=np.int32), + "xyz": np.zeros((len(contents), 3), dtype=np.int32), } for i, line in enumerate(contents): parts = line.split() - xyz = read_xyz(parts[2:5], offset=offset) - swc_dict["id"][i] = int(parts[0]) - swc_dict["radius"][i] = float(parts[-2]) - swc_dict["pid"][i] = int(parts[-1]) - swc_dict["xyz"].append(xyz) + swc_dict["id"][i] = parts[0] + swc_dict["radius"][i] = parts[-2] + swc_dict["pid"][i] = parts[-1] + swc_dict["xyz"][i] = read_xyz(parts[2:5], offset=offset) # Reindex from zero min_id = np.min(swc_dict["id"]) diff --git a/src/deep_neurographs/utils.py b/src/deep_neurographs/utils.py index a1d5797..2e95906 100644 --- a/src/deep_neurographs/utils.py +++ b/src/deep_neurographs/utils.py @@ -97,6 +97,32 @@ def remove_key(my_dict, key): return my_dict +def append_dict_value(my_dict, key, value): + """ + Appends "value" to the list stored at "key". + + Parameters + ---------- + my_dict : dict + Dictionary to be queried. + key : hashable data type + Key to be query. + value : list item type + Value to append to list stored at "key". + + Returns + ------- + my_dict : dict + Updated dictionary. + + """ + if key in my_dict.keys(): + my_dict[key].append(value) + else: + my_dict[key] = [value] + return my_dict + + # --- os utils --- def mkdir(path, delete=False): """ @@ -500,7 +526,7 @@ def progress_bar(current, total, bar_length=50, eta=None, runtime=None): bar = f"[{'=' * progress}{' ' * (bar_length - progress)}]" eta = f"Time Remaining: {eta}" if eta else "" runtime = f"Estimated Total Runtime: {runtime}" if runtime else "" - print(f"\r{bar} {n_completed} | {eta} | {runtime}", end="", flush=True) + print(f"\r{bar} {n_completed} | {eta} | {runtime} ", end="", flush=True) def xor(a, b):