diff --git a/src/deep_neurographs/intake.py b/src/deep_neurographs/intake.py index 87581ce..80fc569 100644 --- a/src/deep_neurographs/intake.py +++ b/src/deep_neurographs/intake.py @@ -147,7 +147,7 @@ def build_neurograph_from_gcs_zips( t0 = time() swc_dicts = download_gcs_zips(bucket_name, cloud_path, min_size) t, unit = utils.time_writer(time() - t0) - print(f"\ndownload_gcs_zips(): {t} {unit}") + print(f"\ndownload_gcs_zips(): {t} {unit} \n") # Build neurograph t0 = time() @@ -190,17 +190,19 @@ def download_gcs_zips(bucket_name, cloud_path, min_size): storage_client = storage.Client() bucket = storage_client.bucket(bucket_name) zip_paths = utils.list_gcs_filenames(bucket, cloud_path, ".zip") - chunk_size = int(len(zip_paths) * 0.1) + chunk_size = int(len(zip_paths) * 0.05) print(f"# zip files: {len(zip_paths)} \n") # Parse cnt = 1 t0 = time() + t1 = time() swc_dicts = dict() + print("Process swc files...") for i, path in enumerate(zip_paths): swc_dicts.update(process_gsc_zip(bucket, path, min_size=min_size)) if i > cnt * chunk_size: - cnt, t0 = report_progress(i, len(zip_paths), chunk_size, cnt, t0) + cnt, t1 = report_progress(i, len(zip_paths), chunk_size, cnt, t0, t1) return swc_dicts @@ -223,10 +225,14 @@ def build_neurograph( smooth=SMOOTH, ): # Extract irreducibles - t0 = time() + cnt = 1 + chunk_size = int(len(swc_dicts) * 0.05) irreducibles = dict() + print("Build graph...") + print("# connected components:", len(swc_dicts)) with ProcessPoolExecutor() as executor: # Assign Processes + t0 = time() processes = [None] * len(swc_dicts) for i, key in enumerate(swc_dicts.keys()): processes[i] = executor.submit( @@ -237,13 +243,23 @@ def build_neurograph( prune_depth, smooth, ) - print("assigned threads") - for process in as_completed(processes): + t, unit = utils.time_writer(time() - t0) + print(f"assigned_threads(): {t} {unit}") + + # Store results + t0 = time() + t1 = time() + for i, process in enumerate(as_completed(processes)): process_id, result = process.result() irreducibles[process_id] = result - print(f" --> get_irreducibles(): {time() - t0} seconds") + if i > cnt * chunk_size: + cnt, t1 = report_progress(i, len(swc_dicts), chunk_size, cnt, t0, t1) + t, unit = utils.time_writer(time() - t0) + print("") + print(f"get_irreducibles(): {t} {unit}") + stop - # Build neurograph + # Build neurograph neurograph = NeuroGraph(bbox=bbox, img_path=img_path, swc_paths=swc_paths) start_ids = get_start_ids(swc_dicts) @@ -252,7 +268,8 @@ def build_neurograph( neurograph.add_immutables( irreducibles[key], swc_dicts[key], key ) - print(f" --> synchronous - add_irreducibles(): {time() - t0} seconds") + del swc_dicts[key] + print(f"add_irreducibles(): {time() - t0} seconds") """ t0 = time() @@ -283,16 +300,26 @@ def get_paths(swc_dir): return paths -def report_progress(current, total, chunk_size, cnt, t0): - eta = get_eta(current, total, chunk_size, t0) - utils.progress_bar(current, total, eta=eta) +def report_progress(current, total, chunk_size, cnt, t0, t1): + # Compute + eta = get_eta(current, total, chunk_size, t1) + runtime = get_runtime(current, total, chunk_size, t0, t1) + + # Write results + utils.progress_bar(current, total, eta=eta, runtime=runtime) return cnt + 1, time() -def get_eta(current, total, chunk_size, t0): +def get_eta(current, total, chunk_size, t0, return_str=True): chunk_runtime = time() - t0 remaining = total - current eta = remaining * (chunk_runtime / chunk_size) t, unit = utils.time_writer(eta) + return f"{round(t, 4)} {unit}" if return_str else eta + + +def get_runtime(current, total, chunk_size, t0, t1): + eta = get_eta(current, total, chunk_size, t1, return_str=False) + total_runtime = time() - t0 + eta + t, unit = utils.time_writer(total_runtime) return f"{round(t, 4)} {unit}" - \ No newline at end of file diff --git a/src/deep_neurographs/swc_utils.py b/src/deep_neurographs/swc_utils.py index f468229..40594ca 100644 --- a/src/deep_neurographs/swc_utils.py +++ b/src/deep_neurographs/swc_utils.py @@ -124,11 +124,10 @@ def fast_parse(contents): """ contents, offset = get_contents(contents) - dtype = np.int16 if len(contents) < 2 ** 16 else np.int32 swc_dict = { - "id": np.zeros((len(contents)), dtype=dtype), - "radius": np.zeros((len(contents)), dtype=np.float16), - "pid": np.zeros((len(contents)), dtype=dtype), + "id": np.zeros((len(contents)), dtype=int), + "radius": np.zeros((len(contents)), dtype=float), + "pid": np.zeros((len(contents)), dtype=int), "xyz": [] } @@ -203,7 +202,7 @@ def read_xyz(xyz, offset=[0, 0, 0]): The (x,y,z) coordinates from an swc file. """ - return tuple([np.float32(xyz[i]) + offset[i] for i in range(3)]) + return tuple([float(xyz[i]) + offset[i] for i in range(3)]) def write(path, contents): diff --git a/src/deep_neurographs/utils.py b/src/deep_neurographs/utils.py index 78a64e9..2d7b238 100644 --- a/src/deep_neurographs/utils.py +++ b/src/deep_neurographs/utils.py @@ -489,12 +489,13 @@ def time_writer(t, unit="seconds"): return t, unit -def progress_bar(current, total, bar_length=50, eta=None): +def progress_bar(current, total, bar_length=50, eta=None, runtime=None): progress = int(current / total * bar_length) n_completed = f"Completed: {current}/{total}" bar = f"[{'=' * progress}{' ' * (bar_length - progress)}]" eta = f"Time Remaining: {eta}" if eta else "" - print(f"\r{bar} {n_completed} | {eta}", end="", flush=True) + runtime = f"Estimated Total Runtime: {runtime}" if runtime else "" + print(f"\r{bar} {n_completed} | {eta} | {runtime}", end="", flush=True) def xor(a, b):