diff --git a/src/BenchmarkManager.py b/src/BenchmarkManager.py index a6010cd1..fc4d5377 100644 --- a/src/BenchmarkManager.py +++ b/src/BenchmarkManager.py @@ -125,59 +125,64 @@ def run_benchmark(self, benchmark_backlog: list, repetitions: int): """ git_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", ) git_revision_number, git_uncommitted_changes = get_git_revision(git_dir) - - try: - for idx_backlog, backlog_item in enumerate(benchmark_backlog): - benchmark_records: [BenchmarkRecord] = [] - path = f"{self.store_dir}/benchmark_{idx_backlog}" - Path(path).mkdir(parents=True, exist_ok=True) - with open(f"{path}/application_config.json", 'w') as filehandler: - json.dump(backlog_item["config"], filehandler, indent=2) - for i in range(1, repetitions + 1): - logging.info(f"Running backlog item {idx_backlog + 1}/{len(benchmark_backlog)}," - f" Iteration {i}/{repetitions}:") - try: - - self.benchmark_record_template = BenchmarkRecord(idx_backlog, - datetime.today().strftime('%Y-%m-%d-%H-%M-%S'), - git_revision_number, git_uncommitted_changes, - i, repetitions) - self.application.metrics.set_module_config(backlog_item["config"]) - problem, preprocessing_time = self.application.preprocess(None, backlog_item["config"], - store_dir=path, rep_count=i) - self.application.metrics.set_preprocessing_time(preprocessing_time) - self.application.save(path, i) - - processed_input, benchmark_record = self.traverse_config(backlog_item["submodule"], problem, - path, rep_count=i) - - _, postprocessing_time = self.application.postprocess(processed_input, None, store_dir=path, - rep_count=i) - self.application.metrics.set_postprocessing_time(postprocessing_time) - self.application.metrics.validate() - benchmark_record.append_module_record_left(deepcopy(self.application.metrics)) - benchmark_records.append(benchmark_record) - - except Exception as error: - logging.exception(f"Error during benchmark run: {error}", exc_info=True) - if self.fail_fast: - raise - - for record in benchmark_records: - record.sum_up_times() - - # Wait until all MPI processes have finished and save results on rank 0 - comm.Barrier() - if comm.Get_rank() == 0: - with open(f"{path}/results.json", 'w') as filehandler: - json.dump([x.get() for x in benchmark_records], filehandler, indent=2, cls=NumpyEncoder) - - logging.info("") - logging.info(" =============== Run finished =============== ") - logging.info("") - - except KeyboardInterrupt: - logging.warning("CTRL-C detected. Still trying to create results.json.") + break_flag = False + + for idx_backlog, backlog_item in enumerate(benchmark_backlog): + benchmark_records: [BenchmarkRecord] = [] + path = f"{self.store_dir}/benchmark_{idx_backlog}" + Path(path).mkdir(parents=True, exist_ok=True) + with open(f"{path}/application_config.json", 'w') as filehandler: + json.dump(backlog_item["config"], filehandler, indent=2) + for i in range(1, repetitions + 1): + logging.info(f"Running backlog item {idx_backlog + 1}/{len(benchmark_backlog)}," + f" Iteration {i}/{repetitions}:") + try: + + self.benchmark_record_template = BenchmarkRecord(idx_backlog, + datetime.today().strftime('%Y-%m-%d-%H-%M-%S'), + git_revision_number, git_uncommitted_changes, + i, repetitions) + self.application.metrics.set_module_config(backlog_item["config"]) + problem, preprocessing_time = self.application.preprocess(None, backlog_item["config"], + store_dir=path, rep_count=i) + self.application.metrics.set_preprocessing_time(preprocessing_time) + self.application.save(path, i) + + processed_input, benchmark_record = self.traverse_config(backlog_item["submodule"], problem, + path, rep_count=i) + + _, postprocessing_time = self.application.postprocess(processed_input, None, store_dir=path, + rep_count=i) + self.application.metrics.set_postprocessing_time(postprocessing_time) + self.application.metrics.validate() + benchmark_record.append_module_record_left(deepcopy(self.application.metrics)) + benchmark_records.append(benchmark_record) + + except KeyboardInterrupt: + logging.warning("CTRL-C detected during run_benchmark. Still trying to create results.json.") + break_flag = True + break + + except Exception as error: + logging.exception(f"Error during benchmark run: {error}", exc_info=True) + if self.fail_fast: + raise + + for record in benchmark_records: + record.sum_up_times() + + # Wait until all MPI processes have finished and save results on rank 0 + comm.Barrier() + if comm.Get_rank() == 0: + with open(f"{path}/results.json", 'w') as filehandler: + json.dump([x.get() for x in benchmark_records], filehandler, indent=2, cls=NumpyEncoder) + + logging.info("") + logging.info(" =============== Run finished =============== ") + logging.info("") + + if break_flag: + break def traverse_config(self, module: dict, input_data: any, path: str, rep_count: int) -> (any, BenchmarkRecord): """