From 19a4e02926f1f7d6174820f48a2c2e02fd661aa7 Mon Sep 17 00:00:00 2001 From: foldr Date: Tue, 29 Aug 2023 11:45:44 +0200 Subject: [PATCH] dani/VAN 517 circom llvm fails to compile webb tools circuits (#51) * Remove unnecessary logging in release and remove forgotten debug prints * Improve benchmarking script --- scripts/benchmark.py | 125 +++++++++++++++++++++++++++---------------- 1 file changed, 79 insertions(+), 46 deletions(-) diff --git a/scripts/benchmark.py b/scripts/benchmark.py index 82b148ce4..972c60660 100755 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -10,6 +10,7 @@ import os.path import re import subprocess +import tempfile import time from pathlib import Path from tempfile import TemporaryDirectory @@ -27,26 +28,48 @@ def setup_dirs(src: str, out: str) -> Tuple[str, str, TemporaryDirectory]: return src, out, tmp -def check_is_missing_feature(message: str) -> Match[str] | None: - check = re.compile(r"thread 'main' panicked at 'not yet implemented', (.*):(.*):(.*)") - return check.match(message) +def check_is_missing_feature(fd) -> Match[str] | None: + fd.seek(0) + for line in fd: + check = re.compile(r"thread 'main' panicked at 'not yet implemented', (.*):(.*):(.*)") + match = check.match(line.decode("utf-8")) + if match is not None: + return match + return -def check_is_other_panic(message: str) -> Match[str] | None: - check = re.compile(r"thread 'main' panicked at (.*), (.*):(.*):(.*)") - return check.match(message) +def check_is_other_panic(fd) -> Match[str] | None: + fd.seek(0) + for line in fd: + check = re.compile(r"thread 'main' panicked at (.*), (.*):(.*):(.*)") + match = check.match(line.decode("utf-8")) + if match is not None: + return match + return -def is_circom_error(message: str) -> bool: - return re.compile(r"^error\[.*\]:").match(message) is not None +def is_circom_error(fd) -> bool: + fd.seek(0) + for line in fd: + if re.compile(r"^error\[.*\]:").match(line.decode("utf-8")) is not None: + return True + return False -def is_llvm_validation_error(message: str) -> bool: - return "LLVM Module verification failed" in message +def is_llvm_validation_error(fd) -> bool: + fd.seek(0) + for line in fd: + if "LLVM Module verification failed" in line.decode("utf-8"): + return True + return False -def non_constant_id(message: str) -> bool: - return "is_constant_int()" in message +def non_constant_id(fd) -> bool: + fd.seek(0) + for line in fd: + if "is_constant_int()" in line.decode("utf-8"): + return True + return False class TimedOutExecution: @@ -73,12 +96,13 @@ def extract_number_templates(message) -> int: class Report: - def __init__(self, src: str, cmd: List[str], execution: Union[subprocess.CompletedProcess | TimedOutExecution], run_time: float): + def __init__(self, src: str, cmd: List[str], execution: Union[subprocess.CompletedProcess | TimedOutExecution], run_time: float, stderr): self.src = src self.cmd = cmd self.execution = execution self.run_time = run_time self.test_id = None + self._stderr = stderr @property def successful(self): @@ -107,14 +131,15 @@ def has_panic(self): @property def stderr(self): - if self.execution.stderr: - return escape_ansi(self.execution.stderr.decode("utf-8")) - return "" + #if self.execution.stderr: + # return escape_ansi(self.execution.stderr.decode("utf-8")) + #return "" + return self._stderr @property def stdout(self): - if self.execution.stdout: - return escape_ansi(self.execution.stdout.decode("utf-8")) + #if self.execution.stdout: + # return escape_ansi(self.execution.stdout.decode("utf-8")) return "" @property @@ -141,14 +166,12 @@ def to_dict(self) -> dict: 'cmd': self.cmd, 'return_code': self.execution.returncode, 'successful': self.successful, - 'stdout': self.stdout, - 'stderr': self.stderr, 'run_time': self.run_time, 'missing_feature': self.missing_feature, 'error_class': self.error_class, 'panicked': self.has_panic, 'test_id': self.test_id, - 'template_instances': extract_number_templates(self.stdout) + #'template_instances': extract_number_templates(self.stdout) } @@ -157,9 +180,20 @@ def escape_ansi(line: str) -> str: return ansi_escape.sub('', line) +def tail(f): + execution = subprocess.run(['tail', f.name], capture_output=True) + if execution.returncode == 0: + if execution.stdout: + print(escape_ansi(execution.stdout.decode("utf-8"))) + else: + print(execution.stderr.decode("utf-8")) + exit(1) + + def run_test(src: str, circom: str, debug: bool, cwd: str, libs_path: Optional[str], timeout: int) -> Report: src = os.path.realpath(src) tmp = TemporaryDirectory() + stderr = tempfile.NamedTemporaryFile() cmd = [ circom, '--llvm', @@ -169,29 +203,28 @@ def run_test(src: str, circom: str, debug: bool, cwd: str, libs_path: Optional[s if libs_path: cmd.extend(['-l', libs_path]) cmd.append(src) - if debug: - print("Source file:", src) - print("CMD:", ' '.join(cmd)) + print("Source file:", src) try: start = time.time() - execution = subprocess.run(cmd, capture_output=True, cwd=cwd, timeout=timeout) + execution = subprocess.run(cmd, stderr=stderr, stdout=subprocess.DEVNULL, cwd=cwd, timeout=timeout) end = time.time() + if execution.returncode == 0: + print("Success!") + else: + print("Failure!") + print("CMD:", ' '.join(cmd)) if debug: - if execution.returncode == 0: - print("Success!") - else: - print("Failure!") # if execution.stdout: # print("Circom stdout:\n", escape_ansi(execution.stdout.decode("utf-8"))) - if execution.stderr: - print("Circom stderr:\n", execution.stderr.decode("utf-8")) + print("Circom stderr:\n") + tail(stderr) print("Execution time in seconds:", end - start) - return Report(src, cmd, execution, end - start) + return Report(src, cmd, execution, end - start, stderr) except subprocess.TimeoutExpired as e: - if debug: - print("Test timed out!") - return Report(src, cmd, TimedOutExecution(e), timeout) + print("Test timed out!") + print("CMD:", ' '.join(cmd)) + return Report(src, cmd, TimedOutExecution(e), timeout, stderr) def check_link_libraries(data: dict) -> bool: @@ -207,8 +240,7 @@ def run_setup(data: dict): os.system(cmd) -def evaluate_test(test_path: str, circom: str, debug: bool, libs_path: str, timeout: int) -> List[Report]: - reports = [] +def evaluate_test(test_path: str, circom: str, debug: bool, libs_path: str, timeout: int): with open(test_path) as f: test_data = json.load(f) test_cwd = Path(test_path).parent @@ -222,9 +254,11 @@ def evaluate_test(test_path: str, circom: str, debug: bool, libs_path: str, time main_circom_file = test_cwd.joinpath(test['main']) report = run_test(str(main_circom_file), circom, debug, str(test_cwd), libs_path, timeout) report.test_id = f"{test_data['id']}_{n}" - reports.append(report) - return reports + yield report +def get_reports(tests, circom, debug, src, timeout): + for test in tests: + yield from evaluate_test(test, circom, debug, str(src.joinpath("tests/libs")), timeout) @click.command() @click.option('--src', help='Path where the benchmark is located.') @@ -233,28 +267,27 @@ def evaluate_test(test_path: str, circom: str, debug: bool, libs_path: str, time @click.option('--debug', help="Print debug information", is_flag=True) @click.option('--timeout', help="Timeout for stopping the compilation", default=600) def main(src, out, circom, debug, timeout): - reports = [] src = Path(src) tests = glob.glob(str(src.joinpath(GLOB)), recursive=True) - for test in tests: - reports.extend(evaluate_test(test, circom, debug, str(src.joinpath("tests/libs")), timeout)) + reports = get_reports(tests, circom, debug, src, timeout) with open(out, 'w') as out_csv: - print('test_id,successful,error_class,message,file,line,column,run_time,template_instances', file=out_csv) + print('test_id,successful,error_class,message,file,line,column,run_time', file=out_csv) for report in reports: report = report.to_dict() if report['successful']: print(report['test_id'], report['successful'], '', '', '', '', '', report['run_time'], - report['template_instances'], sep=',', file=out_csv) + sep=',', file=out_csv) else: if report['panicked']: print(report['test_id'], report['successful'], report['error_class'], f"\"{report['panicked']['message']}\"", report['panicked']['file'], report['panicked']['line'], report['panicked']['column'], - report['run_time'], report['template_instances'], sep=',', file=out_csv) + report['run_time'], sep=',', file=out_csv) else: print(report['test_id'], report['successful'], report['error_class'], '', '', '', '', - report['run_time'], report['template_instances'], sep=',', file=out_csv) + report['run_time'], sep=',', file=out_csv) + out_csv.flush() if __name__ == "__main__":