diff --git a/.github/configs/base.yml b/.github/configs/base.yml index db6bd522..e848fe24 100644 --- a/.github/configs/base.yml +++ b/.github/configs/base.yml @@ -68,6 +68,19 @@ modifiers: type: "EnvVar" var: "MMTK_PLAN" val: "{0}" + fail_on_oom: + type: JVMArg + val: "-XX:+CrashOnOutOfMemoryError" + preserve: + type: ProgramArg + val: "-preserve" + +plugins: + keep_stdout_stderr: + type: CopyFile + patterns: + - "scratch/stdout.log" + - "scratch/stderr.log" runtimes: jdk11-master: @@ -76,13 +89,13 @@ runtimes: home: "/home/runner/work/mmtk-openjdk/mmtk-openjdk/bundles/jdk" configs: - - "jdk11-master|ms|s|mmtk_gc-SemiSpace|tph" - - "jdk11-master|ms|s|mmtk_gc-GenCopy|tph" - - "jdk11-master|ms|s|mmtk_gc-Immix|tph" - - "jdk11-master|ms|s|mmtk_gc-GenImmix|tph" - - "jdk11-master|ms|s|mmtk_gc-StickyImmix|tph" - - "jdk11-master|ms|s|mmtk_gc-MarkSweep|tph" - - "jdk11-master|ms|s|mmtk_gc-MarkCompact|tph" + - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-SemiSpace" + - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-GenCopy" + - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-Immix" + - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-GenImmix" + - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-StickyImmix" + - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-MarkSweep" + - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-MarkCompact" benchmarks: dacapo-23.9-RC3-chopin-ci: diff --git a/.github/scripts/ci-expected-results.yml b/.github/scripts/ci-expected-results.yml new file mode 100644 index 00000000..9c533a90 --- /dev/null +++ b/.github/scripts/ci-expected-results.yml @@ -0,0 +1,309 @@ +results: + linux-x64: + fastdebug: + avrora: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + batik: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + biojava: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + cassandra: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: ignore + MarkCompact: ignore + eclipse: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: ignore + MarkCompact: pass + fop: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: ignore + MarkCompact: pass + graphchi: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + h2: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: ignore + MarkCompact: pass + h2o: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + jme: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + jython: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: ignore + MarkCompact: pass + kafka: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: ignore + luindex: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + lusearch: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: ignore + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + pmd: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + sunflow: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + tomcat: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + xalan: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: ignore + MarkSweep: pass + MarkCompact: pass + zxing: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + + release: + avrora: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + batik: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + biojava: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + cassandra: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: ignore + MarkCompact: pass + eclipse: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: ignore + MarkCompact: pass + fop: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: ignore + MarkCompact: pass + graphchi: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + h2: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: ignore + MarkCompact: pass + h2o: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: ignore + MarkCompact: pass + jme: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + jython: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: ignore + MarkCompact: pass + kafka: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + luindex: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + lusearch: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + pmd: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + sunflow: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + tomcat: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: ignore + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + xalan: + SemiSpace: ignore + GenCopy: ignore + Immix: pass + GenImmix: ignore + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass + zxing: + SemiSpace: pass + GenCopy: pass + Immix: pass + GenImmix: pass + StickyImmix: pass + MarkSweep: pass + MarkCompact: pass diff --git a/.github/scripts/ci-matrix-result-check.py b/.github/scripts/ci-matrix-result-check.py new file mode 100644 index 00000000..b029aabf --- /dev/null +++ b/.github/scripts/ci-matrix-result-check.py @@ -0,0 +1,161 @@ +import yaml +import sys +import os +import re + +if len(sys.argv) < 5: + raise ValueError("Invalid arguments") + +script_dir = os.path.dirname(os.path.abspath(__file__)); +config_path = os.path.join(script_dir, "..", "configs", "base.yml") +expected_results_path = os.path.join(script_dir, "ci-expected-results.yml") + +arch = sys.argv[1] +build = sys.argv[2] +benchmark = sys.argv[3] +log_dir = sys.argv[4] + +def read_in_plans(): + # Load the YAML file + with open(config_path, "r") as f: + data = yaml.safe_load(f) + + # Extract the values from the "configs" field + configs = data["configs"] + + # Define the dictionary to store the values + results = {} + + pattern = r"mmtk_gc-(.+?)(\||$)" + + # Loop through each property in configs + for i, prop in enumerate(configs): + # Extract the value behind "mmtk_gc-" + m = re.search(pattern, prop) + if m: + value = m.group(1) + else: + raise ValueError(f"Cannot find a plan string in {prop}") + + # Store the value in the dictionary + key = chr(97+i) + results[key] = value + + return results + +def read_in_actual_results(line, plan_dict): + # Read the input from stdin + input_string = line.strip() + + # Extract the benchmark name and discard the rest + benchmark_name = input_string.split()[0] + input_string = input_string.removeprefix(benchmark_name) + + # Extract the strings from the input, like 0abcdef or 1a.c.ef + pattern = r"(\d+[a-z\.]+)" + matches = re.findall(pattern, input_string) + + # list[0] = "abcdef", list[1] = "a.cd.f", etc + raw_results = list() + for m in matches: + print(m) + index = int(m[0]) + result = m[1:] + assert len(raw_results) == index + raw_results.append(result) + + # Format the raw results into a dict + # dict['SemiSpace'] = true/false + result_dict = {} + for s in raw_results: + # Start with a + key = 97 + for c in s: + plan = plan_dict[chr(key)] + key += 1 + success = (c != '.') + if plan in result_dict: + result_dict[plan] = result_dict[plan] and success + else: + result_dict[plan] = success + + # Rewrite True/False into pass/fail + for key in result_dict.keys(): + if result_dict[key]: + result_dict[key] = 'pass' + else: + result_dict[key] = 'fail' + + return result_dict + +def read_in_expected_results(build, benchmark): + # Load the YAML file + with open(expected_results_path, "r") as f: + data = yaml.safe_load(f) + + return data["results"][arch][build][benchmark] + +def print_log(directory, search_string): + import gzip + + # Check if the provided path is a directory + if not os.path.isdir(directory): + print(f"Error: {directory} is not a directory.") + sys.exit(1) + + # Walk through the directory + for root, dirs, files in os.walk(directory): + for file in files: + if search_string in file: + file_path = os.path.join(root, file) + # Check if the file has a .gz extension + if file_path.endswith('log.gz'): + with gzip.open(file_path, 'rt') as f: + content = f.read() + print(f"----------------------------------------------") + print(f"START: {file_path}") + print(content) + print(f"END: {file_path}") + print(f"----------------------------------------------") + +# dict['a'] = 'SemiSpace', etc +plan_dict = read_in_plans() + +actual = read_in_actual_results(sys.stdin.readline(), plan_dict) +expected = read_in_expected_results(build, benchmark) + +print("Expected:") +print(expected) +print("Actual:") +print(actual) + +print("=====") + +# Return code. If we ignore results, we may still return 0 (no error) +error_no = 0 +# All the failed plans. As long as the run failed, we print the log for the plan, even if the result is ignored. +failed_plans = [] + +for plan in expected: + if plan in actual: + if actual[plan] == 'fail': + failed_plans.append(plan) + + if expected[plan] == "ignore": + print(f"Result for {plan} is ignored") + continue + + if expected[plan] != actual[plan]: + error_no = 1 + if expected[plan] == "pass": + print(f"Expect {plan} to pass, but it failed.") + else: + print(f"Expect {plan} to fail, but it passed.") + print(f"- If we have fixed a bug and expect the benchmark to run, please update ci-expected-results.yml") + +print(f"\nPrint logs for all failed runs: {failed_plans}\n") + +for failed_plan in failed_plans: + print_log(log_dir, failed_plan) + +exit(error_no) diff --git a/.github/scripts/visualize-expected-results.py b/.github/scripts/visualize-expected-results.py new file mode 100644 index 00000000..10013311 --- /dev/null +++ b/.github/scripts/visualize-expected-results.py @@ -0,0 +1,59 @@ +import yaml +import sys + + +def convert_result_to_emoji(result): + """Convert result to corresponding emoji.""" + conversion = { + 'pass': ':white_check_mark:', + 'fail': ':x:', + 'ignore': ':question:' + } + return conversion.get(result, result) + + +def yaml_to_markdown_table(file_path): + # Load YAML data from the given file path + with open(file_path, 'r') as f: + data = yaml.safe_load(f) + + markdown_tables = "" + + # Process each platform separately + for platform_name in data['results'].keys(): + + # Dynamically extract configurations + configurations = list(data['results'][platform_name].keys()) + + # Dynamically extract plan names from the data + sample_test = list(data['results'][platform_name][configurations[0]].keys())[0] + plans = list(data['results'][platform_name][configurations[0]][sample_test].keys()) + + # Define the header for the markdown table + header = f"### {platform_name}\n\n" + header += "| Test Name |" + for plan in plans: + for config in configurations: + header += f" {plan} ({config}) |" + header += "\n|" + "-----------|" * (len(plans) * len(configurations) + 1) + "\n" + + # Extract data and construct the table content + table_content = "" + for test_name in data['results'][platform_name][configurations[0]].keys(): + row = f"| {test_name} |" + for plan in plans: + for config in configurations: + result = data['results'][platform_name][config][test_name].get(plan, '') + row += f" {convert_result_to_emoji(result)} |" + table_content += row + "\n" + + # Append the table for the current platform to the final result + markdown_tables += header + table_content + "\n\n" + + return markdown_tables + +# Load YAML from a given file path +file_path = sys.argv[1] + +markdown_tables = yaml_to_markdown_table(file_path) +print(markdown_tables) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7e1f075a..9fc5deee 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -194,4 +194,5 @@ jobs: run: | RUNNING_OUTPUT=`sed -n "s/^\(${{ matrix.benchmark }} .*\)$/\1/p" < /tmp/running.stdout` echo $RUNNING_OUTPUT - echo $RUNNING_OUTPUT | grep -vq "\." + pip3 install pyyaml + echo $RUNNING_OUTPUT | python3 .github/scripts/ci-matrix-result-check.py linux-x64 ${{ matrix.debug-level }} ${{ matrix.benchmark }} /tmp/${{ steps.extract-running-run-id.outputs.run-id }}/