Check results for new CI, allow some benchmarks to fail (#211)

This PR adds a yaml file that records our expected results for DaCapo tests. It also adds a script to check the results for each CI run. We should generally expect a CI run to succeed (which means the results are expected) once this PR is merged. However, as we may have random failures that we haven't noticed, it is possible a CI run still fails. In such a case, we should update the `ci-expected-results.yml`, create an Github issue, and update #246.
mmtk · Sep 20, 2023 · 0e04c17 · 0e04c17
1 parent 7a40312
commit 0e04c17
Show file tree

Hide file tree

Showing 5 changed files with 551 additions and 8 deletions.
diff --git a/.github/configs/base.yml b/.github/configs/base.yml
@@ -68,6 +68,19 @@ modifiers:
     type: "EnvVar"
     var: "MMTK_PLAN"
     val: "{0}"
+  fail_on_oom:
+    type: JVMArg
+    val: "-XX:+CrashOnOutOfMemoryError"
+  preserve:
+    type: ProgramArg
+    val: "-preserve"
+
+plugins:
+  keep_stdout_stderr:
+    type: CopyFile
+    patterns:
+      - "scratch/stdout.log"
+      - "scratch/stderr.log"
 
 runtimes:
   jdk11-master:
@@ -76,13 +89,13 @@ runtimes:
     home: "/home/runner/work/mmtk-openjdk/mmtk-openjdk/bundles/jdk"
 
 configs:
-  - "jdk11-master|ms|s|mmtk_gc-SemiSpace|tph"
-  - "jdk11-master|ms|s|mmtk_gc-GenCopy|tph"
-  - "jdk11-master|ms|s|mmtk_gc-Immix|tph"
-  - "jdk11-master|ms|s|mmtk_gc-GenImmix|tph"
-  - "jdk11-master|ms|s|mmtk_gc-StickyImmix|tph"
-  - "jdk11-master|ms|s|mmtk_gc-MarkSweep|tph"
-  - "jdk11-master|ms|s|mmtk_gc-MarkCompact|tph"
+  - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-SemiSpace"
+  - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-GenCopy"
+  - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-Immix"
+  - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-GenImmix"
+  - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-StickyImmix"
+  - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-MarkSweep"
+  - "jdk11-master|ms|s|fail_on_oom|tph|preserve|mmtk_gc-MarkCompact"
 
 benchmarks:
   dacapo-23.9-RC3-chopin-ci:
diff --git a/.github/scripts/ci-expected-results.yml b/.github/scripts/ci-expected-results.yml
@@ -0,0 +1,309 @@
+results:
+  linux-x64:
+    fastdebug:
+      avrora:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      batik:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      biojava:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      cassandra:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: ignore
+        MarkCompact: ignore
+      eclipse:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: ignore
+        MarkCompact: pass
+      fop:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: ignore
+        MarkCompact: pass
+      graphchi:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      h2:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: ignore
+        MarkCompact: pass
+      h2o:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      jme:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      jython:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: ignore
+        MarkCompact: pass
+      kafka:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: ignore
+      luindex:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      lusearch:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: ignore
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      pmd:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      sunflow:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      tomcat:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      xalan:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: ignore
+        MarkSweep: pass
+        MarkCompact: pass
+      zxing:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+
+    release:
+      avrora:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      batik:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      biojava:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      cassandra:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: ignore
+        MarkCompact: pass
+      eclipse:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: ignore
+        MarkCompact: pass
+      fop:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: ignore
+        MarkCompact: pass
+      graphchi:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      h2:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: ignore
+        MarkCompact: pass
+      h2o:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: ignore
+        MarkCompact: pass
+      jme:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      jython:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: ignore
+        MarkCompact: pass
+      kafka:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      luindex:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      lusearch:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      pmd:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      sunflow:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      tomcat:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: ignore
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      xalan:
+        SemiSpace: ignore
+        GenCopy: ignore
+        Immix: pass
+        GenImmix: ignore
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass
+      zxing:
+        SemiSpace: pass
+        GenCopy: pass
+        Immix: pass
+        GenImmix: pass
+        StickyImmix: pass
+        MarkSweep: pass
+        MarkCompact: pass