diff --git a/tests/src/test/java/org/mozilla/javascript/tests/Test262SuiteTest.java b/tests/src/test/java/org/mozilla/javascript/tests/Test262SuiteTest.java index a0f27f5945..a08f64f8fe 100644 --- a/tests/src/test/java/org/mozilla/javascript/tests/Test262SuiteTest.java +++ b/tests/src/test/java/org/mozilla/javascript/tests/Test262SuiteTest.java @@ -71,8 +71,6 @@ public class Test262SuiteTest { /** The test must be executed just once--in non-strict mode, only. */ private static final String FLAG_NO_STRICT = "noStrict"; - static final int[] OPT_LEVELS; - private static final File testDir = new File("test262/test"); private static final String testHarnessDir = "test262/harness/"; private static final String testProperties; @@ -132,35 +130,11 @@ public class Test262SuiteTest { includeUnsupported = updateProps.isEmpty() || updateProps.indexOf("unsupported") != -1; } - - if (getOverriddenLevel() != null) { - System.out.println( - "Ignoring custom optLevels because the updateTest262Properties param is set"); - } - - OPT_LEVELS = Utils.DEFAULT_OPT_LEVELS; } else { updateTest262Properties = rollUpEnabled = statsEnabled = includeUnsupported = false; - - // Reduce the number of tests that we run by a factor of three... - String overriddenLevel = getOverriddenLevel(); - if (overriddenLevel != null) { - OPT_LEVELS = new int[] {Integer.parseInt(overriddenLevel)}; - } else { - OPT_LEVELS = Utils.DEFAULT_OPT_LEVELS; - } } } - private static String getOverriddenLevel() { - String optLevel = System.getProperty("TEST_OPTLEVEL"); - - if (optLevel == null || optLevel.isEmpty()) { - optLevel = System.getenv("TEST_262_OPTLEVEL"); - } - return optLevel; - } - @BeforeAll public static void setUpClass() { CTX_FACTORY.setLanguageVersion(Context.VERSION_ES6); @@ -279,7 +253,7 @@ public static void tearDownClass() { } if (!testFile.isDirectory()) { - testResult = tt.getResult(OPT_LEVELS, testCases[j]); + testResult = tt.getResult(testCases[j]); if (testResult == null) { // At least one passing test in currentParent directory, so prevent @@ -420,7 +394,9 @@ public static void tearDownClass() { */ private static final Pattern LINE_SPLITTER = Pattern.compile( - "(~|(?:\\s*)(?:!|#)(?:\\s*)|\\s+)?(\\S+)(?:[^\\S\\r\\n]+(?:strict|non-strict|non-interpreted|\\d+/\\d+ \\(\\d+(?:\\.\\d+)?%%\\)|\\{(?:non-strict|strict|unsupported): \\[.*\\],?\\}))?[^\\S\\r\\n]*(.*)"); + "(~|(?:\\s*)(?:!|#)(?:\\s*)|\\s+)?(\\S+)(?:[^\\S\\r\\n]+" + + "(?:strict|non-strict|compiled-strict|compiled-non-strict|interpreted-strict|interpreted-non-strict|compiled|interpreted|" + + "\\d+/\\d+ \\(\\d+(?:\\.\\d+)?%%\\)|\\{(?:non-strict|strict|unsupported): \\[.*\\],?\\}))?[^\\S\\r\\n]*(.*)"); /** * @see https://github.com/tc39/test262/blob/main/INTERPRETING.md#host-defined-functions @@ -1051,7 +1027,7 @@ public void passes(TestMode mode, boolean useStrict) { modes.remove(makeKey(mode, useStrict)); } - public String getResult(int[] optLevels, Test262Case tc) { + public String getResult(Test262Case tc) { // success on all optLevels in both strict and non-strict mode if (modes.isEmpty()) { return null; @@ -1083,8 +1059,33 @@ public String getResult(int[] optLevels, Test262Case tc) { return ""; } - // mix of mode and optLevel successes and failures - return '{' + String.join(",", new ArrayList<>(modes)) + '}'; + // simplify the output for some cases + ArrayList res = new ArrayList<>(modes); + if (res.contains("compiled-non-strict") && res.contains("interpreted-non-strict")) { + res.remove("compiled-non-strict"); + res.remove("interpreted-non-strict"); + res.add("non-strict"); + } + if (res.contains("compiled-strict") && res.contains("interpreted-strict")) { + res.remove("compiled-strict"); + res.remove("interpreted-strict"); + res.add("strict"); + } + if (res.contains("compiled-strict") && res.contains("compiled-non-strict")) { + res.remove("compiled-strict"); + res.remove("compiled-non-strict"); + res.add("compiled"); + } + if (res.contains("interpreted-strict") && res.contains("interpreted-non-strict")) { + res.remove("interpreted-strict"); + res.remove("interpreted-non-strict"); + res.add("interpreted"); + } + + if (res.size() > 1) { + return '{' + String.join(",", res) + '}'; + } + return String.join(",", res); } } } diff --git a/tests/testsrc/README.md b/tests/testsrc/README.md index 0447932802..1818d58c24 100644 --- a/tests/testsrc/README.md +++ b/tests/testsrc/README.md @@ -31,7 +31,6 @@ This behavior can be changed through different means: ``` ./gradlew test -DoptLevel=9 ``` -3. Setting an explicit optimization level through the `TEST_262_OPTLEVEL` environment variable ## Running a specific TestSuite ``` diff --git a/tests/testsrc/test262.properties b/tests/testsrc/test262.properties index e340fae792..516aa6903a 100644 --- a/tests/testsrc/test262.properties +++ b/tests/testsrc/test262.properties @@ -674,19 +674,19 @@ built-ins/Function 184/508 (36.22%) prototype/apply/argarray-not-object-realm.js prototype/apply/not-a-constructor.js prototype/apply/resizable-buffer.js {unsupported: [resizable-arraybuffer]} - prototype/apply/S15.3.4.3_A3_T1.js non-interpreted - prototype/apply/S15.3.4.3_A3_T2.js non-interpreted - prototype/apply/S15.3.4.3_A3_T3.js non-interpreted - prototype/apply/S15.3.4.3_A3_T4.js non-interpreted - prototype/apply/S15.3.4.3_A3_T5.js non-interpreted - prototype/apply/S15.3.4.3_A3_T6.js non-interpreted - prototype/apply/S15.3.4.3_A3_T7.js non-interpreted - prototype/apply/S15.3.4.3_A3_T8.js non-interpreted - prototype/apply/S15.3.4.3_A5_T4.js non-interpreted - prototype/apply/S15.3.4.3_A7_T1.js non-interpreted - prototype/apply/S15.3.4.3_A7_T2.js non-interpreted - prototype/apply/S15.3.4.3_A7_T5.js non-interpreted - prototype/apply/S15.3.4.3_A7_T7.js non-interpreted + prototype/apply/S15.3.4.3_A3_T1.js compiled + prototype/apply/S15.3.4.3_A3_T2.js compiled + prototype/apply/S15.3.4.3_A3_T3.js compiled + prototype/apply/S15.3.4.3_A3_T4.js compiled + prototype/apply/S15.3.4.3_A3_T5.js compiled + prototype/apply/S15.3.4.3_A3_T6.js compiled-non-strict + prototype/apply/S15.3.4.3_A3_T7.js compiled + prototype/apply/S15.3.4.3_A3_T8.js compiled-non-strict + prototype/apply/S15.3.4.3_A5_T4.js compiled-non-strict + prototype/apply/S15.3.4.3_A7_T1.js compiled + prototype/apply/S15.3.4.3_A7_T2.js compiled + prototype/apply/S15.3.4.3_A7_T5.js compiled + prototype/apply/S15.3.4.3_A7_T7.js compiled prototype/apply/this-not-callable-realm.js prototype/bind/BoundFunction_restricted-properties.js prototype/bind/get-fn-realm.js @@ -706,19 +706,19 @@ built-ins/Function 184/508 (36.22%) prototype/call/15.3.4.4-2-s.js strict prototype/call/15.3.4.4-3-s.js strict prototype/call/not-a-constructor.js - prototype/call/S15.3.4.4_A3_T1.js non-interpreted - prototype/call/S15.3.4.4_A3_T2.js non-interpreted - prototype/call/S15.3.4.4_A3_T3.js non-interpreted - prototype/call/S15.3.4.4_A3_T4.js non-interpreted - prototype/call/S15.3.4.4_A3_T5.js non-interpreted - prototype/call/S15.3.4.4_A3_T6.js non-interpreted - prototype/call/S15.3.4.4_A3_T7.js non-interpreted - prototype/call/S15.3.4.4_A3_T8.js non-interpreted - prototype/call/S15.3.4.4_A5_T4.js non-interpreted - prototype/call/S15.3.4.4_A6_T1.js non-interpreted - prototype/call/S15.3.4.4_A6_T2.js non-interpreted - prototype/call/S15.3.4.4_A6_T5.js non-interpreted - prototype/call/S15.3.4.4_A6_T7.js non-interpreted + prototype/call/S15.3.4.4_A3_T1.js compiled + prototype/call/S15.3.4.4_A3_T2.js compiled + prototype/call/S15.3.4.4_A3_T3.js compiled + prototype/call/S15.3.4.4_A3_T4.js compiled + prototype/call/S15.3.4.4_A3_T5.js compiled + prototype/call/S15.3.4.4_A3_T6.js compiled-non-strict + prototype/call/S15.3.4.4_A3_T7.js compiled + prototype/call/S15.3.4.4_A3_T8.js compiled-non-strict + prototype/call/S15.3.4.4_A5_T4.js compiled-non-strict + prototype/call/S15.3.4.4_A6_T1.js compiled + prototype/call/S15.3.4.4_A6_T2.js compiled + prototype/call/S15.3.4.4_A6_T5.js compiled + prototype/call/S15.3.4.4_A6_T7.js compiled prototype/Symbol.hasInstance/length.js prototype/Symbol.hasInstance/name.js prototype/Symbol.hasInstance/prop-desc.js @@ -843,7 +843,7 @@ built-ins/Function 184/508 (36.22%) ~built-ins/GeneratorFunction built-ins/GeneratorPrototype 38/60 (63.33%) - next/from-state-executing.js non-interpreted + next/from-state-executing.js compiled next/length.js next/name.js next/not-a-constructor.js @@ -851,7 +851,7 @@ built-ins/GeneratorPrototype 38/60 (63.33%) next/this-val-not-generator.js next/this-val-not-object.js return 22/22 (100.0%) - throw/from-state-executing.js non-interpreted + throw/from-state-executing.js compiled throw/length.js throw/name.js throw/not-a-constructor.js @@ -3438,23 +3438,23 @@ language/destructuring 8/18 (44.44%) binding/typedarray-backed-by-resizable-buffer.js {unsupported: [resizable-arraybuffer]} language/directive-prologue 18/62 (29.03%) - 14.1-1-s.js {non-strict: [-1]} - 14.1-10-s.js {non-strict: [-1]} - 14.1-11-s.js {non-strict: [-1]} - 14.1-12-s.js {non-strict: [-1]} - 14.1-13-s.js {non-strict: [-1]} - 14.1-14-s.js {non-strict: [-1]} - 14.1-15-s.js {non-strict: [-1]} - 14.1-16-s.js non-interpreted - 14.1-17-s.js non-interpreted - 14.1-2-s.js {non-strict: [-1]} - 14.1-3-s.js non-interpreted - 14.1-4-s.js non-interpreted - 14.1-5-s.js non-interpreted - 14.1-6-s.js non-interpreted - 14.1-7-s.js non-interpreted - 14.1-8-s.js {non-strict: [-1]} - 14.1-9-s.js {non-strict: [-1]} + 14.1-1-s.js interpreted-non-strict + 14.1-10-s.js interpreted-non-strict + 14.1-11-s.js interpreted-non-strict + 14.1-12-s.js interpreted-non-strict + 14.1-13-s.js interpreted-non-strict + 14.1-14-s.js interpreted-non-strict + 14.1-15-s.js interpreted-non-strict + 14.1-16-s.js compiled-non-strict + 14.1-17-s.js compiled-non-strict + 14.1-2-s.js interpreted-non-strict + 14.1-3-s.js compiled-non-strict + 14.1-4-s.js compiled-non-strict + 14.1-5-s.js compiled-non-strict + 14.1-6-s.js compiled-non-strict + 14.1-7-s.js compiled-non-strict + 14.1-8-s.js interpreted-non-strict + 14.1-9-s.js interpreted-non-strict func-decl-inside-func-decl-parse.js non-strict language/eval-code 241/347 (69.45%) @@ -4684,7 +4684,7 @@ language/expressions/generators 185/290 (63.79%) prototype-value.js rest-param-strict-body.js scope-body-lex-distinct.js non-strict - scope-name-var-close.js non-interpreted + scope-name-var-close.js compiled scope-name-var-open-non-strict.js non-strict scope-name-var-open-strict.js strict scope-param-rest-elem-var-close.js non-strict @@ -5522,10 +5522,10 @@ language/expressions/object 719/1169 (61.51%) __proto__-permitted-dup-shorthand.js accessor-name-computed-in.js accessor-name-computed-yield-id.js non-strict - accessor-name-literal-numeric-binary.js {strict: [-1], non-strict: [-1]} - accessor-name-literal-numeric-exponent.js {strict: [-1], non-strict: [-1]} - accessor-name-literal-numeric-hex.js {strict: [-1], non-strict: [-1]} - accessor-name-literal-numeric-octal.js {strict: [-1], non-strict: [-1]} + accessor-name-literal-numeric-binary.js interpreted + accessor-name-literal-numeric-exponent.js interpreted + accessor-name-literal-numeric-hex.js interpreted + accessor-name-literal-numeric-octal.js interpreted computed-__proto__.js cpn-obj-lit-computed-property-name-from-async-arrow-function-expression.js cpn-obj-lit-computed-property-name-from-await-expression.js {unsupported: [module, async]} @@ -5775,17 +5775,17 @@ language/function-code 122/217 (56.22%) 10.4.3-1-65gs.js 10.4.3-1-66-s.js 10.4.3-1-66gs.js - 10.4.3-1-67-s.js {strict: [-1], non-strict: [-1]} - 10.4.3-1-67gs.js {strict: [-1], non-strict: [-1]} - 10.4.3-1-68-s.js {strict: [-1], non-strict: [-1]} - 10.4.3-1-68gs.js {strict: [-1], non-strict: [-1]} + 10.4.3-1-67-s.js interpreted + 10.4.3-1-67gs.js interpreted + 10.4.3-1-68-s.js interpreted + 10.4.3-1-68gs.js interpreted 10.4.3-1-7-s.js strict 10.4.3-1-71-s.js 10.4.3-1-71gs.js - 10.4.3-1-72-s.js {strict: [-1], non-strict: [-1]} - 10.4.3-1-72gs.js {strict: [-1], non-strict: [-1]} - 10.4.3-1-73-s.js {strict: [-1], non-strict: [-1]} - 10.4.3-1-73gs.js {strict: [-1], non-strict: [-1]} + 10.4.3-1-72-s.js interpreted + 10.4.3-1-72gs.js interpreted + 10.4.3-1-73-s.js interpreted + 10.4.3-1-73gs.js interpreted 10.4.3-1-76-s.js 10.4.3-1-76gs.js 10.4.3-1-77-s.js @@ -5794,18 +5794,18 @@ language/function-code 122/217 (56.22%) 10.4.3-1-78gs.js 10.4.3-1-7gs.js strict 10.4.3-1-8-s.js non-strict - 10.4.3-1-86-s.js non-interpreted - 10.4.3-1-86gs.js non-interpreted - 10.4.3-1-87-s.js non-interpreted - 10.4.3-1-87gs.js non-interpreted + 10.4.3-1-86-s.js compiled-non-strict + 10.4.3-1-86gs.js compiled-non-strict + 10.4.3-1-87-s.js compiled-non-strict + 10.4.3-1-87gs.js compiled-non-strict 10.4.3-1-8gs.js non-strict 10.4.3-1-9-s.js strict - 10.4.3-1-90-s.js non-interpreted - 10.4.3-1-90gs.js non-interpreted - 10.4.3-1-91-s.js non-interpreted - 10.4.3-1-91gs.js non-interpreted - 10.4.3-1-92-s.js non-interpreted - 10.4.3-1-92gs.js non-interpreted + 10.4.3-1-90-s.js compiled-non-strict + 10.4.3-1-90gs.js compiled-non-strict + 10.4.3-1-91-s.js compiled-non-strict + 10.4.3-1-91gs.js compiled-non-strict + 10.4.3-1-92-s.js compiled-non-strict + 10.4.3-1-92gs.js compiled-non-strict 10.4.3-1-9gs.js strict block-decl-onlystrict.js strict eval-param-env-with-computed-key.js non-strict @@ -7522,7 +7522,7 @@ language/statements/try 113/201 (56.22%) 12.14-15.js non-strict 12.14-16.js non-strict completion-values.js - completion-values-fn-finally-abrupt.js non-interpreted + completion-values-fn-finally-abrupt.js compiled cptn-catch.js cptn-catch-empty-break.js cptn-catch-empty-continue.js