From 01c2618d6f831a9f4a4d6c8606b507160e8156ea Mon Sep 17 00:00:00 2001 From: Ronald Brill Date: Sat, 21 Dec 2024 18:04:39 +0100 Subject: [PATCH 1/3] make the report backward compatible --- .../javascript/tests/Test262SuiteTest.java | 64 ++++++------- tests/testsrc/test262.properties | 90 +++++++++---------- 2 files changed, 77 insertions(+), 77 deletions(-) diff --git a/tests/src/test/java/org/mozilla/javascript/tests/Test262SuiteTest.java b/tests/src/test/java/org/mozilla/javascript/tests/Test262SuiteTest.java index a0f27f5945..74acd27f74 100644 --- a/tests/src/test/java/org/mozilla/javascript/tests/Test262SuiteTest.java +++ b/tests/src/test/java/org/mozilla/javascript/tests/Test262SuiteTest.java @@ -71,8 +71,6 @@ public class Test262SuiteTest { /** The test must be executed just once--in non-strict mode, only. */ private static final String FLAG_NO_STRICT = "noStrict"; - static final int[] OPT_LEVELS; - private static final File testDir = new File("test262/test"); private static final String testHarnessDir = "test262/harness/"; private static final String testProperties; @@ -132,35 +130,11 @@ public class Test262SuiteTest { includeUnsupported = updateProps.isEmpty() || updateProps.indexOf("unsupported") != -1; } - - if (getOverriddenLevel() != null) { - System.out.println( - "Ignoring custom optLevels because the updateTest262Properties param is set"); - } - - OPT_LEVELS = Utils.DEFAULT_OPT_LEVELS; } else { updateTest262Properties = rollUpEnabled = statsEnabled = includeUnsupported = false; - - // Reduce the number of tests that we run by a factor of three... - String overriddenLevel = getOverriddenLevel(); - if (overriddenLevel != null) { - OPT_LEVELS = new int[] {Integer.parseInt(overriddenLevel)}; - } else { - OPT_LEVELS = Utils.DEFAULT_OPT_LEVELS; - } } } - private static String getOverriddenLevel() { - String optLevel = System.getProperty("TEST_OPTLEVEL"); - - if (optLevel == null || optLevel.isEmpty()) { - optLevel = System.getenv("TEST_262_OPTLEVEL"); - } - return optLevel; - } - @BeforeAll public static void setUpClass() { CTX_FACTORY.setLanguageVersion(Context.VERSION_ES6); @@ -279,7 +253,7 @@ public static void tearDownClass() { } if (!testFile.isDirectory()) { - testResult = tt.getResult(OPT_LEVELS, testCases[j]); + testResult = tt.getResult(testCases[j]); if (testResult == null) { // At least one passing test in currentParent directory, so prevent @@ -1051,8 +1025,8 @@ public void passes(TestMode mode, boolean useStrict) { modes.remove(makeKey(mode, useStrict)); } - public String getResult(int[] optLevels, Test262Case tc) { - // success on all optLevels in both strict and non-strict mode + public String getResult(Test262Case tc) { + // success in interpreted and optimized mode in both strict and non-strict mode if (modes.isEmpty()) { return null; } @@ -1078,13 +1052,39 @@ public String getResult(int[] optLevels, Test262Case tc) { return "{unsupported: " + Arrays.toString(feats.toArray()) + "}"; } - // failure on all optLevels in both strict and non-strict mode + // failure in interpreted and optimized mode in both strict and non-strict mode + // no need to add more details if (modes.size() == 4) { return ""; } - // mix of mode and optLevel successes and failures - return '{' + String.join(",", new ArrayList<>(modes)) + '}'; + // simplify the output for some cases + ArrayList res = new ArrayList<>(modes); + if (modes.contains("compiled-non-strict") && modes.contains("interpreted-non-strict")) { + res.remove("compiled-non-strict"); + res.remove("interpreted-non-strict"); + res.add("non-strict"); + } + if (modes.contains("compiled-strict") && modes.contains("interpreted-strict")) { + res.remove("compiled-strict"); + res.remove("interpreted-strict"); + res.add("strict"); + } + if (modes.contains("compiled-strict") && modes.contains("compiled-non-strict")) { + res.remove("compiled-strict"); + res.remove("compiled-non-strict"); + res.add("non-interpreted"); // maybe "compiled" + } + if (modes.contains("interpreted-strict") && modes.contains("interpreted-non-strict")) { + res.remove("interpreted-strict"); + res.remove("interpreted-non-strict"); + res.add("interpreted"); + } + + if (res.size() > 1) { + return '{' + String.join(",", res) + '}'; + } + return String.join(",", res); } } } diff --git a/tests/testsrc/test262.properties b/tests/testsrc/test262.properties index e340fae792..540b0e4ec2 100644 --- a/tests/testsrc/test262.properties +++ b/tests/testsrc/test262.properties @@ -679,10 +679,10 @@ built-ins/Function 184/508 (36.22%) prototype/apply/S15.3.4.3_A3_T3.js non-interpreted prototype/apply/S15.3.4.3_A3_T4.js non-interpreted prototype/apply/S15.3.4.3_A3_T5.js non-interpreted - prototype/apply/S15.3.4.3_A3_T6.js non-interpreted + prototype/apply/S15.3.4.3_A3_T6.js compiled-non-strict prototype/apply/S15.3.4.3_A3_T7.js non-interpreted - prototype/apply/S15.3.4.3_A3_T8.js non-interpreted - prototype/apply/S15.3.4.3_A5_T4.js non-interpreted + prototype/apply/S15.3.4.3_A3_T8.js compiled-non-strict + prototype/apply/S15.3.4.3_A5_T4.js compiled-non-strict prototype/apply/S15.3.4.3_A7_T1.js non-interpreted prototype/apply/S15.3.4.3_A7_T2.js non-interpreted prototype/apply/S15.3.4.3_A7_T5.js non-interpreted @@ -711,10 +711,10 @@ built-ins/Function 184/508 (36.22%) prototype/call/S15.3.4.4_A3_T3.js non-interpreted prototype/call/S15.3.4.4_A3_T4.js non-interpreted prototype/call/S15.3.4.4_A3_T5.js non-interpreted - prototype/call/S15.3.4.4_A3_T6.js non-interpreted + prototype/call/S15.3.4.4_A3_T6.js compiled-non-strict prototype/call/S15.3.4.4_A3_T7.js non-interpreted - prototype/call/S15.3.4.4_A3_T8.js non-interpreted - prototype/call/S15.3.4.4_A5_T4.js non-interpreted + prototype/call/S15.3.4.4_A3_T8.js compiled-non-strict + prototype/call/S15.3.4.4_A5_T4.js compiled-non-strict prototype/call/S15.3.4.4_A6_T1.js non-interpreted prototype/call/S15.3.4.4_A6_T2.js non-interpreted prototype/call/S15.3.4.4_A6_T5.js non-interpreted @@ -3438,23 +3438,23 @@ language/destructuring 8/18 (44.44%) binding/typedarray-backed-by-resizable-buffer.js {unsupported: [resizable-arraybuffer]} language/directive-prologue 18/62 (29.03%) - 14.1-1-s.js {non-strict: [-1]} - 14.1-10-s.js {non-strict: [-1]} - 14.1-11-s.js {non-strict: [-1]} - 14.1-12-s.js {non-strict: [-1]} - 14.1-13-s.js {non-strict: [-1]} - 14.1-14-s.js {non-strict: [-1]} - 14.1-15-s.js {non-strict: [-1]} - 14.1-16-s.js non-interpreted - 14.1-17-s.js non-interpreted - 14.1-2-s.js {non-strict: [-1]} - 14.1-3-s.js non-interpreted - 14.1-4-s.js non-interpreted - 14.1-5-s.js non-interpreted - 14.1-6-s.js non-interpreted - 14.1-7-s.js non-interpreted - 14.1-8-s.js {non-strict: [-1]} - 14.1-9-s.js {non-strict: [-1]} + 14.1-1-s.js interpreted-non-strict + 14.1-10-s.js interpreted-non-strict + 14.1-11-s.js interpreted-non-strict + 14.1-12-s.js interpreted-non-strict + 14.1-13-s.js interpreted-non-strict + 14.1-14-s.js interpreted-non-strict + 14.1-15-s.js interpreted-non-strict + 14.1-16-s.js compiled-non-strict + 14.1-17-s.js compiled-non-strict + 14.1-2-s.js interpreted-non-strict + 14.1-3-s.js compiled-non-strict + 14.1-4-s.js compiled-non-strict + 14.1-5-s.js compiled-non-strict + 14.1-6-s.js compiled-non-strict + 14.1-7-s.js compiled-non-strict + 14.1-8-s.js interpreted-non-strict + 14.1-9-s.js interpreted-non-strict func-decl-inside-func-decl-parse.js non-strict language/eval-code 241/347 (69.45%) @@ -5522,10 +5522,10 @@ language/expressions/object 719/1169 (61.51%) __proto__-permitted-dup-shorthand.js accessor-name-computed-in.js accessor-name-computed-yield-id.js non-strict - accessor-name-literal-numeric-binary.js {strict: [-1], non-strict: [-1]} - accessor-name-literal-numeric-exponent.js {strict: [-1], non-strict: [-1]} - accessor-name-literal-numeric-hex.js {strict: [-1], non-strict: [-1]} - accessor-name-literal-numeric-octal.js {strict: [-1], non-strict: [-1]} + accessor-name-literal-numeric-binary.js interpreted + accessor-name-literal-numeric-exponent.js interpreted + accessor-name-literal-numeric-hex.js interpreted + accessor-name-literal-numeric-octal.js interpreted computed-__proto__.js cpn-obj-lit-computed-property-name-from-async-arrow-function-expression.js cpn-obj-lit-computed-property-name-from-await-expression.js {unsupported: [module, async]} @@ -5775,17 +5775,17 @@ language/function-code 122/217 (56.22%) 10.4.3-1-65gs.js 10.4.3-1-66-s.js 10.4.3-1-66gs.js - 10.4.3-1-67-s.js {strict: [-1], non-strict: [-1]} - 10.4.3-1-67gs.js {strict: [-1], non-strict: [-1]} - 10.4.3-1-68-s.js {strict: [-1], non-strict: [-1]} - 10.4.3-1-68gs.js {strict: [-1], non-strict: [-1]} + 10.4.3-1-67-s.js interpreted + 10.4.3-1-67gs.js interpreted + 10.4.3-1-68-s.js interpreted + 10.4.3-1-68gs.js interpreted 10.4.3-1-7-s.js strict 10.4.3-1-71-s.js 10.4.3-1-71gs.js - 10.4.3-1-72-s.js {strict: [-1], non-strict: [-1]} - 10.4.3-1-72gs.js {strict: [-1], non-strict: [-1]} - 10.4.3-1-73-s.js {strict: [-1], non-strict: [-1]} - 10.4.3-1-73gs.js {strict: [-1], non-strict: [-1]} + 10.4.3-1-72-s.js interpreted + 10.4.3-1-72gs.js interpreted + 10.4.3-1-73-s.js interpreted + 10.4.3-1-73gs.js interpreted 10.4.3-1-76-s.js 10.4.3-1-76gs.js 10.4.3-1-77-s.js @@ -5794,18 +5794,18 @@ language/function-code 122/217 (56.22%) 10.4.3-1-78gs.js 10.4.3-1-7gs.js strict 10.4.3-1-8-s.js non-strict - 10.4.3-1-86-s.js non-interpreted - 10.4.3-1-86gs.js non-interpreted - 10.4.3-1-87-s.js non-interpreted - 10.4.3-1-87gs.js non-interpreted + 10.4.3-1-86-s.js compiled-non-strict + 10.4.3-1-86gs.js compiled-non-strict + 10.4.3-1-87-s.js compiled-non-strict + 10.4.3-1-87gs.js compiled-non-strict 10.4.3-1-8gs.js non-strict 10.4.3-1-9-s.js strict - 10.4.3-1-90-s.js non-interpreted - 10.4.3-1-90gs.js non-interpreted - 10.4.3-1-91-s.js non-interpreted - 10.4.3-1-91gs.js non-interpreted - 10.4.3-1-92-s.js non-interpreted - 10.4.3-1-92gs.js non-interpreted + 10.4.3-1-90-s.js compiled-non-strict + 10.4.3-1-90gs.js compiled-non-strict + 10.4.3-1-91-s.js compiled-non-strict + 10.4.3-1-91gs.js compiled-non-strict + 10.4.3-1-92-s.js compiled-non-strict + 10.4.3-1-92gs.js compiled-non-strict 10.4.3-1-9gs.js strict block-decl-onlystrict.js strict eval-param-env-with-computed-key.js non-strict From 5329eff5fc2d159253faaa1bb8be3b045ff51f6a Mon Sep 17 00:00:00 2001 From: Ronald Brill Date: Sat, 21 Dec 2024 20:21:06 +0100 Subject: [PATCH 2/3] fixes, the update should now be stable again --- .../javascript/tests/Test262SuiteTest.java | 19 ++++---- tests/testsrc/test262.properties | 48 +++++++++---------- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/tests/src/test/java/org/mozilla/javascript/tests/Test262SuiteTest.java b/tests/src/test/java/org/mozilla/javascript/tests/Test262SuiteTest.java index 74acd27f74..a08f64f8fe 100644 --- a/tests/src/test/java/org/mozilla/javascript/tests/Test262SuiteTest.java +++ b/tests/src/test/java/org/mozilla/javascript/tests/Test262SuiteTest.java @@ -394,7 +394,9 @@ public static void tearDownClass() { */ private static final Pattern LINE_SPLITTER = Pattern.compile( - "(~|(?:\\s*)(?:!|#)(?:\\s*)|\\s+)?(\\S+)(?:[^\\S\\r\\n]+(?:strict|non-strict|non-interpreted|\\d+/\\d+ \\(\\d+(?:\\.\\d+)?%%\\)|\\{(?:non-strict|strict|unsupported): \\[.*\\],?\\}))?[^\\S\\r\\n]*(.*)"); + "(~|(?:\\s*)(?:!|#)(?:\\s*)|\\s+)?(\\S+)(?:[^\\S\\r\\n]+" + + "(?:strict|non-strict|compiled-strict|compiled-non-strict|interpreted-strict|interpreted-non-strict|compiled|interpreted|" + + "\\d+/\\d+ \\(\\d+(?:\\.\\d+)?%%\\)|\\{(?:non-strict|strict|unsupported): \\[.*\\],?\\}))?[^\\S\\r\\n]*(.*)"); /** * @see https://github.com/tc39/test262/blob/main/INTERPRETING.md#host-defined-functions @@ -1026,7 +1028,7 @@ public void passes(TestMode mode, boolean useStrict) { } public String getResult(Test262Case tc) { - // success in interpreted and optimized mode in both strict and non-strict mode + // success on all optLevels in both strict and non-strict mode if (modes.isEmpty()) { return null; } @@ -1052,30 +1054,29 @@ public String getResult(Test262Case tc) { return "{unsupported: " + Arrays.toString(feats.toArray()) + "}"; } - // failure in interpreted and optimized mode in both strict and non-strict mode - // no need to add more details + // failure on all optLevels in both strict and non-strict mode if (modes.size() == 4) { return ""; } // simplify the output for some cases ArrayList res = new ArrayList<>(modes); - if (modes.contains("compiled-non-strict") && modes.contains("interpreted-non-strict")) { + if (res.contains("compiled-non-strict") && res.contains("interpreted-non-strict")) { res.remove("compiled-non-strict"); res.remove("interpreted-non-strict"); res.add("non-strict"); } - if (modes.contains("compiled-strict") && modes.contains("interpreted-strict")) { + if (res.contains("compiled-strict") && res.contains("interpreted-strict")) { res.remove("compiled-strict"); res.remove("interpreted-strict"); res.add("strict"); } - if (modes.contains("compiled-strict") && modes.contains("compiled-non-strict")) { + if (res.contains("compiled-strict") && res.contains("compiled-non-strict")) { res.remove("compiled-strict"); res.remove("compiled-non-strict"); - res.add("non-interpreted"); // maybe "compiled" + res.add("compiled"); } - if (modes.contains("interpreted-strict") && modes.contains("interpreted-non-strict")) { + if (res.contains("interpreted-strict") && res.contains("interpreted-non-strict")) { res.remove("interpreted-strict"); res.remove("interpreted-non-strict"); res.add("interpreted"); diff --git a/tests/testsrc/test262.properties b/tests/testsrc/test262.properties index 540b0e4ec2..516aa6903a 100644 --- a/tests/testsrc/test262.properties +++ b/tests/testsrc/test262.properties @@ -674,19 +674,19 @@ built-ins/Function 184/508 (36.22%) prototype/apply/argarray-not-object-realm.js prototype/apply/not-a-constructor.js prototype/apply/resizable-buffer.js {unsupported: [resizable-arraybuffer]} - prototype/apply/S15.3.4.3_A3_T1.js non-interpreted - prototype/apply/S15.3.4.3_A3_T2.js non-interpreted - prototype/apply/S15.3.4.3_A3_T3.js non-interpreted - prototype/apply/S15.3.4.3_A3_T4.js non-interpreted - prototype/apply/S15.3.4.3_A3_T5.js non-interpreted + prototype/apply/S15.3.4.3_A3_T1.js compiled + prototype/apply/S15.3.4.3_A3_T2.js compiled + prototype/apply/S15.3.4.3_A3_T3.js compiled + prototype/apply/S15.3.4.3_A3_T4.js compiled + prototype/apply/S15.3.4.3_A3_T5.js compiled prototype/apply/S15.3.4.3_A3_T6.js compiled-non-strict - prototype/apply/S15.3.4.3_A3_T7.js non-interpreted + prototype/apply/S15.3.4.3_A3_T7.js compiled prototype/apply/S15.3.4.3_A3_T8.js compiled-non-strict prototype/apply/S15.3.4.3_A5_T4.js compiled-non-strict - prototype/apply/S15.3.4.3_A7_T1.js non-interpreted - prototype/apply/S15.3.4.3_A7_T2.js non-interpreted - prototype/apply/S15.3.4.3_A7_T5.js non-interpreted - prototype/apply/S15.3.4.3_A7_T7.js non-interpreted + prototype/apply/S15.3.4.3_A7_T1.js compiled + prototype/apply/S15.3.4.3_A7_T2.js compiled + prototype/apply/S15.3.4.3_A7_T5.js compiled + prototype/apply/S15.3.4.3_A7_T7.js compiled prototype/apply/this-not-callable-realm.js prototype/bind/BoundFunction_restricted-properties.js prototype/bind/get-fn-realm.js @@ -706,19 +706,19 @@ built-ins/Function 184/508 (36.22%) prototype/call/15.3.4.4-2-s.js strict prototype/call/15.3.4.4-3-s.js strict prototype/call/not-a-constructor.js - prototype/call/S15.3.4.4_A3_T1.js non-interpreted - prototype/call/S15.3.4.4_A3_T2.js non-interpreted - prototype/call/S15.3.4.4_A3_T3.js non-interpreted - prototype/call/S15.3.4.4_A3_T4.js non-interpreted - prototype/call/S15.3.4.4_A3_T5.js non-interpreted + prototype/call/S15.3.4.4_A3_T1.js compiled + prototype/call/S15.3.4.4_A3_T2.js compiled + prototype/call/S15.3.4.4_A3_T3.js compiled + prototype/call/S15.3.4.4_A3_T4.js compiled + prototype/call/S15.3.4.4_A3_T5.js compiled prototype/call/S15.3.4.4_A3_T6.js compiled-non-strict - prototype/call/S15.3.4.4_A3_T7.js non-interpreted + prototype/call/S15.3.4.4_A3_T7.js compiled prototype/call/S15.3.4.4_A3_T8.js compiled-non-strict prototype/call/S15.3.4.4_A5_T4.js compiled-non-strict - prototype/call/S15.3.4.4_A6_T1.js non-interpreted - prototype/call/S15.3.4.4_A6_T2.js non-interpreted - prototype/call/S15.3.4.4_A6_T5.js non-interpreted - prototype/call/S15.3.4.4_A6_T7.js non-interpreted + prototype/call/S15.3.4.4_A6_T1.js compiled + prototype/call/S15.3.4.4_A6_T2.js compiled + prototype/call/S15.3.4.4_A6_T5.js compiled + prototype/call/S15.3.4.4_A6_T7.js compiled prototype/Symbol.hasInstance/length.js prototype/Symbol.hasInstance/name.js prototype/Symbol.hasInstance/prop-desc.js @@ -843,7 +843,7 @@ built-ins/Function 184/508 (36.22%) ~built-ins/GeneratorFunction built-ins/GeneratorPrototype 38/60 (63.33%) - next/from-state-executing.js non-interpreted + next/from-state-executing.js compiled next/length.js next/name.js next/not-a-constructor.js @@ -851,7 +851,7 @@ built-ins/GeneratorPrototype 38/60 (63.33%) next/this-val-not-generator.js next/this-val-not-object.js return 22/22 (100.0%) - throw/from-state-executing.js non-interpreted + throw/from-state-executing.js compiled throw/length.js throw/name.js throw/not-a-constructor.js @@ -4684,7 +4684,7 @@ language/expressions/generators 185/290 (63.79%) prototype-value.js rest-param-strict-body.js scope-body-lex-distinct.js non-strict - scope-name-var-close.js non-interpreted + scope-name-var-close.js compiled scope-name-var-open-non-strict.js non-strict scope-name-var-open-strict.js strict scope-param-rest-elem-var-close.js non-strict @@ -7522,7 +7522,7 @@ language/statements/try 113/201 (56.22%) 12.14-15.js non-strict 12.14-16.js non-strict completion-values.js - completion-values-fn-finally-abrupt.js non-interpreted + completion-values-fn-finally-abrupt.js compiled cptn-catch.js cptn-catch-empty-break.js cptn-catch-empty-continue.js From 82c052ae89466f9ee5e3aaaa88b2859fca02821d Mon Sep 17 00:00:00 2001 From: Ronald Brill Date: Sun, 22 Dec 2024 10:20:19 +0100 Subject: [PATCH 3/3] update docu also; setting the testlevel for the 262 test suite is no longer supported --- tests/testsrc/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/testsrc/README.md b/tests/testsrc/README.md index 0447932802..1818d58c24 100644 --- a/tests/testsrc/README.md +++ b/tests/testsrc/README.md @@ -31,7 +31,6 @@ This behavior can be changed through different means: ``` ./gradlew test -DoptLevel=9 ``` -3. Setting an explicit optimization level through the `TEST_262_OPTLEVEL` environment variable ## Running a specific TestSuite ```