Fix running test262 (#1412)

Summary:

Call the new test runner entry and fix the logic with "--test-intl",
now it won't try to update the skiplist config without providing 
"--test-skiplist" together. Update the config w.r.t. to INTL as well.

Differential Revision: D57977650

.circleci/config.yml

@@ -45,7 +45,9 @@ jobs:
               -DHERMESVM_GCKIND=<< parameters.gc >>
             cmake --build build
             cmake --build build --target check-hermes
-            python3 hermes/utils/testsuite/run_testsuite.py --shermes --test-intl test262/test -b build/bin
+            # TODO(zhaogang): test with --shermes flag as well once we address the harness
+            # code issue with shermes.
+            python3 hermes/utils/test_runner.py --test-intl test262/test -b build/bin
 
   test-linux:
     docker:

utils/testsuite/cli.py

@@ -372,7 +372,11 @@ async def wrap_task(fut: Awaitable) -> TestCaseResult:
     # Print result
     print_stats(stats)
     print_failed_tests(failed_cases)
-    if len(skipped_passed) > 0:
+    # The "intl_tests" list is simply a folder and we don't want to unfold
+    # it. All intl tests that we can't pass are included in "skip_list". So
+    # we will only remove them if both "--test-skiplist" and "--test-intl"
+    # are provided.
+    if len(skipped_passed) > 0 and test_skiplist:
         print_skipped_passed_tests(skipped_passed)
         remove_tests_from_skiplist(skipped_passed, skipped_paths_features)
 

utils/testsuite/skiplist.json

@@ -33,6 +33,15 @@
         "test262/test/language/expressions/async-arrow-function/"
       ],
       "comment": "Lazy arrow functions are not supported for now."
+    },
+    {
+      "paths": [
+        "test262/test/intl402/DateTimeFormat/prototype/formatToParts/",
+        "test262/test/intl402/DateTimeFormat/prototype/formatRange/",
+        "test262/test/intl402/DateTimeFormat/prototype/formatRangeToParts/",
+        "test262/test/intl402/NumberFormat/prototype/formatToParts"
+      ],
+      "comment": "Remove these once we have better Intl support based on ICU."
     }
   ],
   "skip_list": [
@@ -935,28 +944,21 @@
     "test262/test/intl402/DateTimeFormat/constructor-options-order-fractionalSecondDigits.js",
     "test262/test/intl402/DateTimeFormat/constructor-options-order-dayPeriod.js",
     "test262/test/intl402/DateTimeFormat/constructor-options-order.js",
-    "test262/test/intl402/DateTimeFormat/constructor-options-dayPeriod-invalid.js",
     "test262/test/intl402/DateTimeFormat/constructor-options-fractionalSecondDigits-valid.js",
-    "test262/test/intl402/DateTimeFormat/constructor-options-throwing-getters-dayPeriod.js",
     "test262/test/intl402/DateTimeFormat/constructor-options-order-timedate-style.js",
     "test262/test/intl402/DateTimeFormat/constructor-options-dayPeriod-valid.js",
-    "test262/test/intl402/DateTimeFormat/constructor-options-style-conflict.js",
     "test262/test/intl402/DateTimeFormat/prototype/resolvedOptions/order.js",
     "test262/test/intl402/DateTimeFormat/prototype/resolvedOptions/order-fractionalSecondDigits.js",
     "test262/test/intl402/DateTimeFormat/prototype/resolvedOptions/order-dayPeriod.js",
     "test262/test/intl402/DateTimeFormat/prototype/resolvedOptions/hourCycle-timeStyle.js",
     "test262/test/intl402/DateTimeFormat/prototype/resolvedOptions/order-style.js",
-    "test262/test/intl402/DateTimeFormat/prototype/formatToParts",
     "test262/test/intl402/DateTimeFormat/prototype/format/timedatestyle-en.js",
     "test262/test/intl402/DateTimeFormat/prototype/format/dayPeriod-long-en.js",
     "test262/test/intl402/DateTimeFormat/prototype/format/dayPeriod-narrow-en.js",
     "test262/test/intl402/DateTimeFormat/prototype/format/fractionalSecondDigits.js",
     "test262/test/intl402/DateTimeFormat/prototype/format/dayPeriod-short-en.js",
     "test262/test/intl402/DateTimeFormat/constructor-options-numberingSystem-invalid.js",
-    "test262/test/intl402/DateTimeFormat/default-options-object-prototype.js",
     "test262/test/intl402/DateTimeFormat/prototype/format/proleptic-gregorian-calendar.js",
-    "test262/test/intl402/DateTimeFormat/prototype/formatRange",
-    "test262/test/intl402/DateTimeFormat/prototype/formatRangeToParts",
     "test262/test/intl402/DateTimeFormat/prototype/resolvedOptions/offset-timezone-basic.js",
     "test262/test/intl402/DateTimeFormat/prototype/resolvedOptions/offset-timezone-change.js",
     "test262/test/intl402/DateTimeFormat/prototype/resolvedOptions/hourCycle-default.js",
@@ -967,10 +969,7 @@
     "test262/test/intl402/NumberFormat/constructor-locales-hasproperty.js",
     "test262/test/intl402/NumberFormat/prototype/resolvedOptions/order.js",
     "test262/test/intl402/NumberFormat/constructor-order.js",
-    "test262/test/intl402/NumberFormat/constructor-options-roundingMode-invalid.js",
-    "test262/test/intl402/NumberFormat/constructor-options-throwing-getters-rounding-mode.js",
     "test262/test/intl402/NumberFormat/ignore-invalid-unicode-ext-values.js",
-    "test262/test/intl402/NumberFormat/prototype/resolvedOptions/roundingMode.js",
     "test262/test/intl402/NumberFormat/prototype/format/signDisplay-en-US.js",
     "test262/test/intl402/NumberFormat/prototype/format/signDisplay-currency-en-US.js",
     "test262/test/intl402/NumberFormat/prototype/format/engineering-scientific-ja-JP.js",
@@ -995,7 +994,6 @@
     "test262/test/intl402/NumberFormat/prototype/format/signDisplay-zh-TW.js",
     "test262/test/intl402/NumberFormat/prototype/format/notation-compact-de-DE.js",
     "test262/test/intl402/NumberFormat/prototype/format/signDisplay-currency-zh-TW.js",
-    "test262/test/intl402/NumberFormat/prototype/formatToParts",
     "test262/test/intl402/NumberFormat/throws-for-minimumFractionDigits-over-limit.js",
     "test262/test/intl402/NumberFormat/throws-for-maximumFractionDigits-over-limit.js",
     "test262/test/language/identifiers/part-unicode-5.2.0-escaped.js",