From 0bf7da61b442f029a980c837ed363a5b1de0552a Mon Sep 17 00:00:00 2001
From: "Gang Zhao (Hermes)" <zhaogang@meta.com>
Date: Thu, 30 May 2024 16:27:31 -0700
Subject: [PATCH] Add a script wrapper to run directly and update README

Summary:
Add a script outside of testsuite package so we can run it directly.
Update the README in testsuite as well.

Reviewed By: avp

Differential Revision: D57984322

fbshipit-source-id: 9d5c6c8c5b30dd8983af792971170fb56e1911c9
---
 utils/test_runner.py      | 13 +++++++++++++
 utils/testsuite/README.md | 27 +++++++++++++++++++--------
 utils/testsuite/cli.py    |  5 -----
 3 files changed, 32 insertions(+), 13 deletions(-)
 create mode 100755 utils/test_runner.py

diff --git a/utils/test_runner.py b/utils/test_runner.py
new file mode 100755
index 00000000000..b5e2cbc9c6a
--- /dev/null
+++ b/utils/test_runner.py
@@ -0,0 +1,13 @@
+#!/usr/bin/env fbpython
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import asyncio
+import sys
+
+from testsuite.cli import main
+
+if __name__ == "__main__":
+    sys.exit(asyncio.run(main()))
diff --git a/utils/testsuite/README.md b/utils/testsuite/README.md
index 36324daadb4..b4312e49faa 100644
--- a/utils/testsuite/README.md
+++ b/utils/testsuite/README.md
@@ -2,10 +2,12 @@
 
 ## How to run
 
-The runner (`cli.py`) requires two paths:
-1. Path to the test suite directory that we support (i.e., test262, mjsunit, esprima, flow and a CVE testsuite). This can be
+The runner requires two paths:
+1. Path to the test suite directory that we support (i.e., test262, mjsunit,
+esprima, flow and a CVE testsuite). This can be
 a list separated by whitespace, so you can run multiple test suites at once.
-2. Path to the directory of Hermes binaries, including `hermesc`, `hvm` (and `hermes`, `shermes` depending on extra flags).
+2. Path to the directory of Hermes binaries, including `hermes` (and`shermes`
+depending on extra flags).
 
 Example usage:
 
@@ -13,14 +15,23 @@ Example usage:
 # ./build is the cmake build directory.
 
 # flow
-./utils/testsuite2/cli.py external/flowtest/test/flow -b ./build/bin
+./utils/test_runner.py external/flowtest/test/flow -b ./build/bin
 
 # esprima
-./utils/testsuite2/cli.py external/esprima/test_fixtures -b ./build/bin
+.utils/test_runner.py  external/esprima/test_fixtures -b ./build/bin
 ```
 
 ## Skiplist
 
-Run with flag `--test-skiplist` to force running tests that are included in the `skiplist.json` config.
-
-[TODO]
+Run with flag `--test-skiplist` to force running tests that are included in the
+`skiplist.json` config (but we will scan only "skiplist" and "manual_skiplist"
+in the config, and "lazy_skip_list" if `--lazy` is given, "intl_tests" if
+`--test-intl` is given).
+
+After it finishes, all passed tests included in the skiplist config file will be
+printed, and you can tell the runner to remove these tests from the config by
+following the prompt ("manual_list" won't be touched). Note that if some tests
+are covered by a folder path in the config, the test runner will first find all
+tests under that folder, exclude those passed, and insert the rest into the
+config file. If you don't want this behavior for a specific folder, please add
+it to the manual list with a comment explaining the reason.
diff --git a/utils/testsuite/cli.py b/utils/testsuite/cli.py
index e06d997a331..69c57d3d0aa 100755
--- a/utils/testsuite/cli.py
+++ b/utils/testsuite/cli.py
@@ -9,7 +9,6 @@
 import importlib.resources
 import os
 import shutil
-import sys
 import tempfile
 import time
 from asyncio import Semaphore
@@ -452,7 +451,3 @@ async def main() -> int:
         tmp_work_dir.cleanup()
 
     return exit_code
-
-
-if __name__ == "__main__":
-    sys.exit(asyncio.run(main()))