[lldb][test] Support remote run of Shell tests

lldb/docs/resources/test.rst

@@ -592,15 +592,17 @@ test suite, but there are two things to have in mind:
    multiple connections. For more information on how to setup remote debugging
    see the Remote debugging page.
 2. You must tell the test-suite how to connect to the remote system. This is
-   achieved using the ``--platform-name``, ``--platform-url`` and
-   ``--platform-working-dir`` parameters to ``dotest.py``. These parameters
-   correspond to the platform select and platform connect LLDB commands. You
-   will usually also need to specify the compiler and architecture for the
-   remote system.
-
-Currently, running the remote test suite is supported only with ``dotest.py`` (or
-dosep.py with a single thread), but we expect this issue to be addressed in the
-near future.
+   achieved using the ``LLDB_TEST_PLATFORM_URL``, ``LLDB_TEST_PLATFORM_WORKING_DIR``
+   flags to cmake, and ``--platform-name`` parameter to ``dotest.py``.
+   These parameters correspond to the platform select and platform connect
+   LLDB commands. You will usually also need to specify the compiler and
+   architecture for the remote system.
+3. Remote Shell tests execution is currently supported only for Linux target
+   platform. It's triggered when ``LLDB_TEST_SYSROOT`` is provided for building
+   test sources. It can be disabled by setting ``LLDB_SHELL_TESTS_DISABLE_REMOTE=On``.
+   Shell tests are not guaranteed to pass against remote target if the compiler
+   being used is other than Clang.
+
 
 Running tests in QEMU System Emulation Environment
 ``````````````````````````````````````````````````

lldb/test/API/lit.cfg.py

@@ -283,6 +283,13 @@ def delete_module_cache(path):
 # In particular, (1) is visited at the top of the file, since the script
 # derives other information from it.
 
+if is_configured("lldb_platform_url"):
+    dotest_cmd += ["--platform-url", config.lldb_platform_url]
+if is_configured("lldb_platform_working_dir"):
+    dotest_cmd += ["--platform-working-dir", config.lldb_platform_working_dir]
+if is_configured("cmake_sysroot"):
+    dotest_cmd += ["--sysroot", config.cmake_sysroot]
+
 if is_configured("dotest_user_args_str"):
     dotest_cmd.extend(config.dotest_user_args_str.split(";"))
 

lldb/test/API/lit.site.cfg.py.in

@@ -24,6 +24,9 @@ config.lua_executable = "@Lua_EXECUTABLE@"
 config.lua_test_entry = "TestLuaAPI.py"
 config.dotest_common_args_str = lit_config.substitute("@LLDB_TEST_COMMON_ARGS@")
 config.dotest_user_args_str = lit_config.substitute("@LLDB_TEST_USER_ARGS@")
+config.lldb_platform_url = lit_config.substitute("@LLDB_TEST_PLATFORM_URL@")
+config.lldb_platform_working_dir = lit_config.substitute("@LLDB_TEST_PLATFORM_WORKING_DIR@")
+config.cmake_sysroot = lit_config.substitute("@LLDB_TEST_SYSROOT@" or "@DEFAULT_SYSROOT@")
 config.lldb_enable_python = @LLDB_ENABLE_PYTHON@
 config.dotest_lit_args_str = None
 config.enabled_plugins = []

lldb/test/CMakeLists.txt

@@ -234,6 +234,8 @@ if (CMAKE_SIZEOF_VOID_P EQUAL 8)
   set(LLDB_IS_64_BITS 1)
 endif()
 
+set(LLDB_SHELL_TESTS_DISABLE_REMOTE OFF CACHE BOOL "Disable remote Shell tests execution")
+
 # These values are not canonicalized within LLVM.
 llvm_canonicalize_cmake_booleans(
   LLDB_BUILD_INTEL_PT
@@ -243,6 +245,7 @@ llvm_canonicalize_cmake_booleans(
   LLVM_ENABLE_ZLIB
   LLVM_ENABLE_SHARED_LIBS
   LLDB_HAS_LIBCXX
+  LLDB_SHELL_TESTS_DISABLE_REMOTE
   LLDB_TOOL_LLDB_SERVER_BUILD
   LLDB_USE_SYSTEM_DEBUGSERVER
   LLDB_IS_64_BITS)

lldb/test/Shell/Settings/TestEchoCommands.test

@@ -1,8 +1,3 @@
 # RUN: %lldb -x -b -o 'settings set interpreter.echo-comment-commands true'  -s %S/Inputs/EchoCommandsTest.in | FileCheck %S/Inputs/EchoCommandsAll.out
 # RUN: %lldb -x -b -o 'settings set interpreter.echo-comment-commands false' -s %S/Inputs/EchoCommandsTest.in | FileCheck %S/Inputs/EchoCommandsNoComments.out
 # RUN: %lldb -x -b -o 'settings set interpreter.echo-commands false'         -s %S/Inputs/EchoCommandsTest.in | FileCheck %S/Inputs/EchoCommandsNone.out
-
-RUN: echo start >%t.file
-RUN: %lldb -x -b --source-quietly -s %S/Inputs/EchoCommandsTest.in >>%t.file
-RUN: echo done >>%t.file
-RUN: FileCheck  %S/Inputs/EchoCommandsQuiet.out <%t.file

lldb/test/Shell/Settings/TestEchoCommandsQuiet.test

@@ -0,0 +1,6 @@
+Can't mute platform select command.
+UNSUPPORTED: remote-linux
+RUN: echo start >%t.file
+RUN: %lldb -x -b --source-quietly -s %S/Inputs/EchoCommandsTest.in >>%t.file
+RUN: echo done >>%t.file
+RUN: FileCheck  %S/Inputs/EchoCommandsQuiet.out <%t.file

lldb/test/Shell/Target/target-label.test

@@ -4,16 +4,16 @@
 
 target create -l "ls" /bin/ls
 target list
-# CHECK: * target #0 (ls): /bin/ls
+# CHECK: * target #0 (ls): [[LS_PATH:.*]]
 
 script lldb.target.SetLabel("")
 target list
-# CHECK: * target #0: /bin/ls
+# CHECK: * target #0: [[LS_PATH]]
 
 target create -l "cat" /bin/cat
 target list
-# CHECK: target #0: /bin/ls
-# CHECK-NEXT: * target #1 (cat): /bin/cat
+# CHECK: target #0: [[LS_PATH]]
+# CHECK-NEXT: * target #1 (cat): [[CAT_PATH:.*]]
 
 target create -l "cat" /bin/cat
 # CHECK: Cannot use label 'cat' since it's set in target #1.
@@ -22,12 +22,12 @@ target create -l 42 /bin/cat
 # CHECK: error: Cannot use integer as target label.
 
 target select 0
-# CHECK: * target #0: /bin/ls
-# CHECK-NEXT: target #1 (cat): /bin/cat
+# CHECK: * target #0: [[LS_PATH]]
+# CHECK-NEXT: target #1 (cat): [[CAT_PATH]]
 
 target select cat
-# CHECK: target #0: /bin/ls
-# CHECK-NEXT: * target #1 (cat): /bin/cat
+# CHECK: target #0: [[LS_PATH]]
+# CHECK-NEXT: * target #1 (cat): [[CAT_PATH]]
 
 script lldb.target.GetLabel()
 # CHECK: 'cat'
@@ -36,5 +36,5 @@ script lldb.debugger.GetTargetAtIndex(0).SetLabel('Not cat')
 # CHECK: success
 
 target list
-# CHECK: target #0 (Not cat): /bin/ls
-# CHECK-NEXT: * target #1 (cat): /bin/cat
+# CHECK: target #0 (Not cat): [[LS_PATH]]
+# CHECK-NEXT: * target #1 (cat): [[CAT_PATH]]

lldb/test/Shell/helper/toolchain.py

@@ -1,14 +1,17 @@
 import os
 import itertools
 import platform
+import re
 import subprocess
 import sys
 
 import lit.util
+from lit.formats import ShTest
 from lit.llvm import llvm_config
 from lit.llvm.subst import FindTool
 from lit.llvm.subst import ToolSubst
 
+import posixpath
 
 def _get_lldb_init_path(config):
     return os.path.join(config.test_exec_root, "lit-lldb-init-quiet")
@@ -22,6 +25,68 @@ def _disallow(config, execName):
     config.substitutions.append((" {0} ".format(execName), warning.format(execName)))
 
 
+def get_lldb_args(config, suffix=""):
+    lldb_args = []
+    if "remote-linux" in config.available_features:
+        lldb_args += [
+            "-O",
+            '"platform select remote-linux"',
+            "-O",
+            f'"platform connect {config.lldb_platform_url}"',
+        ]
+        if config.lldb_platform_working_dir:
+            dir = posixpath.join(f"{config.lldb_platform_working_dir}", "shell")
+            if suffix:
+                dir += posixpath.join(dir, f"{suffix}")
+            lldb_args += [
+                "-O",
+                f'"platform shell mkdir -p {dir}"',
+                "-O",
+                f'"platform settings -w {dir}"',
+            ]
+    lldb_args += ["--no-lldbinit", "-S", _get_lldb_init_path(config)]
+    return lldb_args
+
+
+class ShTestLldb(ShTest):
+    def __init__(
+        self, execute_external=False, extra_substitutions=[], preamble_commands=[]
+    ):
+        super().__init__(execute_external, extra_substitutions, preamble_commands)
+
+    def execute(self, test, litConfig):
+        # Run each Shell test in a separate directory (on remote).
+
+        # Find directory change command in %lldb substitution.
+        for i, t in enumerate(test.config.substitutions):
+            if re.match(t[0], "%lldb"):
+                cmd = t[1]
+                if '-O "platform settings -w ' in cmd:
+                    # If command is present, it is added by get_lldb_args.
+                    # Replace the path with the tests' path in suite.
+                    # Example:
+                    # bin/lldb
+                    #   -O "platform shell mkdir -p /home/user/shell"
+                    #   -O "platform settings -w /home/user/shell" ...
+                    # =>
+                    # bin/lldb
+                    #   -O "platform shell mkdir -p /home/user/shell/SymbolFile/Breakpad/inline-record.test"
+                    #   -O "platform settings -w /home/user/shell/SymbolFile/Breakpad/inline-record.test" ...
+                    args_def = " ".join(get_lldb_args(test.config))
+                    args_unique = " ".join(
+                        get_lldb_args(
+                            test.config,
+                            posixpath.join(*test.path_in_suite),
+                        )
+                    )
+                    test.config.substitutions[i] = (
+                        t[0],
+                        cmd.replace(args_def, args_unique),
+                    )
+                break
+        return super().execute(test, litConfig)
+
+
 def use_lldb_substitutions(config):
     # Set up substitutions for primary tools.  These tools must come from config.lldb_tools_dir
     # which is basically the build output directory.  We do not want to find these in path or
@@ -34,7 +99,9 @@ def use_lldb_substitutions(config):
     build_script = os.path.join(build_script, "build.py")
     build_script_args = [
         build_script,
-        "--compiler=any",  # Default to best compiler
+        (
+            "--compiler=clang" if config.enable_remote else "--compiler=any"
+        ),  # Default to best compiler
         "--arch=" + str(config.lldb_bitness),
     ]
     if config.lldb_lit_tools_dir:
@@ -56,7 +123,7 @@ def use_lldb_substitutions(config):
         ToolSubst(
             "%lldb",
             command=FindTool("lldb"),
-            extra_args=["--no-lldbinit", "-S", lldb_init],
+            extra_args=get_lldb_args(config),
             unresolved="fatal",
         ),
         ToolSubst(
@@ -138,7 +205,10 @@ def use_support_substitutions(config):
     # Set up substitutions for support tools.  These tools can be overridden at the CMake
     # level (by specifying -DLLDB_LIT_TOOLS_DIR), installed, or as a last resort, we can use
     # the just-built version.
-    host_flags = ["--target=" + config.host_triple]
+    if config.enable_remote:
+        host_flags = ["--target=" + config.target_triple]
+    else:
+        host_flags = ["--target=" + config.host_triple]
     if platform.system() in ["Darwin"]:
         try:
             out = subprocess.check_output(["xcrun", "--show-sdk-path"]).strip()
@@ -165,6 +235,12 @@ def use_support_substitutions(config):
     if config.cmake_sysroot:
         host_flags += ["--sysroot={}".format(config.cmake_sysroot)]
 
+    if config.enable_remote and config.has_libcxx:
+        host_flags += [
+            "-L{}".format(config.libcxx_libs_dir),
+            "-lc++",
+        ]
+
     host_flags = " ".join(host_flags)
     config.substitutions.append(("%clang_host", "%clang " + host_flags))
     config.substitutions.append(("%clangxx_host", "%clangxx " + host_flags))

lldb/test/Shell/lit.cfg.py

@@ -21,7 +21,7 @@
 config.name = "lldb-shell"
 
 # testFormat: The test format to use to interpret tests.
-config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
+config.test_format = toolchain.ShTestLldb(not llvm_config.use_lit_shell)
 
 # suffixes: A list of file extensions to treat as test files. This is overriden
 # by individual lit.local.cfg files in the test subdirectories.
@@ -68,6 +68,13 @@
     lit_config.note("Running Shell tests in {} mode.".format(lldb_repro_mode))
     toolchain.use_lldb_repro_substitutions(config, lldb_repro_mode)
 
+if config.lldb_platform_url and config.cmake_sysroot and config.enable_remote:
+    if re.match(r".*-linux.*", config.target_triple):
+        config.available_features.add("remote-linux")
+else:
+    # After this, enable_remote == True iff remote testing is going to be used.
+    config.enable_remote = False
+
 llvm_config.use_default_substitutions()
 toolchain.use_lldb_substitutions(config)
 toolchain.use_support_substitutions(config)

lldb/test/Shell/lit.site.cfg.py.in

@@ -10,10 +10,15 @@ config.lldb_src_root = "@LLDB_SOURCE_DIR@"
 config.lldb_obj_root = "@LLDB_BINARY_DIR@"
 config.lldb_libs_dir = lit_config.substitute("@LLDB_LIBS_DIR@")
 config.lldb_tools_dir = lit_config.substitute("@LLDB_TOOLS_DIR@")
+config.lldb_platform_url = lit_config.substitute("@LLDB_TEST_PLATFORM_URL@")
+config.lldb_platform_working_dir = lit_config.substitute("@LLDB_TEST_PLATFORM_WORKING_DIR@")
 # Since it comes from the command line, it may have backslashes which
 # should not need to be escaped.
 config.lldb_lit_tools_dir = lit_config.substitute(r"@LLDB_LIT_TOOLS_DIR@")
-config.cmake_sysroot = lit_config.substitute("@CMAKE_SYSROOT@")
+config.cmake_sysroot = lit_config.substitute("@LLDB_TEST_SYSROOT@" or "@DEFAULT_SYSROOT@")
+config.has_libcxx = @LLDB_HAS_LIBCXX@
+config.enable_remote = not @LLDB_SHELL_TESTS_DISABLE_REMOTE@
+config.libcxx_libs_dir = "@LIBCXX_LIBRARY_DIR@"
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
 config.python_executable = "@Python3_EXECUTABLE@"
 config.have_zlib = @LLVM_ENABLE_ZLIB@