lit: Add module to produce and merge profile data for benchmarks comp…

…iled with -fprofile-instr-generate

git-svn-id: https://llvm.org/svn/llvm-project/test-suite/trunk@260357 91177308-0d34-0410-b5e6-96231b3b80d8

CMakeLists.txt

@@ -50,6 +50,20 @@ set(TEST_SUITE_RUN_UNDER "" CACHE STRING "RunSafely.sh run-under (-u) parameter"
 set(TEST_SUITE_RUN_TYPE "train" CACHE STRING
     "Type of benchmark inputs (may be test,train or ref)")
 
+# Enable profile generate mode in lit. Note that this does not automatically
+# add something like -fprofile-instr-generate to the compiler flags.
+set(TEST_SUITE_PROFILE_GENERATE "FALSE" CACHE BOOL
+    "Enable lit profile generate mode")
+set(TEST_SUITE_LLVM_PROFDATA "llvm-profdata" CACHE STRING
+    "llvm-profdata executable")
+mark_as_advanced(TEST_SUITE_LLVM_PROFDATA)
+# Set value to python style True/False
+if(TEST_SUITE_PROFILE_GENERATE)
+  set(TEST_SUITE_PROFILE_GENERATE "True")
+else()
+  set(TEST_SUITE_PROFILE_GENERATE "False")
+endif()
+
 include(MakefileFunctions)
 include(SingleMultiSource)
 find_package(YACC)

RunSafely.sh

@@ -199,10 +199,16 @@ else
   rm -f "${OUTFILE}.time${REMOTE_SUFFIX}" "${OUTFILE}${REMOTE_SUFFIX}"
   rm -f "${STDOUT_FILE}${REMOTE_SUFFIX}" "${STDERR_FILE}${REMOTE_SUFFIX}"
 
+  # Pass LLVM_PROFILE_FILE environment variable along
+  PREFIX=""
+  if [ "${LLVM_PROFILE_FILE}" != "" ]; then
+     PREFIX="LLVM_PROFILE_FILE=\"$LLVM_PROFILE_FILE\" "
+  fi
+
   # Create .command script
   COMMANDFILE="${OUTFILE}.command"
   rm -f "${COMMANDFILE}"
-  echo "$TIMEIT $TIMEITFLAGS $COMMAND" > "${COMMANDFILE}"
+  echo "${PREFIX}${TIMEIT} $TIMEITFLAGS $COMMAND" > "${COMMANDFILE}"
   chmod +x "${COMMANDFILE}"
 
   ( $RCLIENT $RFLAGS $RHOST "ls ${COMMANDFILE}" ) > /dev/null 2>&1

lit.site.cfg.in

@@ -7,5 +7,7 @@ config.remote_host = "@TEST_SUITE_REMOTE_HOST@"
 config.remote_user = "@TEST_SUITE_REMOTE_USER@"
 config.remote_port = "@TEST_SUITE_REMOTE_PORT@"
 config.run_under = "@TEST_SUITE_RUN_UNDER@"
+config.profile_generate = @TEST_SUITE_PROFILE_GENERATE@
+config.llvm_profdata = "@TEST_SUITE_LLVM_PROFDATA@"
 
 lit_config.load_config(config, "@CMAKE_SOURCE_DIR@/lit.cfg")

litsupport/profilegen.py

@@ -0,0 +1,29 @@
+import shellcommand
+
+
+def wrapScript(context, script):
+    """Adjust runscript to set a different value to the LLVM_PROFILE_FILE
+    environment variable for each execution."""
+    i = 0
+    adjusted_script = []
+    context.profilefiles = []
+    for line in script:
+        number = ""
+        if len(script) > 1:
+            number = "-%s" % (i,)
+            i += 1
+        profilefile = "%s%s.profraw" % (context.tmpBase, number)
+        prefix = "LLVM_PROFILE_FILE=%s " % quote(profilefile)
+        context.profilefiles.append(profilefile)
+        adjusted_script.append(prefix + line)
+    return adjusted_script
+
+
+def getMergeProfilesScript(context):
+    executable = shellcommand.getMainExecutable(context)
+    if not executable:
+        return None
+    profdatafile = executable + ".profdata"
+    mergecmd = [config.llvm_profdata, 'merge', '-output=%s' % profdatafile]
+    mergecmd += context.profilefiles
+    return [mergecmd]

litsupport/test.py

@@ -7,10 +7,12 @@
     applySubstitutions, getTempPaths
 from lit import Test
 from lit.util import to_bytes, to_string
-import testscript
-import runsafely
-import perf
+
 import compiletime
+import perf
+import profilegen
+import runsafely
+import testscript
 import timeit
 
 
@@ -68,6 +70,9 @@ def execute(self, test, litConfig):
 
         runscript = runsafely.wrapScript(context, runscript, suffix=".out")
 
+        if config.profile_generate:
+            runscript = profilegen.wrapScript(context, runscript)
+
         # Create the output directory if it does not already exist.
         lit.util.mkdir_p(os.path.dirname(tmpBase))
 
@@ -101,6 +106,11 @@ def execute(self, test, litConfig):
                                                  suffix=".perf.out")
             runScript(context, context.profilescript) # ignore result
 
+        # Merge llvm profile data
+        if config.profile_generate:
+            mergescript = profilegen.getMergeProfilesScript(context)
+            runScript(context, mergescript) # ignore result
+
         # Run verification script (the "VERIFY:" part)
         if len(verifyscript) > 0:
             res = runScript(context, verifyscript)