scons: Support --compiler-launcher

--compiler-launcher may be specified to employ "ccache"

Launched is propagated to 3rd-party libs, except:
 - openssl
 - ragel
 - gengetopt

SConstruct

@@ -113,6 +113,13 @@ AddOption('--compiler',
                 " supported names: empty (detect what available), {}".format(
                 ', '.join(["'{}'".format(s) for s in supported_compilers]))))
 
+AddOption('--compiler-launcher',
+          dest='compiler_launcher',
+          action='store',
+          type='string',
+          help=("optional launching tool for c and c++ compilers,"
+              " e.g. 'ccache'"))
+
 AddOption('--sanitizers',
           dest='sanitizers',
           action='store',
@@ -630,6 +637,9 @@ elif meta.compiler == 'cc':
 conf.env['LINK'] = env['CXXLD']
 conf.env['SHLINK'] = env['CXXLD']
 
+if GetOption('compiler_launcher'):
+    conf.FindProgram('COMPILER_LAUNCHER', GetOption('compiler_launcher'))
+
 if meta.platform == 'darwin':
     conf.FindTool('LIPO', [''], [('lipo', None)], required=False)
     conf.FindTool('INSTALL_NAME_TOOL', [''], [('install_name_tool', None)], required=False)
@@ -1101,11 +1111,17 @@ env.Append(LIBPATH=[env['ROC_BUILDDIR']])
 for senv in subenvs.all:
     senv.MergeFrom(env)
 
+# enable compiler launcher
+for senv in subenvs.all:
+    if 'COMPILER_LAUNCHER' in senv.Dictionary():
+        for var in ['CC', 'CXX']:
+            senv[var] = env.WrapLauncher(senv[var], senv['COMPILER_LAUNCHER'])
+
 # enable generation of compile_commands.json (a.k.a. clangdb)
 if meta.compiler in ['gcc', 'clang']:
     for senv in subenvs.all:
         for var in ['CC', 'CXX']:
-            senv[var] = env.GetClangDbWriter(senv[var], env['ROC_BUILDDIR'])
+            senv[var] = env.WrapClangDb(senv[var], env['ROC_BUILDDIR'])
 
     compile_commands = '{}/compile_commands.json'.format(env['ROC_BUILDDIR'])
 

docs/sphinx/building/developer_cookbook.rst

@@ -8,7 +8,7 @@ Developer cookbook
 Build options
 =============
 
-Developer build:
+Full developer build:
 
 .. code::
 
@@ -20,18 +20,18 @@ Explanation:
 
 * ``-Q`` enables compact colored build output
 * ``--build-3rdparty`` specifies the list of dependencies to be downloaded and built automatically
-* ``--enable-werror`` turns compiler and Doxygen warnings into error
+* ``--enable-werror`` turns compiler and Doxygen warnings into error (CI requires no warnings)
 * ``--enable-debug`` enables debug build
-* ``--enable-tests`` enables building of unit tests
-* ``--enable-benchmarks`` enables building of micro benchmarks
-* ``--enable-examples`` enables building of API usage examples
-* ``--enable-doxygen`` enables running Doxygen and producing warnings for undocumented members
+* ``--enable-tests`` enables building of unit tests (CI requires all tests to pass)
+* ``--enable-benchmarks`` enables building of micro benchmarks (not needed most time)
+* ``--enable-examples`` enables building of API usage examples (not needed most time)
+* ``--enable-doxygen`` enables running Doxygen and producing warnings for undocumented members (CI requires no warnings)
 * ``test`` is the target to run unit tests
 * ``bench`` is the target to run micro benchmarks
 
 For ``--build-3rdparty`` option, see :doc:`/building/user_cookbook`.
 
-For developer build, you may also want to automatically download and build CppUTest (for unut tests) and Google Benchmark (for micro behcmarks):
+For developer build, you may want to automatically download and build CppUTest (for unit tests) and Google Benchmark (for micro behcmarks):
 
 .. code::
 
@@ -104,6 +104,19 @@ or:
 
 The full list of the available options and variables is documented in :doc:`/building/scons_options`.
 
+Compiler launcher
+=================
+
+To speed up fresh build (i.e. subsequent builds after full clean), you can enable `ccache <https://ccache.dev/>`_.
+
+.. code::
+
+    $ scons -Q --compiler-launcher=ccache ...
+
+Here, ``--compiler-launcher`` option defines launcher program that should be used for C and C++ compilers. For example, ``gcc <args>`` will be replaced with ``ccache gcc <args>``.
+
+When ``--build-3rdparty`` is used, the specified launcher will be passed to third-party libraries as well.
+
 macOS options
 =============
 

docs/sphinx/building/scons_options.rst

@@ -47,6 +47,7 @@ Options
 --host=HOST                                    system name where Roc will run, e.g. 'arm-linux-gnueabihf', auto-detected by default
 --platform=PLATFORM                            platform name where Roc will run, supported values: empty (detect from host), 'linux', 'unix', 'darwin', 'android'
 --compiler=COMPILER                            compiler name and optional version, e.g. 'gcc-4.9', supported names: empty (detect what available), 'clang', 'gcc', 'cc'
+--compiler-launcher=COMPILER_LAUNCHER          optional launching tool for c and c++ compilers, e.g. 'ccache'
 --sanitizers=SANITIZERS                        list of gcc/clang sanitizers, supported names: empty (no sanitizers), 'all', 'undefined', 'address'
 --enable-debug                                 enable debug build for Roc
 --enable-debug-3rdparty                        enable debug build for 3rdparty libraries

scripts/scons_helpers/build-3rdparty.py

@@ -253,6 +253,12 @@ def _getvar(var, default):
             '-DCMAKE_RANLIB=' + quote(find_tool(_getvar('RANLIB', 'ranlib'))),
         ]
 
+    if ctx.env.get('COMPILER_LAUNCHER', None):
+        args += [
+            '-DCMAKE_CXX_COMPILER_LAUNCHER=' + quote(ctx.env['COMPILER_LAUNCHER']),
+            '-DCMAKE_C_COMPILER_LAUNCHER=' + quote(ctx.env['COMPILER_LAUNCHER']),
+        ]
+
     args += [
         '-DCMAKE_POSITION_INDEPENDENT_CODE=ON',
     ]
@@ -375,10 +381,18 @@ def apply_patch(ctx, dir_path, patch_url, patch_name):
         dir_path,
         '../' + patch_name), ignore_error=True)
 
-def format_vars(ctx):
+def format_vars(ctx, disable_launcher=False):
     ret = []
-    for e in ctx.unparsed_env:
-        ret.append(quote(e))
+    for k, v in ctx.env.items():
+        if k == 'COMPILER_LAUNCHER':
+            continue
+        elif k in ['CC', 'CXX'] and not disable_launcher:
+            if ctx.env.get('COMPILER_LAUNCHER', None):
+                ret.append(quote('{}={} {}'.format(k, ctx.env['COMPILER_LAUNCHER'], v)))
+            else:
+                ret.append(quote('{}={}'.format(k, v)))
+        else:
+            ret.append(quote('{}={}'.format(k, v)))
     return ' '.join(ret)
 
 def format_flags(ctx, cflags='', ldflags='', pthread=False):
@@ -448,8 +462,14 @@ def _meson_string(s):
             s = s.replace(k, v)
         return "'" + s + "'"
 
-    def _meson_flags(flags):
-        return '[{}]'.format(', '.join(map(_meson_string, flags)))
+    def _meson_list(l):
+        return '[{}]'.format(', '.join(map(_meson_string, l)))
+
+    def _meson_compiler(compiler, launcher):
+        if launcher:
+            return _meson_list([launcher, compiler])
+        else:
+            return _meson_string(compiler)
 
     msg('[generate] {}', cross_file)
 
@@ -516,8 +536,10 @@ def _meson_flags(flags):
                 cpp = {cxx}
                 ar = {ar}
         """).format(
-            cc=_meson_string(ctx.env['CC']),
-            cxx=_meson_string(ctx.env['CXX']),
+            cc=_meson_compiler(ctx.env['CC'],
+                ctx.env.get('COMPILER_LAUNCHER', None)),
+            cxx=_meson_compiler(ctx.env['CXX'],
+                ctx.env.get('COMPILER_LAUNCHER', None)),
             ar=_meson_string(ctx.env['AR'])))
 
         if pkg_config:
@@ -543,8 +565,8 @@ def _meson_flags(flags):
                 cpp_args = {cflags}
                 cpp_link_args = {ldflags}
             """).format(
-                cflags=_meson_flags(cflags),
-                ldflags=_meson_flags(ldflags)))
+                cflags=_meson_list(cflags),
+                ldflags=_meson_list(ldflags)))
 
         if ctx.toolchain:
             fp.write(textwrap.dedent("""\
@@ -1153,7 +1175,7 @@ def die(text, *args):
     changedir(ctx, 'src/openssl-{ctx.pkg_ver}')
     # see https://github.com/openssl/openssl/blob/master/INSTALL.md#configuration-options
     execute(ctx, '{vars} {flags} ./Configure {platform} {variant} {options}'.format(
-        vars=format_vars(ctx),
+        vars=format_vars(ctx, disable_launcher=True),
         flags=format_flags(ctx),
         platform=detect_openssl_platform(ctx.host),
         variant='--debug' if ctx.variant == 'debug' else '--release',

scripts/scons_plugin/__init__.py

@@ -1,5 +1,4 @@
 import scons_plugin.arguments
-import scons_plugin.clangdb
 import scons_plugin.commands
 import scons_plugin.config
 import scons_plugin.distfiles
@@ -11,6 +10,7 @@
 import scons_plugin.systemdeps
 import scons_plugin.tests
 import scons_plugin.thirdparty
+import scons_plugin.wrapper
 
 # workaround for python3
 import SCons.Subst
@@ -22,7 +22,6 @@
 def generate(env):
     modules = [
         scons_plugin.arguments,
-        scons_plugin.clangdb,
         scons_plugin.commands,
         scons_plugin.config,
         scons_plugin.distfiles,
@@ -34,6 +33,7 @@ def generate(env):
         scons_plugin.systemdeps,
         scons_plugin.tests,
         scons_plugin.thirdparty,
+        scons_plugin.wrapper,
     ]
     for m in modules:
         m.init(env)

scripts/scons_plugin/config.py

@@ -160,6 +160,20 @@ def CheckCompilerOptionSupported(context, opt, language):
         context.Result('no')
         return False
 
+def FindProgram(context, var, tool):
+    if tool:
+        context.Message("Searching {} executable ... ".format(var))
+
+        tool_path = context.env.Which(tool)
+        if tool_path:
+            context.env[var] = tool_path[0]
+        else:
+            context.env[var] = tool
+
+        context.Result(context.env[var])
+
+    return True
+
 def FindTool(context, var, toolchains, commands,
              compiler_dir=None, prepend_path=[], required=True):
     env = context.env
@@ -501,6 +515,7 @@ def init(env):
         'CheckProg': CheckProg,
         'CheckCanRunProgs': CheckCanRunProgs,
         'CheckCompilerOptionSupported': CheckCompilerOptionSupported,
+        'FindProgram': FindProgram,
         'FindTool': FindTool,
         'FindClangFormat': FindClangFormat,
         'FindLLVMDir': FindLLVMDir,

scripts/scons_plugin/thirdparty.py

@@ -28,6 +28,9 @@ def _build_thirdparty(env, versions, name, deps, is_native):
         'RANLIB=' + quote(env['RANLIB']),
     ]
 
+    if 'COMPILER_LAUNCHER' in env.Dictionary():
+        env_vars += ['COMPILER_LAUNCHER=' + quote(env['COMPILER_LAUNCHER'])]
+
     if 'PKG_CONFIG' in env.Dictionary():
         env_vars += ['PKG_CONFIG=' + quote(env['PKG_CONFIG'])]
 

scripts/scons_plugin/wrapper.py

@@ -0,0 +1,23 @@
+import os
+
+def WrapLauncher(env, tool, launcher):
+    return '"{launcher}" "{tool}"'.format(
+        launcher=launcher, tool=tool)
+
+def WrapClangDb(env, tool, build_dir):
+    if not tool.startswith('"'):
+        tool = '"{}"'.format(tool)
+
+    return (
+        '{python_cmd} scripts/scons_helpers/clangdb.py '
+        '"{root_dir}" "{build_dir}" {tool}'.format(
+            python_cmd=env.GetPythonExecutable(),
+            root_dir=env.Dir('#').path,
+            build_dir=env.Dir(build_dir).path,
+            tool=tool
+        )
+    )
+
+def init(env):
+    env.AddMethod(WrapLauncher, 'WrapLauncher')
+    env.AddMethod(WrapClangDb, 'WrapClangDb')