Adds the ability to use absolute paths for bazel managed artifacts

refresh.template.py

@@ -49,6 +49,11 @@ class SGR(enum.Enum):
     FG_BLUE = '\033[0;34m'
 
 
+class BazelInfo(typing.NamedTuple):
+    output_path: pathlib.Path
+    external_path: pathlib.Path
+
+
 def _log_with_sgr(sgr, colored_message, uncolored_message=''):
     """Log a message to stderr wrapped in an SGR context."""
     print(sgr.value, colored_message, SGR.RESET.value, uncolored_message, sep='', file=sys.stderr, flush=True)
@@ -765,7 +770,42 @@ def _all_platform_patch(compile_args: typing.List[str]):
     return compile_args
 
 
-def _get_cpp_command_for_files(compile_action):
+def _path_replacement_for_arg(arg: str, replacements: typing.Mapping[str, str]):
+    for (prefix, replacement) in replacements.items():
+        # Some commands are output with the argument name and the argument value
+        # split across two arguments. This condition checks for these cases by
+        # detecting arguments that start with e.g. bazel-out/.
+        #
+        # Example: -o, bazel-out/...
+        if arg.startswith(prefix):
+            return replacement + arg[len(prefix):]
+        # Bazel adds directories to include search paths using -I options. This
+        # condition checks for these cases.
+        #
+        # See: https://clang.llvm.org/docs/ClangCommandLineReference.html#cmdoption-clang-I-dir
+        #
+        # Example: -Ibazel-out/...
+        elif arg.startswith('-I' + prefix):
+            return replacement + arg[2 + len(prefix):]
+        # Some commands are output with the argument name and the argument value
+        # combined within the same argument, seperated by an equals sign.
+        #
+        # Example: -frandom-seed=bazel-out/...
+        elif '={}'.format(prefix) in arg:
+            return arg.replace('={}'.format(prefix), '={}'.format(replacement), 1)
+    return arg
+
+
+def _apply_path_replacements(compile_args: typing.List[str], bazel_info: BazelInfo):
+    replacements = {
+        "bazel-out/": os.fspath(bazel_info.output_path) + "/",
+        "external/": os.fspath(bazel_info.external_path) + "/",
+    }
+
+    return [_path_replacement_for_arg(arg, replacements) for arg in compile_args]
+
+
+def _get_cpp_command_for_files(compile_action, bazel_info):
     """Reformat compile_action into a compile command clangd can understand.
 
     Undo Bazel-isms and figures out which files clangd should apply the command to.
@@ -775,12 +815,15 @@ def _get_cpp_command_for_files(compile_action):
     compile_action.arguments = _apple_platform_patch(compile_action.arguments)
     # Android and Linux and grailbio LLVM toolchains: Fine as is; no special patching needed.
 
+    if {rewrite_bazel_paths}:
+        compile_action.arguments = _apply_path_replacements(compile_action.arguments, bazel_info)
+
     source_files, header_files = _get_files(compile_action)
 
     return source_files, header_files, compile_action.arguments
 
 
-def _convert_compile_commands(aquery_output):
+def _convert_compile_commands(aquery_output, bazel_info):
     """Converts from Bazel's aquery format to de-Bazeled compile_commands.json entries.
 
     Input: jsonproto output from aquery, pre-filtered to (Objective-)C(++) compile actions for a given build.
@@ -799,12 +842,15 @@ def _convert_compile_commands(aquery_output):
             assert not target.startswith('//external'), f"Expecting external targets will start with @. Found //external for action {action}, target {target}"
             action.is_external = target.startswith('@') and not target.startswith('@//')
 
+    def worker(compile_action):
+        return _get_cpp_command_for_files(compile_action, bazel_info)
+
     # Process each action from Bazelisms -> file paths and their clang commands
     # Threads instead of processes because most of the execution time is farmed out to subprocesses. No need to sidestep the GIL. Might change after https://github.com/clangd/clangd/issues/123 resolved
     with concurrent.futures.ThreadPoolExecutor(
         max_workers=min(32, (os.cpu_count() or 1) + 4) # Backport. Default in MIN_PY=3.8. See "using very large resources implicitly on many-core machines" in https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor
     ) as threadpool:
-        outputs = threadpool.map(_get_cpp_command_for_files, aquery_output.actions)
+        outputs = threadpool.map(worker, aquery_output.actions)
 
     # Yield as compile_commands.json entries
     header_files_already_written = set()
@@ -924,7 +970,25 @@ def _get_commands(target: str, flags: str):
     Continuing gracefully...""")
         return
 
-    yield from _convert_compile_commands(parsed_aquery_output)
+    output_path_process = subprocess.run(
+        ['bazel', 'info', 'output_path'],
+        capture_output=True,
+        encoding=locale.getpreferredencoding(),
+    )
+
+    output_path = pathlib.Path(output_path_process.stdout.strip())
+
+    output_base_process = subprocess.run(
+        ['bazel', 'info', 'output_base'],
+        capture_output=True,
+        encoding=locale.getpreferredencoding(),
+    )
+
+    output_base = pathlib.Path(output_base_process.stdout.strip())
+    external_path = output_base.joinpath("external")
+
+    bazel_info = BazelInfo(output_path, external_path)
+    yield from _convert_compile_commands(parsed_aquery_output, bazel_info)
 
 
     # Log clear completion messages
@@ -1052,8 +1116,12 @@ def _ensure_cwd_is_workspace_root():
 
 def main():
     _ensure_cwd_is_workspace_root()
-    _ensure_gitignore_entries_exist()
-    _ensure_external_workspaces_link_exists()
+
+    if {update_gitignore}:
+        _ensure_gitignore_entries_exist()
+
+    if {link_external}:
+        _ensure_external_workspaces_link_exists()
 
     target_flag_pairs = [
         # Begin: template filled by Bazel

refresh_compile_commands.bzl

@@ -49,6 +49,15 @@ refresh_compile_commands(
         # exclude_headers = "external",
     # Still not fast enough?
         # Make sure you're specifying just the targets you care about by setting `targets`, above.
+
+    # refresh_compile_commands will automatically update .git/info/exclude by default.
+        # You can disable this behavior with update_gitignore = False.
+
+    # refresh_compile_commands will automatically create a symlink for external workspaces at /external.
+        # You can disable this behavior with link_external = False.
+
+    # refresh_compile_commands does not work with --experimental_convenience_symlinks=ignore.
+        # For these workspaces, you can use absolute paths to the bazel build artifacts by setting rewrite_bazel_paths = True.
 ```
 """
 
@@ -63,6 +72,9 @@ def refresh_compile_commands(
         targets = None,
         exclude_headers = None,
         exclude_external_sources = False,
+        update_gitignore = True,
+        link_external = True,
+        rewrite_bazel_paths = False,
         **kwargs):  # For the other common attributes. Tags, compatible_with, etc. https://docs.bazel.build/versions/main/be/common-definitions.html#common-attributes.
     # Convert the various, acceptable target shorthands into the dictionary format
     # In Python, `type(x) == y` is an antipattern, but [Starlark doesn't support inheritance](https://bazel.build/rules/language), so `isinstance` doesn't exist, and this is the correct way to switch on type.
@@ -88,7 +100,16 @@ def refresh_compile_commands(
 
     # Generate the core, runnable python script from refresh.template.py
     script_name = name + ".py"
-    _expand_template(name = script_name, labels_to_flags = targets, exclude_headers = exclude_headers, exclude_external_sources = exclude_external_sources, **kwargs)
+    _expand_template(
+        name = script_name,
+        labels_to_flags = targets,
+        exclude_headers = exclude_headers,
+        exclude_external_sources = exclude_external_sources,
+        update_gitignore = update_gitignore,
+        link_external = link_external,
+        rewrite_bazel_paths = rewrite_bazel_paths,
+        **kwargs
+    )
 
     # Combine them so the wrapper calls the main script
     native.py_binary(
@@ -115,6 +136,9 @@ def _expand_template_impl(ctx):
             "        {windows_default_include_paths}": "\n".join(["        %r," % path for path in find_cpp_toolchain(ctx).built_in_include_directories]),  # find_cpp_toolchain is from https://docs.bazel.build/versions/main/integrating-with-rules-cc.html
             "{exclude_headers}": repr(ctx.attr.exclude_headers),
             "{exclude_external_sources}": repr(ctx.attr.exclude_external_sources),
+            "{update_gitignore}": repr(ctx.attr.update_gitignore),
+            "{link_external}": repr(ctx.attr.link_external),
+            "{rewrite_bazel_paths}": repr(ctx.attr.rewrite_bazel_paths),
         },
     )
     return DefaultInfo(files = depset([script]))
@@ -124,6 +148,9 @@ _expand_template = rule(
         "labels_to_flags": attr.string_dict(mandatory = True),  # string keys instead of label_keyed because Bazel doesn't support parsing wildcard target patterns (..., *, :all) in BUILD attributes.
         "exclude_external_sources": attr.bool(default = False),
         "exclude_headers": attr.string(values = ["all", "external", ""]),  # "" needed only for compatibility with Bazel < 3.6.0
+        "update_gitignore": attr.bool(default = True),
+        "link_external": attr.bool(default = True),
+        "rewrite_bazel_paths": attr.bool(default = False),
         "_script_template": attr.label(allow_single_file = True, default = "refresh.template.py"),
         # For Windows INCLUDE. If this were eliminated, for example by the resolution of https://github.com/clangd/clangd/issues/123, we'd be able to just use a macro and skylib's expand_template rule: https://github.com/bazelbuild/bazel-skylib/pull/330
         # Once https://github.com/bazelbuild/bazel/pull/17108 is widely released, we should be able to eliminate this and get INCLUDE directly. Perhaps for 7.0? Should be released in the sucessor to 6.0