From ebeb358c25cc5391955d7b8ae285bd222b1eb4ac Mon Sep 17 00:00:00 2001 From: Akshay K Date: Tue, 12 Oct 2021 21:28:36 -0400 Subject: [PATCH 01/13] Add action to embed compile command update weak symbol attribute Update record compiler action fix comments and compiler path --- src/blight/actions/record_compiler.py | 191 ++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 src/blight/actions/record_compiler.py diff --git a/src/blight/actions/record_compiler.py b/src/blight/actions/record_compiler.py new file mode 100644 index 0000000..aec2e99 --- /dev/null +++ b/src/blight/actions/record_compiler.py @@ -0,0 +1,191 @@ +import os +import re +import hashlib +import logging +import json +import subprocess +import shutil +from pathlib import Path +from typing import List, Dict + +from blight.action import CompilerAction, LDAction +from blight.tool import Tool +from blight.util import flock_append + + +def is_blight_env(key): + """Return True if the key is associated + with blight + """ + return key.startswith("BLIGHT") + + +def SHA256(value): + """Get sha256 hash from the value""" + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def get_compiler_with_sysroot(wrapped_tool) -> str: + """Get the compiler profile with the dummy sysroot + as options + """ + envs = os.environ.copy() + cwd_path = os.getcwd() + proc = subprocess.Popen( + [ + wrapped_tool, + "-E", + "-v", + "-x", + "c++", + "-isysroot", + cwd_path + "/xyz", + "-", + "-fsyntax-only", + "-Wno-missing-sysroot", + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=envs, + ) + value, error = proc.communicate("") + return value.decode("utf-8") + + +def get_compiler_without_sysroot(wrapped_tool) -> str: + """Get the compiler profile without sysroot""" + envs = os.environ.copy() + proc = subprocess.Popen( + [ + wrapped_tool, + "-E", + "-v", + "-x", + "c++", + "-", + "-fsyntax-only", + "-Wno-missing-sysroot", + ], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=envs, + ) + value, error = proc.communicate("") + return value.decode("utf-8") + + +def get_compiler_info(wrapped_tool) -> str: + """Get the compiler info in json format""" + compiler = dict() + sysroot = get_compiler_with_sysroot(wrapped_tool).splitlines() + no_sysroot = get_compiler_without_sysroot(wrapped_tool).splitlines() + + # replace the special character from list to avoid getting issues + # while generating string from json + compiler["sysroot"] = [path.replace('"', "'") for path in sysroot] + compiler["no_sysroot"] = [path.replace('"', "'") for path in no_sysroot] + sha256 = SHA256(json.dumps(compiler)) + + # Embed compiler info hash with the json to map it with compile command + compiler["hash"] = sha256 + return json.dumps(compiler).replace('"', '\\"').replace("\\\\", "\\"), sha256 + + +def cc_as_string(tool_record: Dict, hash_str=None): + tool_record["hash"] = hash_str + return json.dumps(tool_record).replace('"', '\\"').replace("\\\\", "\\") + + +_ALIGN = 4 +def align_string(string: str) -> str: + length = len(string) + length_aligned = length if length % _ALIGN == 0 else (int(length / _ALIGN) + 1) * _ALIGN + return string.ljust(length_aligned, '\0') + + +class CompileCommand(CompilerAction): + def __init__(self, config): + super(CompileCommand, self).__init__(config) + self._tool = None + + def _is_input_assembly(self): + has_assembly = False + if self._tool is not None: + for inpt in self._tool.inputs: + has_assembly = has_assembly or \ + str(inpt).endswith(".S") or \ + str(inpt).endswith(".s") + return has_assembly + + def _get_header_file(self): + output = Path(self._config["output"]) + tool_record = self._tool.asdict() + + cmd_hash = SHA256(json.dumps(tool_record)) + header_file = "{}/{}.h".format(output, cmd_hash) + if os.path.exists(header_file): + os.remove(header_file) + return header_file + + def _get_tool_asdict(self): + if self._tool is None: + return dict() + + tool_record = self._tool.asdict() + envs = tool_record["env"] + updated_envs = dict() + for key, value in envs.items(): + if key == "PS1" or key == "LS_COLORS" or is_blight_env(key): + continue + if isinstance(value, str): + updated_envs[key] = value.replace('"', "'") + elif isinstance(value, list): + updated_envs[key] = [v.replace('"', "'") for v in value] + else: + updated_envs[key] = value + + args = tool_record["args"] + canonicalized_args = tool_record["canonicalized_args"] + tool_record["args"] = [v.replace('"', "'") for v in args] + tool_record["canonicalized_args"] = [v.replace('"', "'") for v in canonicalized_args] + tool_record["env"] = updated_envs + tool_record["wrapped_tool"] = shutil.which(self._tool.wrapped_tool()) + return tool_record + + def before_run(self, tool: Tool) -> None: + self._tool = tool + if self._is_input_assembly(): + return + + comp_info, sha256 = get_compiler_info(self._tool.wrapped_tool()) + cc_string = cc_as_string(self._get_tool_asdict(), sha256) + + file = self._get_header_file() + with flock_append(file) as io: + variable = """ +#ifndef __linux__ + __attribute__((used,section(\"__DATA,.trailofbits_cc\"))) static const char var_{}[] = \"{}\"; + __attribute__((used,selectany,section(\"__DATA,.trailofbits_ci\"))) extern const char compinfo_{}[] = \"{}\"; +#else + __attribute__((used,section(\".trailofbits_cc, \\"S\\", @note;\\n#\"))) static const char var_{}[] = \"''{}\"; + /* __attribute__((used,section(\".trailofbits_ci, \\"S\\", @note;\\n#\"))) extern const char compinfo_{}[] __attribute__((weak)) = \"{}\"; + */ +#endif""".format( + SHA256(cc_string), + cc_string.strip(), + sha256, + comp_info.strip(), + SHA256(cc_string), + cc_string.strip(), + sha256, + comp_info.strip(), + ) + print(variable, file=io) + + tool.args += ["-include", file, + "-Wno-overlength-strings", + "-Wno-error", + "-Wno-extern-initializer", + "-Wno-unknown-escape-sequence"] From cdef64656a6686d94e866d263074081d7fdfcc8c Mon Sep 17 00:00:00 2001 From: Peter Goodman Date: Wed, 28 Jun 2023 22:09:28 -0400 Subject: [PATCH 02/13] Rename command. Fix issue. Bug fix Bug fix Use dictionary syntax Use the Tool properties directly instead of going through the dictionary. Remove leading single quotes. Not sure how those got there. There is still some kind of issue, though. More entropy to include file names Tool.env is not a thing. Go back to old way of checking for .S files. --- src/blight/actions/__init__.py | 2 + .../{record_compiler.py => embed_commands.py} | 97 +++++++++++++------ 2 files changed, 71 insertions(+), 28 deletions(-) rename src/blight/actions/{record_compiler.py => embed_commands.py} (69%) diff --git a/src/blight/actions/__init__.py b/src/blight/actions/__init__.py index fb803b1..93888bb 100644 --- a/src/blight/actions/__init__.py +++ b/src/blight/actions/__init__.py @@ -15,6 +15,7 @@ from .lint import Lint from .record import Record from .skip_strip import SkipStrip +from .embed_commands import EmbedCommands # noqa: F401 __all__ = [ "Benchmark", @@ -30,4 +31,5 @@ "Lint", "Record", "SkipStrip", + "EmbedCommands", ] diff --git a/src/blight/actions/record_compiler.py b/src/blight/actions/embed_commands.py similarity index 69% rename from src/blight/actions/record_compiler.py rename to src/blight/actions/embed_commands.py index aec2e99..b867a73 100644 --- a/src/blight/actions/record_compiler.py +++ b/src/blight/actions/embed_commands.py @@ -1,8 +1,17 @@ +""" +The `EmbedCommands` action. + +`EmbedCommands` embeds JSON compile commands, including environment variables, +inside of a custom section of each built object file. These sections make it +into the final binary. +""" + import os import re import hashlib import logging import json +import random import subprocess import shutil from pathlib import Path @@ -98,17 +107,58 @@ def cc_as_string(tool_record: Dict, hash_str=None): return json.dumps(tool_record).replace('"', '\\"').replace("\\\\", "\\") +def add_to_envp(envp: Dict, key: str, value): + if isinstance(value, str): + envp[key] = value.replace('"', "'") + elif isinstance(value, list): + envp[key] = [v.replace('"', "'") for v in value] + else: + envp[key] = value + + +def is_input_assembly(tool: Tool): + for file_path in tool.inputs: + file_path_str = str(file_path) + if file_path_str.endswith(".S") or file_path_str.endswith(".s"): + return True + return False + + +def cc_as_dict(tool: Tool) -> Dict: + env = {} + tool_dict = tool.asdict() + old_env = tool_dict["env"] + for key, value in old_env.items(): + if key == "PS1" or key == "LS_COLORS" or key.startswith("BLIGHT"): + continue + add_to_envp(env, key, value) + + for key, value in old_env.items(): + if key.startswith("BLIGHT_WRAPPED_"): + add_to_envp(env, key[15:], value) + + return { + "cwd": tool_dict["cwd"], + "env": env, + "args": [v.replace('"', "'") for v in tool.args], + "canonicalized_args": [v.replace('"', "'") for v in tool.canonicalized_args], + "wrapped_tool": shutil.which(tool.wrapped_tool()), + "lang": tool.lang.name, + } + + _ALIGN = 4 + + def align_string(string: str) -> str: length = len(string) length_aligned = length if length % _ALIGN == 0 else (int(length / _ALIGN) + 1) * _ALIGN return string.ljust(length_aligned, '\0') -class CompileCommand(CompilerAction): +class EmbedCommands(CompilerAction): def __init__(self, config): - super(CompileCommand, self).__init__(config) - self._tool = None + super(EmbedCommands, self).__init__(config) def _is_input_assembly(self): has_assembly = False @@ -121,10 +171,8 @@ def _is_input_assembly(self): def _get_header_file(self): output = Path(self._config["output"]) - tool_record = self._tool.asdict() - - cmd_hash = SHA256(json.dumps(tool_record)) - header_file = "{}/{}.h".format(output, cmd_hash) + header_file = "{}/{}_{}_{}.h".format( + output, cmd_hash, os.getpid(), random.randint(0, 9999999)) if os.path.exists(header_file): os.remove(header_file) return header_file @@ -159,32 +207,25 @@ def before_run(self, tool: Tool) -> None: if self._is_input_assembly(): return - comp_info, sha256 = get_compiler_info(self._tool.wrapped_tool()) - cc_string = cc_as_string(self._get_tool_asdict(), sha256) - - file = self._get_header_file() - with flock_append(file) as io: + if is_input_assembly(tool): + return + + cc_string = cc_as_string(cc_as_dict(tool)).strip() + cmd_hash = SHA256(cc_string) + header_file = self._get_header_file(cmd_hash) + with flock_append(header_file) as io: variable = """ #ifndef __linux__ - __attribute__((used,section(\"__DATA,.trailofbits_cc\"))) static const char var_{}[] = \"{}\"; - __attribute__((used,selectany,section(\"__DATA,.trailofbits_ci\"))) extern const char compinfo_{}[] = \"{}\"; +__attribute__((section(\"__DATA,.trailofbits_cc\"))) #else - __attribute__((used,section(\".trailofbits_cc, \\"S\\", @note;\\n#\"))) static const char var_{}[] = \"''{}\"; - /* __attribute__((used,section(\".trailofbits_ci, \\"S\\", @note;\\n#\"))) extern const char compinfo_{}[] __attribute__((weak)) = \"{}\"; - */ -#endif""".format( - SHA256(cc_string), - cc_string.strip(), - sha256, - comp_info.strip(), - SHA256(cc_string), - cc_string.strip(), - sha256, - comp_info.strip(), - ) +__attribute__((section(\".trailofbits_cc, \\"S\\", @note;\\n#\"))) +#endif +__attribute__((used)) +static const char cc_{}[] = \"{}\"; +""".format(cmd_hash, cc_string,) print(variable, file=io) - tool.args += ["-include", file, + tool.args += ["-include", header_file, "-Wno-overlength-strings", "-Wno-error", "-Wno-extern-initializer", From c4c55c1c07a43391aaf396f2d9c97dedd655a8e7 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 6 Jul 2023 15:42:42 -0400 Subject: [PATCH 03/13] EmbedCommands: lintage Signed-off-by: William Woodruff fix review comments --- src/blight/actions/__init__.py | 2 +- src/blight/actions/embed_commands.py | 164 +++++---------------------- 2 files changed, 27 insertions(+), 139 deletions(-) diff --git a/src/blight/actions/__init__.py b/src/blight/actions/__init__.py index 93888bb..03fd1d6 100644 --- a/src/blight/actions/__init__.py +++ b/src/blight/actions/__init__.py @@ -6,6 +6,7 @@ from .cc_for_cxx import CCForCXX from .demo import Demo from .embed_bitcode import EmbedBitcode +from .embed_commands import EmbedCommands # noqa: F401 from .find_inputs import FindInputs from .find_outputs import FindOutputs from .ignore_flags import IgnoreFlags @@ -15,7 +16,6 @@ from .lint import Lint from .record import Record from .skip_strip import SkipStrip -from .embed_commands import EmbedCommands # noqa: F401 __all__ = [ "Benchmark", diff --git a/src/blight/actions/embed_commands.py b/src/blight/actions/embed_commands.py index b867a73..5bdbe02 100644 --- a/src/blight/actions/embed_commands.py +++ b/src/blight/actions/embed_commands.py @@ -6,104 +6,21 @@ into the final binary. """ -import os -import re import hashlib -import logging import json +import os import random -import subprocess import shutil from pathlib import Path -from typing import List, Dict +from typing import Dict -from blight.action import CompilerAction, LDAction +from blight.action import CompilerAction +from blight.enums import Lang from blight.tool import Tool from blight.util import flock_append -def is_blight_env(key): - """Return True if the key is associated - with blight - """ - return key.startswith("BLIGHT") - - -def SHA256(value): - """Get sha256 hash from the value""" - return hashlib.sha256(value.encode("utf-8")).hexdigest() - - -def get_compiler_with_sysroot(wrapped_tool) -> str: - """Get the compiler profile with the dummy sysroot - as options - """ - envs = os.environ.copy() - cwd_path = os.getcwd() - proc = subprocess.Popen( - [ - wrapped_tool, - "-E", - "-v", - "-x", - "c++", - "-isysroot", - cwd_path + "/xyz", - "-", - "-fsyntax-only", - "-Wno-missing-sysroot", - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - env=envs, - ) - value, error = proc.communicate("") - return value.decode("utf-8") - - -def get_compiler_without_sysroot(wrapped_tool) -> str: - """Get the compiler profile without sysroot""" - envs = os.environ.copy() - proc = subprocess.Popen( - [ - wrapped_tool, - "-E", - "-v", - "-x", - "c++", - "-", - "-fsyntax-only", - "-Wno-missing-sysroot", - ], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - env=envs, - ) - value, error = proc.communicate("") - return value.decode("utf-8") - - -def get_compiler_info(wrapped_tool) -> str: - """Get the compiler info in json format""" - compiler = dict() - sysroot = get_compiler_with_sysroot(wrapped_tool).splitlines() - no_sysroot = get_compiler_without_sysroot(wrapped_tool).splitlines() - - # replace the special character from list to avoid getting issues - # while generating string from json - compiler["sysroot"] = [path.replace('"', "'") for path in sysroot] - compiler["no_sysroot"] = [path.replace('"', "'") for path in no_sysroot] - sha256 = SHA256(json.dumps(compiler)) - - # Embed compiler info hash with the json to map it with compile command - compiler["hash"] = sha256 - return json.dumps(compiler).replace('"', '\\"').replace("\\\\", "\\"), sha256 - - -def cc_as_string(tool_record: Dict, hash_str=None): - tool_record["hash"] = hash_str +def cc_as_string(tool_record: Dict): return json.dumps(tool_record).replace('"', '\\"').replace("\\\\", "\\") @@ -153,65 +70,31 @@ def cc_as_dict(tool: Tool) -> Dict: def align_string(string: str) -> str: length = len(string) length_aligned = length if length % _ALIGN == 0 else (int(length / _ALIGN) + 1) * _ALIGN - return string.ljust(length_aligned, '\0') + return string.ljust(length_aligned, "\0") class EmbedCommands(CompilerAction): - def __init__(self, config): - super(EmbedCommands, self).__init__(config) - - def _is_input_assembly(self): - has_assembly = False - if self._tool is not None: - for inpt in self._tool.inputs: - has_assembly = has_assembly or \ - str(inpt).endswith(".S") or \ - str(inpt).endswith(".s") - return has_assembly - - def _get_header_file(self): + def __init__(self, config: dict[str, str]) -> None: + super().__init__(config) + + def _get_header_file(self, cmd_hash: str) -> str: output = Path(self._config["output"]) header_file = "{}/{}_{}_{}.h".format( - output, cmd_hash, os.getpid(), random.randint(0, 9999999)) + output, cmd_hash, os.getpid(), random.randint(0, 9999999) + ) if os.path.exists(header_file): os.remove(header_file) return header_file - def _get_tool_asdict(self): - if self._tool is None: - return dict() - - tool_record = self._tool.asdict() - envs = tool_record["env"] - updated_envs = dict() - for key, value in envs.items(): - if key == "PS1" or key == "LS_COLORS" or is_blight_env(key): - continue - if isinstance(value, str): - updated_envs[key] = value.replace('"', "'") - elif isinstance(value, list): - updated_envs[key] = [v.replace('"', "'") for v in value] - else: - updated_envs[key] = value - - args = tool_record["args"] - canonicalized_args = tool_record["canonicalized_args"] - tool_record["args"] = [v.replace('"', "'") for v in args] - tool_record["canonicalized_args"] = [v.replace('"', "'") for v in canonicalized_args] - tool_record["env"] = updated_envs - tool_record["wrapped_tool"] = shutil.which(self._tool.wrapped_tool()) - return tool_record - - def before_run(self, tool: Tool) -> None: - self._tool = tool - if self._is_input_assembly(): + def before_run(self, tool: CompilerTool) -> None: + if tool.lang not in (Lang.C, Lang.Cxx): return if is_input_assembly(tool): return cc_string = cc_as_string(cc_as_dict(tool)).strip() - cmd_hash = SHA256(cc_string) + cmd_hash = hashlib.sha256(cc_string.encode()).hexdigest() header_file = self._get_header_file(cmd_hash) with flock_append(header_file) as io: variable = """ @@ -222,11 +105,16 @@ def before_run(self, tool: Tool) -> None: #endif __attribute__((used)) static const char cc_{}[] = \"{}\"; -""".format(cmd_hash, cc_string,) +""".format( + cmd_hash, + cc_string, + ) print(variable, file=io) - tool.args += ["-include", header_file, - "-Wno-overlength-strings", - "-Wno-error", - "-Wno-extern-initializer", - "-Wno-unknown-escape-sequence"] + tool.args += [ + "-include", + header_file, + "-Wno-overlength-strings", + "-Wno-error", + "-Wno-unknown-escape-sequence", + ] From 40daf196efb428a342d68426bc3fd8614a50e6b1 Mon Sep 17 00:00:00 2001 From: Akshay K Date: Thu, 16 Nov 2023 12:53:37 -0500 Subject: [PATCH 04/13] rebase and address review comments --- src/blight/actions/embed_commands.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/blight/actions/embed_commands.py b/src/blight/actions/embed_commands.py index 5bdbe02..084b4a9 100644 --- a/src/blight/actions/embed_commands.py +++ b/src/blight/actions/embed_commands.py @@ -16,15 +16,15 @@ from blight.action import CompilerAction from blight.enums import Lang -from blight.tool import Tool +from blight.tool import CompilerTool from blight.util import flock_append -def cc_as_string(tool_record: Dict): +def cc_as_string(tool_record: Dict) -> str: return json.dumps(tool_record).replace('"', '\\"').replace("\\\\", "\\") -def add_to_envp(envp: Dict, key: str, value): +def add_to_envp(envp: Dict, key: str, value) -> None: if isinstance(value, str): envp[key] = value.replace('"', "'") elif isinstance(value, list): @@ -33,7 +33,7 @@ def add_to_envp(envp: Dict, key: str, value): envp[key] = value -def is_input_assembly(tool: Tool): +def is_input_assembly(tool: CompilerTool) -> bool: for file_path in tool.inputs: file_path_str = str(file_path) if file_path_str.endswith(".S") or file_path_str.endswith(".s"): @@ -41,7 +41,7 @@ def is_input_assembly(tool: Tool): return False -def cc_as_dict(tool: Tool) -> Dict: +def cc_as_dict(tool: CompilerTool) -> Dict: env = {} tool_dict = tool.asdict() old_env = tool_dict["env"] @@ -117,4 +117,4 @@ def before_run(self, tool: CompilerTool) -> None: "-Wno-overlength-strings", "-Wno-error", "-Wno-unknown-escape-sequence", - ] + ] From 43b08e50f96e5267e4ba997cbeb27e1b1aa085d4 Mon Sep 17 00:00:00 2001 From: Akshay K Date: Thu, 16 Nov 2023 13:17:22 -0500 Subject: [PATCH 05/13] use tempfile to generate header --- src/blight/actions/embed_commands.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/blight/actions/embed_commands.py b/src/blight/actions/embed_commands.py index 084b4a9..fc8c82c 100644 --- a/src/blight/actions/embed_commands.py +++ b/src/blight/actions/embed_commands.py @@ -11,6 +11,7 @@ import os import random import shutil +import tempfile from pathlib import Path from typing import Dict @@ -79,12 +80,8 @@ def __init__(self, config: dict[str, str]) -> None: def _get_header_file(self, cmd_hash: str) -> str: output = Path(self._config["output"]) - header_file = "{}/{}_{}_{}.h".format( - output, cmd_hash, os.getpid(), random.randint(0, 9999999) - ) - if os.path.exists(header_file): - os.remove(header_file) - return header_file + header = tempfile.NamedTemporaryFile(suffix='.h', delete=False) + return header.name def before_run(self, tool: CompilerTool) -> None: if tool.lang not in (Lang.C, Lang.Cxx): From 3eb0919a8493837e650ec4d2a028c2387201ab30 Mon Sep 17 00:00:00 2001 From: Akshay K Date: Thu, 16 Nov 2023 13:51:41 -0500 Subject: [PATCH 06/13] update the format string and move to modulelevel variable --- src/blight/actions/embed_commands.py | 31 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/blight/actions/embed_commands.py b/src/blight/actions/embed_commands.py index fc8c82c..ce50ec2 100644 --- a/src/blight/actions/embed_commands.py +++ b/src/blight/actions/embed_commands.py @@ -67,6 +67,18 @@ def cc_as_dict(tool: CompilerTool) -> Dict: _ALIGN = 4 +_VARIABLE_TEMPLATE = """ +#ifndef __linux__ +__attribute__((section(\"__DATA,.trailofbits_cc\"))) +#elifndef __clang__ +__attribute__((section(\".trailofbits_cc, \\"S\\", @note;\\n#\"))) +#else +__attribute__((section(\".trailofbits_cc\"))) +#endif +__attribute__((used)) +static const char cc_{}[] = \"{}\"; +""" + def align_string(string: str) -> str: length = len(string) @@ -80,8 +92,8 @@ def __init__(self, config: dict[str, str]) -> None: def _get_header_file(self, cmd_hash: str) -> str: output = Path(self._config["output"]) - header = tempfile.NamedTemporaryFile(suffix='.h', delete=False) - return header.name + f = tempfile.NamedTemporaryFile(suffix='.h', delete=False) + return f.name def before_run(self, tool: CompilerTool) -> None: if tool.lang not in (Lang.C, Lang.Cxx): @@ -93,19 +105,8 @@ def before_run(self, tool: CompilerTool) -> None: cc_string = cc_as_string(cc_as_dict(tool)).strip() cmd_hash = hashlib.sha256(cc_string.encode()).hexdigest() header_file = self._get_header_file(cmd_hash) - with flock_append(header_file) as io: - variable = """ -#ifndef __linux__ -__attribute__((section(\"__DATA,.trailofbits_cc\"))) -#else -__attribute__((section(\".trailofbits_cc, \\"S\\", @note;\\n#\"))) -#endif -__attribute__((used)) -static const char cc_{}[] = \"{}\"; -""".format( - cmd_hash, - cc_string, - ) + with flock_append(Path(header_file)) as io: + variable = _VARIABLE_TEMPLATE.format(cmd_hash, cc_string) print(variable, file=io) tool.args += [ From 57875478107e74304b3de422078d0228962d8f5a Mon Sep 17 00:00:00 2001 From: Akshay K Date: Thu, 16 Nov 2023 14:38:29 -0500 Subject: [PATCH 07/13] fix linter issues --- src/blight/actions/embed_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/blight/actions/embed_commands.py b/src/blight/actions/embed_commands.py index ce50ec2..97a6088 100644 --- a/src/blight/actions/embed_commands.py +++ b/src/blight/actions/embed_commands.py @@ -92,7 +92,7 @@ def __init__(self, config: dict[str, str]) -> None: def _get_header_file(self, cmd_hash: str) -> str: output = Path(self._config["output"]) - f = tempfile.NamedTemporaryFile(suffix='.h', delete=False) + f = tempfile.NamedTemporaryFile(suffix=".h", delete=False) return f.name def before_run(self, tool: CompilerTool) -> None: @@ -115,4 +115,4 @@ def before_run(self, tool: CompilerTool) -> None: "-Wno-overlength-strings", "-Wno-error", "-Wno-unknown-escape-sequence", - ] + ] From e34574a491ba0b2c2f898ecbdc928ec8366c50da Mon Sep 17 00:00:00 2001 From: Akshay K Date: Thu, 16 Nov 2023 15:14:20 -0500 Subject: [PATCH 08/13] fix linter issues --- src/blight/actions/embed_commands.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/blight/actions/embed_commands.py b/src/blight/actions/embed_commands.py index 97a6088..63fd765 100644 --- a/src/blight/actions/embed_commands.py +++ b/src/blight/actions/embed_commands.py @@ -8,12 +8,10 @@ import hashlib import json -import os -import random import shutil import tempfile from pathlib import Path -from typing import Dict +from typing import Any, Dict from blight.action import CompilerAction from blight.enums import Lang @@ -25,7 +23,7 @@ def cc_as_string(tool_record: Dict) -> str: return json.dumps(tool_record).replace('"', '\\"').replace("\\\\", "\\") -def add_to_envp(envp: Dict, key: str, value) -> None: +def add_to_envp(envp: Dict, key: str, value: Any) -> None: if isinstance(value, str): envp[key] = value.replace('"', "'") elif isinstance(value, list): @@ -43,7 +41,7 @@ def is_input_assembly(tool: CompilerTool) -> bool: def cc_as_dict(tool: CompilerTool) -> Dict: - env = {} + env: Dict[str, Any] = {} tool_dict = tool.asdict() old_env = tool_dict["env"] for key, value in old_env.items(): @@ -91,7 +89,6 @@ def __init__(self, config: dict[str, str]) -> None: super().__init__(config) def _get_header_file(self, cmd_hash: str) -> str: - output = Path(self._config["output"]) f = tempfile.NamedTemporaryFile(suffix=".h", delete=False) return f.name From 9841157a3ed6887af2e127ab6a76dd59d6ccf349 Mon Sep 17 00:00:00 2001 From: Akshay K Date: Thu, 16 Nov 2023 16:17:11 -0500 Subject: [PATCH 09/13] add test for embed commands --- test/actions/test_embed_commands.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 test/actions/test_embed_commands.py diff --git a/test/actions/test_embed_commands.py b/test/actions/test_embed_commands.py new file mode 100644 index 0000000..51933bb --- /dev/null +++ b/test/actions/test_embed_commands.py @@ -0,0 +1,10 @@ +from blight.actions import EmbedCommands +from blight.tool import CC + + +def test_embed_bitcode(): + embed_bitcode = EmbedCommands({}) + cc1 = CC(["-o", "foo"]) + embed_bitcode.before_run(cc1) + cc2 = CC(["foo.S"]) + embed_bitcode.before_run(cc2) From 652b8cae9f5872545cbc785039369a644524473f Mon Sep 17 00:00:00 2001 From: Peter Goodman Date: Wed, 7 Feb 2024 00:24:18 -0500 Subject: [PATCH 10/13] Update embed_commands.py Don't convert double quotes to single quotes. --- src/blight/actions/embed_commands.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/blight/actions/embed_commands.py b/src/blight/actions/embed_commands.py index 63fd765..f350074 100644 --- a/src/blight/actions/embed_commands.py +++ b/src/blight/actions/embed_commands.py @@ -20,14 +20,14 @@ def cc_as_string(tool_record: Dict) -> str: - return json.dumps(tool_record).replace('"', '\\"').replace("\\\\", "\\") + return json.dumps(tool_record).replace('"', '\\"') def add_to_envp(envp: Dict, key: str, value: Any) -> None: if isinstance(value, str): - envp[key] = value.replace('"', "'") + envp[key] = value elif isinstance(value, list): - envp[key] = [v.replace('"', "'") for v in value] + envp[key] = [v for v in value] else: envp[key] = value @@ -56,8 +56,8 @@ def cc_as_dict(tool: CompilerTool) -> Dict: return { "cwd": tool_dict["cwd"], "env": env, - "args": [v.replace('"', "'") for v in tool.args], - "canonicalized_args": [v.replace('"', "'") for v in tool.canonicalized_args], + "args": [v for v in tool.args], + "canonicalized_args": [v for v in tool.canonicalized_args], "wrapped_tool": shutil.which(tool.wrapped_tool()), "lang": tool.lang.name, } From e1ea0f08ecadfe582e9e54758a1868c7e5c3b0c0 Mon Sep 17 00:00:00 2001 From: Peter Goodman Date: Wed, 7 Feb 2024 00:32:23 -0500 Subject: [PATCH 11/13] Update embed_commands.py --- src/blight/actions/embed_commands.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/blight/actions/embed_commands.py b/src/blight/actions/embed_commands.py index f350074..0f0954a 100644 --- a/src/blight/actions/embed_commands.py +++ b/src/blight/actions/embed_commands.py @@ -20,7 +20,7 @@ def cc_as_string(tool_record: Dict) -> str: - return json.dumps(tool_record).replace('"', '\\"') + return json.dumps(tool_record).replace('"', '\\"').replace('\\\\"', '\\"') def add_to_envp(envp: Dict, key: str, value: Any) -> None: @@ -66,9 +66,9 @@ def cc_as_dict(tool: CompilerTool) -> Dict: _ALIGN = 4 _VARIABLE_TEMPLATE = """ -#ifndef __linux__ +#if !defined(__linux__) __attribute__((section(\"__DATA,.trailofbits_cc\"))) -#elifndef __clang__ +#elif !defined(__clang__) __attribute__((section(\".trailofbits_cc, \\"S\\", @note;\\n#\"))) #else __attribute__((section(\".trailofbits_cc\"))) From 157df27f88192d85a6cd4a591ce6e549c0e9cdff Mon Sep 17 00:00:00 2001 From: Peter Goodman Date: Wed, 7 Feb 2024 00:41:14 -0500 Subject: [PATCH 12/13] Update embed_commands.py --- src/blight/actions/embed_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/blight/actions/embed_commands.py b/src/blight/actions/embed_commands.py index 0f0954a..0c164d6 100644 --- a/src/blight/actions/embed_commands.py +++ b/src/blight/actions/embed_commands.py @@ -20,7 +20,7 @@ def cc_as_string(tool_record: Dict) -> str: - return json.dumps(tool_record).replace('"', '\\"').replace('\\\\"', '\\"') + return json.dumps(tool_record).replace('\\', '\\\\').replace('"', '\\"') def add_to_envp(envp: Dict, key: str, value: Any) -> None: From d37122ba78aa2c39b27b4bac1b8c692cdc03e10f Mon Sep 17 00:00:00 2001 From: Peter Goodman Date: Thu, 21 Mar 2024 20:49:12 -0400 Subject: [PATCH 13/13] Update embed_commands.py --- src/blight/actions/embed_commands.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/src/blight/actions/embed_commands.py b/src/blight/actions/embed_commands.py index 0c164d6..64172b4 100644 --- a/src/blight/actions/embed_commands.py +++ b/src/blight/actions/embed_commands.py @@ -31,11 +31,12 @@ def add_to_envp(envp: Dict, key: str, value: Any) -> None: else: envp[key] = value +VALID_EXTENSIONS = ("c", "cc", "cxx", "cpp", "c++") -def is_input_assembly(tool: CompilerTool) -> bool: +def is_input_source_code(tool: CompilerTool) -> bool: for file_path in tool.inputs: - file_path_str = str(file_path) - if file_path_str.endswith(".S") or file_path_str.endswith(".s"): + extension = str(file_path).lower().split(".")[-1] + if extension in VALID_EXTENSIONS: return True return False @@ -63,8 +64,6 @@ def cc_as_dict(tool: CompilerTool) -> Dict: } -_ALIGN = 4 - _VARIABLE_TEMPLATE = """ #if !defined(__linux__) __attribute__((section(\"__DATA,.trailofbits_cc\"))) @@ -78,12 +77,6 @@ def cc_as_dict(tool: CompilerTool) -> Dict: """ -def align_string(string: str) -> str: - length = len(string) - length_aligned = length if length % _ALIGN == 0 else (int(length / _ALIGN) + 1) * _ALIGN - return string.ljust(length_aligned, "\0") - - class EmbedCommands(CompilerAction): def __init__(self, config: dict[str, str]) -> None: super().__init__(config) @@ -93,10 +86,7 @@ def _get_header_file(self, cmd_hash: str) -> str: return f.name def before_run(self, tool: CompilerTool) -> None: - if tool.lang not in (Lang.C, Lang.Cxx): - return - - if is_input_assembly(tool): + if not is_input_source_code(tool): return cc_string = cc_as_string(cc_as_dict(tool)).strip()