691 mangled names

CMakeLists.txt

@@ -164,5 +164,7 @@ else()
     execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_patch_applicator ${DP_LOCAL_BIN_DIR}/discopop_patch_applicator)
     message(STATUS "--> discopop_patch_generator")
     execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_patch_generator ${DP_LOCAL_BIN_DIR}/discopop_patch_generator)
+    message(STATUS "--> discopop_preprocessor")
+    execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_preprocessor ${DP_LOCAL_BIN_DIR}/discopop_preprocessor)
 
 endif()

discopop_library/PreProcessor/PreProcessorArguments.py

@@ -0,0 +1,25 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+import json
+import sys
+from dataclasses import dataclass
+from typing import List
+
+from discopop_library.ArgumentClasses.GeneralArguments import GeneralArguments
+
+
+@dataclass
+class PreProcessorArguments(GeneralArguments):
+    """Container Class for the arguments passed to the discopop_preprocessor"""
+
+    def __post_init__(self) -> None:
+        self.__validate()
+
+    def __validate(self) -> None:
+        """Validate the arguments passed to the discopop_preprocessor, e.g check if given files exist"""
+        pass

discopop_library/PreProcessor/__main__.py

@@ -0,0 +1,45 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+
+from argparse import ArgumentParser
+from discopop_library.GlobalLogger.setup import setup_logger
+from discopop_library.PreProcessor.PreProcessorArguments import PreProcessorArguments
+from discopop_library.PreProcessor.pre_processor import run
+
+
+def parse_args() -> PreProcessorArguments:
+    """Parse the arguments passed to the discopop_preprocessor"""
+    parser = ArgumentParser(description="DiscoPoP Preprocessor")
+    # all flags that are not considered stable should be added to the experimental_parser
+    experimental_parser = parser.add_argument_group(
+        "EXPERIMENTAL",
+        "Arguments for the task pattern detector and other experimental features. These flags are considered EXPERIMENTAL and they may or may not be removed or changed in the near future.",
+    )
+
+    # fmt: off
+    parser.add_argument("--log", type=str, default="WARNING", help="Specify log level: DEBUG, INFO, WARNING, ERROR, CRITICAL")
+    parser.add_argument("--write-log", action="store_true", help="Create Logfile.")
+    # EXPERIMENTAL FLAGS:
+    # fmt: on
+
+    arguments = parser.parse_args()
+
+    return PreProcessorArguments(
+        log_level=arguments.log.upper(),
+        write_log=arguments.write_log,
+    )
+
+
+def main() -> None:
+    arguments = parse_args()
+    setup_logger(arguments)
+    run(arguments)
+
+
+if __name__ == "__main__":
+    main()

discopop_library/PreProcessor/demangle/data_xml.py

@@ -0,0 +1,72 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+
+import logging
+import os
+import re
+from discopop_library.PreProcessor.PreProcessorArguments import PreProcessorArguments
+from discopop_library.PreProcessor.demangle.utilities import demangle_tag
+
+
+def demangle_data_xml(arguments: PreProcessorArguments, parent_logger: logging.Logger, cxxfilt_bin: str) -> None:
+    logger = parent_logger.getChild("Data.xml")
+    logger.info("Starting..")
+
+    # check prerequisites
+    data_xml_path = os.path.join(os.getcwd(), "profiler", "Data.xml")
+    if not os.path.exists(data_xml_path):
+        raise FileNotFoundError(data_xml_path)
+
+    processed_data_xml_path = os.path.join(os.getcwd(), "profiler", "Data.xml.processed")
+
+    line_count = 0
+    with open(data_xml_path, "r") as f:
+        for line in f.readlines():
+            line_count += 1
+
+    if os.path.exists(processed_data_xml_path):
+        os.remove(processed_data_xml_path)
+
+    with open(processed_data_xml_path, "w+") as of:
+        with open(data_xml_path, "r") as f:
+            for idx, line in enumerate(f.readlines()):
+                if idx % 1000 == 0:
+                    logger.info("Progress: " + str(round((idx / line_count) * 100, 2)) + "%")
+                line = line.replace("\n", "")
+                orig_line = line
+                # match name:  "name\=\"\S+\""
+                name_matches = re.findall(r"name\=\"\S+\"", line)
+                for name_match in name_matches:
+                    name = name_match[6:-1]
+                    name_demangled = demangle_tag(name, logger, cxxfilt_bin)
+                    if "(" in name_demangled:
+                        # function call found. remove arguments
+                        name_cleaned = name_demangled[: name_demangled.index("(")]
+                        name_demangled = name_cleaned
+                    line = line.replace(name_match, 'name="' + name_demangled + '"')
+
+                # match >...< : "\>\S+\<"
+                tag_matches = re.findall(r"\>\S+\<", line)
+                for tag_match in tag_matches:
+                    tag = tag_match[1:-1]
+                    tag_demangled = demangle_tag(tag, logger, cxxfilt_bin)
+
+                    line = line.replace(tag_match, ">" + tag_demangled + "<")
+
+                if len(orig_line) != len(line):
+                    logger.debug("line " + str(idx) + " : " + orig_line)
+                    logger.debug("line " + str(idx) + " : " + line)
+                    logger.debug("")
+
+                of.write(line + "\n")
+
+    # overwrite Data.xml
+    os.remove(data_xml_path)
+    os.rename(processed_data_xml_path, data_xml_path)
+
+    logger.info("Done.")

discopop_library/PreProcessor/demangle/dependency_metadata.py

@@ -0,0 +1,84 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+
+import logging
+import os
+import re
+from discopop_library.PreProcessor.PreProcessorArguments import PreProcessorArguments
+from discopop_library.PreProcessor.demangle.utilities import demangle_tag
+
+
+def demangle_dependency_metadata(
+    arguments: PreProcessorArguments, parent_logger: logging.Logger, cxxfilt_bin: str
+) -> None:
+    logger = parent_logger.getChild("dependency_metadata.txt")
+    logger.info("Starting..")
+
+    # check prerequisites
+    dependency_metadata_path = os.path.join(os.getcwd(), "profiler", "dependency_metadata.txt")
+    if not os.path.exists(dependency_metadata_path):
+        raise FileNotFoundError(dependency_metadata_path)
+
+    processed_dependency_metadata_path = os.path.join(os.getcwd(), "profiler", "dependency_metadata.txt.processed")
+
+    line_count = 0
+    with open(dependency_metadata_path, "r") as f:
+        for line in f.readlines():
+            line_count += 1
+
+    if os.path.exists(processed_dependency_metadata_path):
+        os.remove(processed_dependency_metadata_path)
+
+    with open(processed_dependency_metadata_path, "w+") as of:
+        with open(dependency_metadata_path, "r") as f:
+            for idx, line in enumerate(f.readlines()):
+                if idx % 100 == 0:
+                    logger.info("Progress: " + str(round((idx / line_count) * 100, 2)) + "%")
+                line = line.replace("\n", "")
+                orig_line = line
+
+                tags = line.split(" ")
+                demangled_tags = []
+                for tag in tags:
+                    # reduce calls to llvm-cxxfilt
+                    call_required = True
+                    if (
+                        tag in ["RAW", "WAR", "WAW"]
+                        or tag.startswith("IEC[")
+                        or tag.startswith("IEI[")
+                        or tag.startswith("IAC[")
+                        or tag.startswith("IAI[")
+                        or tag.startswith("SINK_ANC[")
+                        or tag.startswith("SOURCE_ANC[")
+                    ):
+                        call_required = False
+
+                    if call_required:
+                        if tag.startswith("GEPRESULT_"):
+                            demangled_tags.append(
+                                "GEPRESULT_" + demangle_tag(tag.replace("GEPRESULT_", ""), logger, cxxfilt_bin)
+                            )
+                        else:
+                            demangled_tags.append(demangle_tag(tag, logger, cxxfilt_bin))
+                    else:
+                        demangled_tags.append(tag)
+
+                line = " ".join(demangled_tags)
+
+                if len(orig_line) != len(line):
+                    logger.debug("line " + str(idx) + " : " + orig_line)
+                    logger.debug("line " + str(idx) + " : " + line)
+                    logger.debug("")
+
+                of.write(line + "\n")
+
+    # overwrite dependency_metadata.txt
+    os.remove(dependency_metadata_path)
+    os.rename(processed_dependency_metadata_path, dependency_metadata_path)
+
+    logger.info("Done.")

discopop_library/PreProcessor/demangle/driver.py

@@ -0,0 +1,42 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+
+import os
+import subprocess
+from discopop_library.PreProcessor.PreProcessorArguments import PreProcessorArguments
+
+import logging
+
+from discopop_library.PreProcessor.demangle.data_xml import demangle_data_xml
+from discopop_library.PreProcessor.demangle.dependency_metadata import demangle_dependency_metadata
+from discopop_library.PreProcessor.demangle.dynamic_dependencies import demangle_dynamic_dependencies
+from discopop_library.PreProcessor.demangle.reduction import demangle_reduction
+from discopop_library.PreProcessor.demangle.static_dependencies import demangle_static_dependencies
+
+
+def demangle(arguments: PreProcessorArguments, parent_logger: logging.Logger) -> None:
+    logger = parent_logger.getChild("Demangle")
+    logger.info("Demangling value and function names")
+
+    # get required path of llvm-cxxfilt
+    llvm_bin_dir = subprocess.run(["discopop_config_provider", "--llvm-bin-dir"], stdout=subprocess.PIPE).stdout.decode(
+        "UTF-8"
+    )
+    llvm_bin_dir = llvm_bin_dir.replace("\n", "")
+    cxxfilt_bin = os.path.join(llvm_bin_dir, "llvm-cxxfilt")
+
+    # Data.xml
+    # demangle_data_xml(arguments, logger, cxxfilt_bin)
+    # dependency_metadata.txt
+    # demangle_dependency_metadata(arguments, logger, cxxfilt_bin)
+    # dynamic_dependencies.txt
+    # demangle_dynamic_dependencies(arguments, logger, cxxfilt_bin)
+    # reduction.txt
+    demangle_reduction(arguments, logger, cxxfilt_bin)
+    # static_dependencies.txt
+    # demangle_static_dependencies(arguments, logger, cxxfilt_bin)

discopop_library/PreProcessor/demangle/dynamic_dependencies.py

@@ -0,0 +1,112 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+
+import logging
+import os
+import re
+from typing import List
+from discopop_library.PreProcessor.PreProcessorArguments import PreProcessorArguments
+from discopop_library.PreProcessor.demangle.utilities import demangle_tag
+
+
+def demangle_dynamic_dependencies(
+    arguments: PreProcessorArguments, parent_logger: logging.Logger, cxxfilt_bin: str
+) -> None:
+    logger = parent_logger.getChild("dynamic_dependencies.txt")
+    logger.info("Starting..")
+
+    # check prerequisites
+    dynamic_dependencies_path = os.path.join(os.getcwd(), "profiler", "dynamic_dependencies.txt")
+    if not os.path.exists(dynamic_dependencies_path):
+        raise FileNotFoundError(dynamic_dependencies_path)
+
+    processed_dynamic_dependencies_path = os.path.join(os.getcwd(), "profiler", "dynamic_dependencies.txt.processed")
+
+    line_count = 0
+    with open(dynamic_dependencies_path, "r") as f:
+        for line in f.readlines():
+            line_count += 1
+
+    if os.path.exists(processed_dynamic_dependencies_path):
+        os.remove(processed_dynamic_dependencies_path)
+
+    with open(processed_dynamic_dependencies_path, "w+") as of:
+        with open(dynamic_dependencies_path, "r") as f:
+            for idx, line in enumerate(f.readlines()):
+                if idx % 100 == 0:
+                    logger.info("Progress: " + str(round((idx / line_count) * 100, 2)) + "%")
+                line = line.replace("\n", "")
+                orig_line = line
+
+                tags = line.split(" ")
+                demangled_tags = []
+                for tag in tags:
+                    # reduce calls to llvm-cxxfilt
+                    call_required = True
+                    if tag in ["RAW", "WAR", "WAW", "NOM", "INIT", "END", "BGN"]:
+                        call_required = False
+
+                    if call_required:
+                        sub_tags = unpack(tag)
+                        demangled_sub_tags = []
+
+                        for sub_tag in sub_tags:
+
+                            if sub_tag.startswith("GEPRESULT_"):
+                                demangled_sub_tags.append(
+                                    "GEPRESULT_" + demangle_tag(sub_tag.replace("GEPRESULT_", ""), logger, cxxfilt_bin)
+                                )
+                            else:
+                                demangled_sub_tags.append(demangle_tag(sub_tag, logger, cxxfilt_bin))
+
+                        demangled_tags.append(pack(demangled_sub_tags))
+
+                    else:
+                        demangled_tags.append(tag)
+
+                line = " ".join(demangled_tags)
+
+                if len(orig_line) != len(line):
+                    logger.debug("line " + str(idx) + " : " + orig_line)
+                    logger.debug("line " + str(idx) + " : " + line)
+                    logger.debug("")
+
+                of.write(line + "\n")
+
+    # overwrite dynamic_dependencies.txt
+    os.remove(dynamic_dependencies_path)
+    os.rename(processed_dynamic_dependencies_path, dynamic_dependencies_path)
+
+    logger.info("Done.")
+
+
+def unpack(tag: str) -> List[str]:
+    # example: 1:2390|_ZL4tmp1(_ZL4tmp1)  --> returns ("1:2390", "_ZL4tmp1", "_ZL4tmp1")
+    if "|" in tag:
+        first = tag[: tag.index("|")]
+        tag = tag[tag.index("|") + 1 :]
+        if "(" in tag:
+            second = tag[: tag.index("(")]
+            third = tag[tag.index("(") + 1 : -1]
+            return [first, second, third]
+        else:
+            second = tag
+            return [first, second]
+    else:
+        return [tag]
+
+
+def pack(sub_tags: List[str]) -> str:
+    # example: ("1:2390", "_ZL4tmp1", "_ZL4tmp1") returns: "1:2390|_ZL4tmp1(_ZL4tmp1)"
+    if len(sub_tags) == 1:
+        return sub_tags[0]
+    if len(sub_tags) == 2:
+        return sub_tags[0] + "|" + sub_tags[1]
+    if len(sub_tags) == 3:
+        return sub_tags[0] + "|" + sub_tags[1] + "(" + sub_tags[2] + ")"
+    raise ValueError("Invalid length: " + str(len(sub_tags)))