Feat/patch management

discopop_explorer/discopop_explorer.py

@@ -18,7 +18,8 @@
 import pstats2  # type:ignore
 from pluginbase import PluginBase  # type:ignore
 
-from discopop_library.PathManagement.PathManagement import get_path
+from discopop_library.LineMapping.initialize import initialize_line_mapping
+from discopop_library.PathManagement.PathManagement import get_path, load_file_mapping
 from discopop_library.discopop_optimizer.Microbench.ExtrapInterpolatedMicrobench import (
     ExtrapInterpolatedMicrobench,
 )
@@ -215,6 +216,9 @@ def run(arguments: ExplorerArguments):
             stats = pstats2.Stats(profile, stream=f).sort_stats("time").reverse_order()
             stats.print_stats()
 
+    # initialize the line_mapping.json
+    initialize_line_mapping(load_file_mapping(arguments.file_mapping_file), arguments.project_path)
+
     print("Time taken for pattern detection: {0}".format(end - start))
 
     # demonstration of Microbenchmark possibilities

discopop_library/LineMapping/diff_modifications.py

@@ -0,0 +1,149 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+import os.path
+import subprocess
+from typing import List, Tuple, Dict, Optional, Set
+
+from discopop_library.LineMapping.load import load_line_mapping
+from discopop_library.LineMapping.save import save_line_mapping
+
+
+def apply_line_mapping_modifications_from_files(file_id: int, original_file: str, modified_file: str):
+    """Calculates diff between original_file and modified_file and applied modifications from the diff to the line_mapping"""
+    if not os.path.exists(original_file):
+        raise FileNotFoundError(original_file)
+    if not os.path.exists(modified_file):
+        raise FileNotFoundError(original_file)
+
+    command = [
+        "diff",
+        original_file,
+        modified_file,
+    ]
+    result = subprocess.run(
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        universal_newlines=True,
+    )
+    if result.returncode == 1:
+        # print("RESULT: ", result.returncode)
+        # print("STDERR:")
+        # print(result.stderr)
+        # print("STDOUT: ")
+        # print(result.stdout)
+        diff = result.stdout
+        # print("DIFF: ", diff)
+    else:
+        diff = ""
+
+    apply_line_mapping_modifications_from_diff(file_id, diff)
+
+
+def apply_line_mapping_modifications_from_diff(file_id: int, diff: str):
+    """parse diff, apply line num modifications according to c,a,d values"""
+    # get cleaned diff
+    cleaned_diff: List[str] = []
+    for line in diff.split("\n"):
+        if line.startswith("<") or line.startswith(">") or line.startswith("-") or len(line) == 0:
+            continue
+        line = line.replace("\n", "")
+        cleaned_diff.append(line)
+
+    # get line_mapping
+    line_mapping: Dict[str, Dict[str, int]] = load_line_mapping()
+
+    deletions: Set[str] = set()
+    shifts: Dict[str, int] = dict()
+
+    # initialize
+    for key in line_mapping[str(file_id)]:
+        shifts[key] = 0
+
+    # parse diff
+    for diff_entry in cleaned_diff:
+        if "a" in diff_entry:
+            lhs = diff_entry[: diff_entry.index("a")]
+            if "," in lhs:
+                base_line = int(lhs.split(",")[0])
+            else:
+                base_line = int(lhs)
+            rhs = diff_entry[diff_entry.index("a") + 1 :]
+            if "," in rhs:
+                added_lines_count = int(rhs.split(",")[1]) - int(rhs.split(",")[0])
+            else:
+                added_lines_count = 1
+            for key in line_mapping[str(file_id)]:
+                if line_mapping[str(file_id)][key] < 0:
+                    # invalid due to deletion
+                    continue
+                if line_mapping[str(file_id)][key] > base_line:
+                    shifts[key] += added_lines_count
+
+        if "d" in diff_entry:
+            lhs = diff_entry[: diff_entry.index("d")]
+            if "," in lhs:
+                base_line = int(lhs.split(",")[0])
+                deleted_lines_count = int(lhs.split(",")[1]) - int(lhs.split(",")[0])
+            else:
+                base_line = int(lhs)
+                deleted_lines_count = 1
+            for key in line_mapping[str(file_id)]:
+                if line_mapping[str(file_id)][key] < 0:
+                    # invalid due to deletion
+                    continue
+                if line_mapping[str(file_id)][key] in range(base_line, base_line + deleted_lines_count):
+                    deletions.add(key)
+                elif line_mapping[str(file_id)][key] >= base_line + deleted_lines_count:
+                    shifts[key] -= deleted_lines_count
+
+        if "c" in diff_entry:
+            lhs = diff_entry[: diff_entry.index("c")]
+            if "," in lhs:
+                base_line = int(lhs.split(",")[0])
+                deleted_lines_count = int(lhs.split(",")[1]) - int(lhs.split(",")[0])
+            else:
+                base_line = int(lhs)
+                deleted_lines_count = 1
+            rhs = diff_entry[diff_entry.index("c") + 1 :]
+            if "," in rhs:
+                added_lines_count = int(rhs.split(",")[1]) - int(rhs.split(",")[0])
+            else:
+                added_lines_count = 1
+
+            for key in line_mapping[str(file_id)]:
+                if line_mapping[str(file_id)][key] < 0:
+                    # invalid due to deletion
+                    continue
+                if line_mapping[str(file_id)][key] > base_line:
+                    shifts[key] += added_lines_count
+
+            for key in line_mapping[str(file_id)]:
+                if line_mapping[str(file_id)][key] < 0:
+                    # invalid due to deletion
+                    continue
+                if line_mapping[str(file_id)][key] in range(base_line, base_line + deleted_lines_count):
+                    deletions.add(key)
+                elif line_mapping[str(file_id)][key] >= base_line + deleted_lines_count:
+                    shifts[key] -= deleted_lines_count
+
+    # apply deletions
+    for key in deletions:
+        line_mapping[str(file_id)][key] = -1
+
+    # apply shifts
+    for key in shifts:
+        if line_mapping[str(file_id)][key] < 0:
+            # invalid due to deletion
+            continue
+        line_mapping[str(file_id)][key] = line_mapping[str(file_id)][key] + shifts[key]
+
+    # save updated line mapping
+    save_line_mapping(line_mapping)
+
+    pass

discopop_library/LineMapping/initialize.py

@@ -0,0 +1,40 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+
+import json
+import os.path
+from pathlib import Path
+from typing import Dict
+
+from discopop_library.LineMapping.load import load_line_mapping
+from discopop_library.LineMapping.save import save_line_mapping
+
+
+def initialize_line_mapping(
+    file_mapping: Dict[int, Path],
+    discopop_path: str = ".discopop",
+):
+    """initializes the line mapping dictionary to track line shifts due to inserted pragmas.
+    The Dictionary will be stored in .discopop/line_mapping.json.
+    Line ids start with 1."""
+    line_mapping_dict: Dict[str, Dict[str, int]] = dict()
+
+    # initialize line mapping (1->1, 2->2, ...)
+    for file_id in file_mapping:
+        if str(file_id) not in line_mapping_dict:
+            line_mapping_dict[str(file_id)] = dict()
+        line_id = 1
+        with open(file_mapping[file_id], "r") as f:
+            for line in f.readlines():
+                line_mapping_dict[str(file_id)][str(line_id)] = line_id
+                line_id += 1
+
+    save_line_mapping(line_mapping_dict, discopop_path)
+
+    # debug
+    load_line_mapping(discopop_path)

discopop_library/LineMapping/load.py

@@ -0,0 +1,18 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+import json
+import os.path
+from typing import Dict
+
+
+def load_line_mapping(discopop_path: str = "") -> Dict[str, Dict[str, int]]:
+    """Loads and returns the line_mapping dictionary"""
+    line_mapping_dict: Dict[str, Dict[str, int]] = dict()
+    with open(os.path.join(discopop_path, "line_mapping.json")) as f:
+        line_mapping_dict = json.load(f)
+    return line_mapping_dict

discopop_library/LineMapping/save.py

@@ -0,0 +1,17 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+import json
+import os
+from typing import Dict
+
+
+def save_line_mapping(line_mapping_dict: Dict[str, Dict[str, int]], discopop_path: str = ""):
+    """dumps line_mapping_dict to line_mapping.json"""
+    # dump line mapping to json file
+    with open(os.path.join(discopop_path, "line_mapping.json"), "w+") as f:
+        json.dump(line_mapping_dict, f)

discopop_library/PatchApplicator/PatchApplicatorArguments.py

@@ -0,0 +1,50 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+import sys
+from dataclasses import dataclass
+from typing import List
+
+
+@dataclass
+class PatchApplicatorArguments(object):
+    """Container Class for the arguments passed to the discopop_patch_applicator"""
+
+    verbose: bool
+    apply: List[str]
+    rollback: List[str]
+    clear: bool
+    load: bool
+    list: bool
+
+    def __post_init__(self):
+        self.__validate()
+
+    def __validate(self):
+        """Validate the arguments passed to the discopop_patch_applicator, e.g check if given files exist"""
+        # check mutually exclusive arguments
+        exit_required = False
+        if self.clear and self.load:
+            print("Please use only one of `--clear` or `--load`.")
+            exit_required = True
+        if len(self.apply) > 0 and len(self.rollback) > 0:
+            print("Please use only one of '--apply' or '--rollback'.")
+            exit_required = True
+        if self.clear and (len(self.apply) > 0 or len(self.rollback)):
+            print("Please use either '--clear' or any of '--apply' or '--rollback'.")
+            exit_required = True
+        if self.load and (len(self.apply) > 0 or len(self.rollback)):
+            print("Please use either '--load' or any of '--apply' or '--rollback'.")
+            exit_required = True
+        if self.list:
+            self.apply = []
+            self.rollback = []
+            self.clear = False
+            self.load = False
+        if exit_required:
+            print("Exiting.")
+            sys.exit(0)

discopop_library/PatchApplicator/__main__.py

@@ -0,0 +1,68 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+
+from argparse import ArgumentParser
+
+from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments
+from discopop_library.PatchApplicator.patch_applicator import run
+
+
+def parse_args() -> PatchApplicatorArguments:
+    """Parse the arguments passed to the discopop_explorer"""
+    parser = ArgumentParser(description="DiscoPoP Patch Generator")
+    # all flags that are not considered stable should be added to the experimental_parser
+    experimental_parser = parser.add_argument_group(
+        "EXPERIMENTAL",
+        "Arguments for the task pattern detector and other experimental features. These flags are considered EXPERIMENTAL and they may or may not be removed or changed in the near future.",
+    )
+
+    # fmt: off
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="Enable verbose output.")
+    parser.add_argument('-a', '--apply', nargs='+', default=[], help="Apply the parallelization suggestions with the "
+                                                                     "given ids.")
+    parser.add_argument('-r', '--rollback', nargs='+', default=[], help="Roll back the application of the "
+                                                                        "parallelization suggestions with the given "
+                                                                        "ids.")
+    parser.add_argument('-C', '--clear', action="store_true", help="Reset the code to it's original state. Preserves "
+                                                                   "the list of applied suggestion for loading."
+                                                                   "Old saves will be overwritten!")
+    # todo: in the long run, saving configurations to different locations could be easily implemented.
+
+    parser.add_argument('-L', '--load', action="store_true", help="Load a previous state after clearing.")
+    parser.add_argument('-l', '--list', action="store_true", help="Show the list of applied suggestions."
+                                                                  "If set, nothing else will be done.")
+    # EXPERIMENTAL FLAGS:
+    # fmt: on
+
+    arguments = parser.parse_args()
+
+    return PatchApplicatorArguments(
+        verbose=arguments.verbose,
+        apply=arguments.apply,
+        rollback=arguments.rollback,
+        clear=arguments.clear,
+        load=arguments.load,
+        list=arguments.list,
+    )
+
+
+def main() -> int:
+    """Return values:"
+    "0: Applied successfully"
+    "1: Nothing applied"
+    "2: Some changes applied successfully
+    """
+    retval = 0
+    arguments = parse_args()
+    retval = run(arguments)
+    return retval
+
+
+if __name__ == "__main__":
+    main()