From fd9c5d95235e0b3edeec4f742d167a896444a05d Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Wed, 25 Oct 2023 17:53:59 +0200 Subject: [PATCH 01/23] feat: generation of patches from individual suggestions --- discopop_library/PatchGenerator/__init__.py | 0 discopop_library/PatchGenerator/__main__.py | 40 ++++++++++ discopop_library/PatchGenerator/diffs.py | 64 ++++++++++++++++ .../PatchGenerator/patch_generator.py | 74 +++++++++++++++++++ setup.py | 1 + 5 files changed, 179 insertions(+) create mode 100644 discopop_library/PatchGenerator/__init__.py create mode 100644 discopop_library/PatchGenerator/__main__.py create mode 100644 discopop_library/PatchGenerator/diffs.py create mode 100644 discopop_library/PatchGenerator/patch_generator.py diff --git a/discopop_library/PatchGenerator/__init__.py b/discopop_library/PatchGenerator/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/discopop_library/PatchGenerator/__main__.py b/discopop_library/PatchGenerator/__main__.py new file mode 100644 index 000000000..46b2e5ac5 --- /dev/null +++ b/discopop_library/PatchGenerator/__main__.py @@ -0,0 +1,40 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. + +from argparse import ArgumentParser + +from discopop_library.PatchGenerator.patch_generator import PatchGeneratorArguments, run + + +def parse_args() -> PatchGeneratorArguments: + """Parse the arguments passed to the discopop_explorer""" + parser = ArgumentParser(description="DiscoPoP Patch Generator") + # all flags that are not considered stable should be added to the experimental_parser + experimental_parser = parser.add_argument_group( + "EXPERIMENTAL", + "Arguments for the task pattern detector and other experimental features. These flags are considered EXPERIMENTAL and they may or may not be removed or changed in the near future.", + ) + + # fmt: off + parser.add_argument("--verbose", action="store_true", + help="Enable verbose output.") + # EXPERIMENTAL FLAGS: + # fmt: on + + arguments = parser.parse_args() + + return PatchGeneratorArguments(verbose=arguments.verbose) + + +def main(): + arguments = parse_args() + run(arguments) + + +if __name__ == "__main__": + main() diff --git a/discopop_library/PatchGenerator/diffs.py b/discopop_library/PatchGenerator/diffs.py new file mode 100644 index 000000000..b80be2f57 --- /dev/null +++ b/discopop_library/PatchGenerator/diffs.py @@ -0,0 +1,64 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. +import os.path +import subprocess +from pathlib import Path +from typing import Dict + +from discopop_library.PatchGenerator.patch_generator import PatchGeneratorArguments + + +def get_diffs_from_modified_code( + file_mapping: Dict[int, Path], file_id_to_modified_code: Dict[int, str], arguments: PatchGeneratorArguments +) -> Dict[int, str]: + patches: Dict[int, str] = dict() + for file_id in file_id_to_modified_code: + # get path to original code + original_file_path = file_mapping[file_id] + # create temporary modified code + modified_file_path = original_file_path.parent / (original_file_path.name + ".discopop_patch_generator.temp") + if arguments.verbose: + print("Original: ", original_file_path) + print("Modified: ", modified_file_path) + + with open(modified_file_path, "w") as f: + f.write(file_id_to_modified_code[file_id]) + + # calculate diff + diff_name = original_file_path.parent / (original_file_path.name + ".discopop_patch_generator.diff") + command = [ + "diff", + "-Naru", + original_file_path.as_posix(), + modified_file_path.as_posix(), + ] + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=os.getcwd(), + ) + if result.returncode != 0: + if arguments.verbose: + print("RESULT: ", result.returncode) + print("STDERR:") + print(result.stderr) + print("STDOUT: ") + print(result.stdout) + + # save diff + patches[file_id] = result.stdout + + # cleanup environment + if os.path.exists(modified_file_path): + os.remove(modified_file_path) + if os.path.exists(diff_name): + os.remove(diff_name) + + return patches diff --git a/discopop_library/PatchGenerator/patch_generator.py b/discopop_library/PatchGenerator/patch_generator.py new file mode 100644 index 000000000..34f545a20 --- /dev/null +++ b/discopop_library/PatchGenerator/patch_generator.py @@ -0,0 +1,74 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. +import os.path +from dataclasses import dataclass +from typing import Dict + +from discopop_library.CodeGenerator.CodeGenerator import from_json_strings +from discopop_library.JSONHandler.JSONHandler import read_patterns_from_json_to_json +from discopop_library.PatchGenerator.diffs import get_diffs_from_modified_code +from discopop_library.PathManagement.PathManagement import load_file_mapping + + +@dataclass +class PatchGeneratorArguments(object): + """Container Class for the arguments passed to the discopop_patch_generator""" + + verbose: bool + + def __post_init__(self): + self.__validate() + + def __validate(self): + """Validate the arguments passed to the discopop_explorer, e.g check if given files exist""" + pass + + +def run(arguments: PatchGeneratorArguments): + if arguments.verbose: + print("Started DiscoPoP Patch Generator...") + pattern_file_path = os.path.join(os.getcwd(), "explorer", "patterns.json") + if not os.path.exists(pattern_file_path): + raise FileNotFoundError( + "No pattern file found. Please execute the discopop_explorer in advance." + + "\nExpected pattern file: " + + pattern_file_path + ) + file_mapping_path = os.path.join(os.getcwd(), "FileMapping.txt") + if not os.path.exists(file_mapping_path): + raise FileNotFoundError( + "No file mapping found. Please execute the discopop_explorer in advance." + + "\nExpected file: " + + file_mapping_path + ) + + if arguments.verbose: + print("Loading file mapping...") + file_mapping = load_file_mapping(file_mapping_path) + + if arguments.verbose: + print("Loading patterns...") + patterns_by_type = read_patterns_from_json_to_json(pattern_file_path, []) + if arguments.verbose: + print("Patterns: ", patterns_by_type) + + # generate code modifications from each suggestion, create a patch and store the patch + # using the suggestions unique id + if arguments.verbose: + print("Generating modified code...") + for suggestion_type in patterns_by_type: + for suggestion in patterns_by_type[suggestion_type]: + if arguments.verbose: + print("Suggestion: ", suggestion) + file_id_to_modified_code: Dict[int, str] = from_json_strings(file_mapping, {suggestion_type: [suggestion]}) + # create patches from the modified codes + file_id_to_patches: Dict[int, str] = get_diffs_from_modified_code( + file_mapping, file_id_to_modified_code, arguments + ) + if arguments.verbose: + print("Patches: ", file_id_to_patches) diff --git a/setup.py b/setup.py index 816dd2e22..3007e78ad 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ "discopop_wizard=discopop_wizard.__main__:main", "discopop_code_generator=discopop_library.CodeGenerator.__main__:main", "discopop_optimizer=discopop_library.discopop_optimizer.__main__:main", + "discopop_patch_generator=discopop_library.PatchGenerator.__main__:main", ] }, zip_safe=True, From 7b0e1adc9446ab9102782a6df5da36c17a9e6d84 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Wed, 25 Oct 2023 18:10:11 +0200 Subject: [PATCH 02/23] feat: create and store patches --- .../PatchGenerator/PatchGeneratorArguments.py | 22 +++++++++++ discopop_library/PatchGenerator/__main__.py | 3 +- discopop_library/PatchGenerator/diffs.py | 2 +- .../PatchGenerator/patch_generator.py | 39 ++++++++++++------- 4 files changed, 49 insertions(+), 17 deletions(-) create mode 100644 discopop_library/PatchGenerator/PatchGeneratorArguments.py diff --git a/discopop_library/PatchGenerator/PatchGeneratorArguments.py b/discopop_library/PatchGenerator/PatchGeneratorArguments.py new file mode 100644 index 000000000..6887aeb05 --- /dev/null +++ b/discopop_library/PatchGenerator/PatchGeneratorArguments.py @@ -0,0 +1,22 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. +from dataclasses import dataclass + + +@dataclass +class PatchGeneratorArguments(object): + """Container Class for the arguments passed to the discopop_patch_generator""" + + verbose: bool + + def __post_init__(self): + self.__validate() + + def __validate(self): + """Validate the arguments passed to the discopop_explorer, e.g check if given files exist""" + pass diff --git a/discopop_library/PatchGenerator/__main__.py b/discopop_library/PatchGenerator/__main__.py index 46b2e5ac5..8b171928f 100644 --- a/discopop_library/PatchGenerator/__main__.py +++ b/discopop_library/PatchGenerator/__main__.py @@ -8,7 +8,8 @@ from argparse import ArgumentParser -from discopop_library.PatchGenerator.patch_generator import PatchGeneratorArguments, run +from discopop_library.PatchGenerator.PatchGeneratorArguments import PatchGeneratorArguments +from discopop_library.PatchGenerator.patch_generator import run def parse_args() -> PatchGeneratorArguments: diff --git a/discopop_library/PatchGenerator/diffs.py b/discopop_library/PatchGenerator/diffs.py index b80be2f57..c12f5857d 100644 --- a/discopop_library/PatchGenerator/diffs.py +++ b/discopop_library/PatchGenerator/diffs.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Dict -from discopop_library.PatchGenerator.patch_generator import PatchGeneratorArguments +from discopop_library.PatchGenerator.PatchGeneratorArguments import PatchGeneratorArguments def get_diffs_from_modified_code( diff --git a/discopop_library/PatchGenerator/patch_generator.py b/discopop_library/PatchGenerator/patch_generator.py index 34f545a20..31cd8e675 100644 --- a/discopop_library/PatchGenerator/patch_generator.py +++ b/discopop_library/PatchGenerator/patch_generator.py @@ -5,33 +5,27 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. +import json import os.path -from dataclasses import dataclass +import shutil from typing import Dict from discopop_library.CodeGenerator.CodeGenerator import from_json_strings from discopop_library.JSONHandler.JSONHandler import read_patterns_from_json_to_json +from discopop_library.PatchGenerator.PatchGeneratorArguments import PatchGeneratorArguments from discopop_library.PatchGenerator.diffs import get_diffs_from_modified_code from discopop_library.PathManagement.PathManagement import load_file_mapping -@dataclass -class PatchGeneratorArguments(object): - """Container Class for the arguments passed to the discopop_patch_generator""" - - verbose: bool - - def __post_init__(self): - self.__validate() - - def __validate(self): - """Validate the arguments passed to the discopop_explorer, e.g check if given files exist""" - pass - - def run(arguments: PatchGeneratorArguments): if arguments.verbose: print("Started DiscoPoP Patch Generator...") + if arguments.verbose: + print("Creating patch_generator directory...") + patch_generator_dir = os.path.join(os.getcwd(), "patch_generator") + if not os.path.exists(patch_generator_dir): + os.mkdir(patch_generator_dir) + pattern_file_path = os.path.join(os.getcwd(), "explorer", "patterns.json") if not os.path.exists(pattern_file_path): raise FileNotFoundError( @@ -72,3 +66,18 @@ def run(arguments: PatchGeneratorArguments): ) if arguments.verbose: print("Patches: ", file_id_to_patches) + # clear old results and save patches + suggestion_dict = json.loads(suggestion) + suggestion_id = suggestion_dict["pattern_id"] + suggestion_folder_path = os.path.join(patch_generator_dir, str(suggestion_id)) + if arguments.verbose: + print("Saving patches for suggestion: ", suggestion_id) + if os.path.exists(suggestion_folder_path): + shutil.rmtree(suggestion_folder_path) + os.mkdir(suggestion_folder_path) + for file_id in file_id_to_patches: + patch_path = os.path.join(suggestion_folder_path, str(file_id) + ".patch") + with open(patch_path, "w") as f: + f.write(file_id_to_patches[file_id]) + if arguments.verbose: + print("Done.") From 349296eea1e6d875905e1d3be399a471ac866401 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Thu, 26 Oct 2023 12:04:37 +0200 Subject: [PATCH 03/23] feat(patch_applicator): prepared program structure --- .../PatchApplicatorArguments.py | 44 +++++++++++++ discopop_library/PatchApplicator/__init__.py | 0 discopop_library/PatchApplicator/__main__.py | 57 +++++++++++++++++ discopop_library/PatchApplicator/apply.py | 15 +++++ discopop_library/PatchApplicator/clear.py | 9 +++ discopop_library/PatchApplicator/load.py | 9 +++ .../PatchApplicator/patch_applicator.py | 63 +++++++++++++++++++ discopop_library/PatchApplicator/rollback.py | 9 +++ setup.py | 1 + 9 files changed, 207 insertions(+) create mode 100644 discopop_library/PatchApplicator/PatchApplicatorArguments.py create mode 100644 discopop_library/PatchApplicator/__init__.py create mode 100644 discopop_library/PatchApplicator/__main__.py create mode 100644 discopop_library/PatchApplicator/apply.py create mode 100644 discopop_library/PatchApplicator/clear.py create mode 100644 discopop_library/PatchApplicator/load.py create mode 100644 discopop_library/PatchApplicator/patch_applicator.py create mode 100644 discopop_library/PatchApplicator/rollback.py diff --git a/discopop_library/PatchApplicator/PatchApplicatorArguments.py b/discopop_library/PatchApplicator/PatchApplicatorArguments.py new file mode 100644 index 000000000..a55fc78f3 --- /dev/null +++ b/discopop_library/PatchApplicator/PatchApplicatorArguments.py @@ -0,0 +1,44 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. +import sys +from dataclasses import dataclass +from typing import List + + +@dataclass +class PatchApplicatorArguments(object): + """Container Class for the arguments passed to the discopop_patch_applicator""" + + verbose: bool + apply: List[str] + rollback: List[str] + clear: bool + load: bool + + def __post_init__(self): + self.__validate() + + def __validate(self): + """Validate the arguments passed to the discopop_patch_applicator, e.g check if given files exist""" + # check mutually exclusive arguments + exit_required = False + if self.clear and self.load: + print("Please use only one of `--clear` or `--load`.") + exit_required = True + if len(self.apply) > 0 and len(self.rollback) > 0: + print("Please use only one of '--apply' or '--rollback'.") + exit_required = True + if self.clear and (len(self.apply) > 0 or len(self.rollback)): + print("Please use either '--clear' or any of '--apply' or '--rollback'.") + exit_required = True + if self.load and (len(self.apply) > 0 or len(self.rollback)): + print("Please use either '--load' or any of '--apply' or '--rollback'.") + exit_required = True + if exit_required: + print("Exiting.") + sys.exit(0) diff --git a/discopop_library/PatchApplicator/__init__.py b/discopop_library/PatchApplicator/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/discopop_library/PatchApplicator/__main__.py b/discopop_library/PatchApplicator/__main__.py new file mode 100644 index 000000000..17b4dd959 --- /dev/null +++ b/discopop_library/PatchApplicator/__main__.py @@ -0,0 +1,57 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. + +from argparse import ArgumentParser + +from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments +from discopop_library.PatchApplicator.patch_applicator import run + + +def parse_args() -> PatchApplicatorArguments: + """Parse the arguments passed to the discopop_explorer""" + parser = ArgumentParser(description="DiscoPoP Patch Generator") + # all flags that are not considered stable should be added to the experimental_parser + experimental_parser = parser.add_argument_group( + "EXPERIMENTAL", + "Arguments for the task pattern detector and other experimental features. These flags are considered EXPERIMENTAL and they may or may not be removed or changed in the near future.", + ) + + # fmt: off + parser.add_argument("-v", "--verbose", action="store_true", + help="Enable verbose output.") + parser.add_argument('-a', '--apply', nargs='+', default=[], help="Apply the parallelization suggestions with the " + "given ids.") + parser.add_argument('-r', '--rollback', nargs='+', default=[], help="Roll back the application of the " + "parallelization suggestions with the given " + "ids. If this process fails, all applied " + "suggestions until the given ones will be " + "rolled back as well.") + parser.add_argument('-c', '--clear', action="store_true", help="Reset the code to it's original state. Preserves " + "the list of applied suggestion for loading.") + parser.add_argument('-l', '--load', action="store_true", help="Load a previous state after clearing.") + # EXPERIMENTAL FLAGS: + # fmt: on + + arguments = parser.parse_args() + + return PatchApplicatorArguments( + verbose=arguments.verbose, + apply=arguments.apply, + rollback=arguments.rollback, + clear=arguments.clear, + load=arguments.load, + ) + + +def main(): + arguments = parse_args() + run(arguments) + + +if __name__ == "__main__": + main() diff --git a/discopop_library/PatchApplicator/apply.py b/discopop_library/PatchApplicator/apply.py new file mode 100644 index 000000000..84948d34c --- /dev/null +++ b/discopop_library/PatchApplicator/apply.py @@ -0,0 +1,15 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. +from pathlib import Path +from typing import Dict + +from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments + + +def apply_patches(file_mapping: Dict[int, Path], arguments: PatchApplicatorArguments): + raise NotImplementedError() diff --git a/discopop_library/PatchApplicator/clear.py b/discopop_library/PatchApplicator/clear.py new file mode 100644 index 000000000..584dccf39 --- /dev/null +++ b/discopop_library/PatchApplicator/clear.py @@ -0,0 +1,9 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. +def clear_patches(): + raise NotImplementedError() diff --git a/discopop_library/PatchApplicator/load.py b/discopop_library/PatchApplicator/load.py new file mode 100644 index 000000000..f2c770729 --- /dev/null +++ b/discopop_library/PatchApplicator/load.py @@ -0,0 +1,9 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. +def load_patches(): + raise NotImplementedError() diff --git a/discopop_library/PatchApplicator/patch_applicator.py b/discopop_library/PatchApplicator/patch_applicator.py new file mode 100644 index 000000000..45263b6d4 --- /dev/null +++ b/discopop_library/PatchApplicator/patch_applicator.py @@ -0,0 +1,63 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. +import json +import os + +from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments +from discopop_library.PatchApplicator.apply import apply_patches +from discopop_library.PatchApplicator.clear import clear_patches +from discopop_library.PatchApplicator.load import load_patches +from discopop_library.PatchApplicator.rollback import rollback_patches +from discopop_library.PathManagement.PathManagement import load_file_mapping + + +def run(arguments: PatchApplicatorArguments): + if arguments.verbose: + print("Started DiscoPoP Patch Applicator...") + print("Working directory: ", os.getcwd()) + print(arguments) + + # create a directory for the patch applicator + patch_applicator_dir = os.path.join(os.getcwd(), "patch_applicator") + if not os.path.exists(patch_applicator_dir): + if arguments.verbose: + print("Creating patch_applicator directory...") + os.mkdir(patch_applicator_dir) + + # create a file to store applied suggestions + applied_suggestions_file = os.path.join(patch_applicator_dir, "applied_suggestions.json") + if not os.path.exists(applied_suggestions_file): + if arguments.verbose: + print("Creating applied_suggestions.json file...") + with open(applied_suggestions_file, "w+") as f: + f.write(json.dumps({"applied": []})) + + # load file mapping + file_mapping_path = os.path.join(os.getcwd(), "FileMapping.txt") + if not os.path.exists(file_mapping_path): + raise FileNotFoundError( + "No file mapping found. Please execute the discopop_explorer in advance." + + "\nExpected file: " + + file_mapping_path + ) + if arguments.verbose: + print("Loading file mapping...") + file_mapping = load_file_mapping(file_mapping_path) + + # handle arguments + if len(arguments.apply) > 0: + apply_patches(file_mapping, arguments) + elif len(arguments.rollback) > 0: + rollback_patches() + elif arguments.clear: + clear_patches() + elif arguments.load: + load_patches() + + if arguments.verbose: + print("Done.") diff --git a/discopop_library/PatchApplicator/rollback.py b/discopop_library/PatchApplicator/rollback.py new file mode 100644 index 000000000..c504322f7 --- /dev/null +++ b/discopop_library/PatchApplicator/rollback.py @@ -0,0 +1,9 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. +def rollback_patches(): + raise NotImplementedError() diff --git a/setup.py b/setup.py index 3007e78ad..dd8d81a16 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,7 @@ "discopop_code_generator=discopop_library.CodeGenerator.__main__:main", "discopop_optimizer=discopop_library.discopop_optimizer.__main__:main", "discopop_patch_generator=discopop_library.PatchGenerator.__main__:main", + "discopop_patch_applicator=discopop_library.PatchApplicator.__main__:main", ] }, zip_safe=True, From 06b2ba22bc0e9a0ba8958d099adf534a6b2a9dde Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Thu, 26 Oct 2023 10:57:11 +0200 Subject: [PATCH 04/23] feat(patch_applicator): apply patches implemented --- discopop_library/PatchApplicator/apply.py | 72 ++++++++++++++++++- .../PatchApplicator/patch_applicator.py | 12 +++- 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/discopop_library/PatchApplicator/apply.py b/discopop_library/PatchApplicator/apply.py index 84948d34c..48cc8842a 100644 --- a/discopop_library/PatchApplicator/apply.py +++ b/discopop_library/PatchApplicator/apply.py @@ -5,11 +5,79 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. +import json +import os +import subprocess from pathlib import Path from typing import Dict from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments -def apply_patches(file_mapping: Dict[int, Path], arguments: PatchApplicatorArguments): - raise NotImplementedError() +def apply_patches( + file_mapping: Dict[int, Path], + arguments: PatchApplicatorArguments, + applied_suggestions_file: str, + patch_generator_dir: str, +): + # get list of applicable suggestions + applicable_suggestions = [name for name in os.listdir(patch_generator_dir)] + + # get already applied suggestions + with open(applied_suggestions_file, "r") as f: + applied_suggestions = json.loads(f.read()) + if arguments.verbose: + print("Previously applied suggestions: ", applied_suggestions["applied"]) + + for suggestion_id in arguments.apply: + if suggestion_id in applied_suggestions["applied"]: + if arguments.verbose: + print("Skipping already applied suggestion: ", suggestion_id) + continue + if suggestion_id in applicable_suggestions: + if arguments.verbose: + print("Applying suggestion ", suggestion_id) + __apply_file_patches(file_mapping, suggestion_id, patch_generator_dir, arguments) + applied_suggestions["applied"].append(suggestion_id) + # write updated applied suggestions to file + with open(applied_suggestions_file, "w") as f: + f.write(json.dumps(applied_suggestions)) + else: + if arguments.verbose: + print("Nothing to apply for suggestion ", suggestion_id) + + +def __apply_file_patches( + file_mapping: Dict[int, Path], suggestion_id: str, patch_generator_dir: str, arguments: PatchApplicatorArguments +): + # get a list of patches for the given suggestion + patch_files = os.listdir(os.path.join(patch_generator_dir, suggestion_id)) + if arguments.verbose: + print("\tFound patch files:", patch_files) + + for patch_file_name in patch_files: + patch_file_id = int(patch_file_name.rstrip(".patch")) + patch_target = file_mapping[patch_file_id] + patch_file_path = os.path.join(patch_generator_dir, suggestion_id, patch_file_name) + + command = [ + "patch", + patch_target.as_posix(), + patch_file_path, + ] + if arguments.verbose: + print("\tapplying: ", " ".join(command)) + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=os.getcwd(), + ) + if result.returncode != 0: + if arguments.verbose: + print("RESULT: ", result.returncode) + print("STDERR:") + print(result.stderr) + print("STDOUT: ") + print(result.stdout) diff --git a/discopop_library/PatchApplicator/patch_applicator.py b/discopop_library/PatchApplicator/patch_applicator.py index 45263b6d4..41905fba3 100644 --- a/discopop_library/PatchApplicator/patch_applicator.py +++ b/discopop_library/PatchApplicator/patch_applicator.py @@ -49,9 +49,19 @@ def run(arguments: PatchApplicatorArguments): print("Loading file mapping...") file_mapping = load_file_mapping(file_mapping_path) + # check if patches have been generated + patch_generator_dir = os.path.join(os.getcwd(), "patch_generator") + if not os.path.exists(patch_generator_dir): + raise FileNotFoundError( + "No patches have been generated.\n" + + "Please execute the discopop_patch_generator in advance.\n" + + "Expected folder: " + + patch_generator_dir + ) + # handle arguments if len(arguments.apply) > 0: - apply_patches(file_mapping, arguments) + apply_patches(file_mapping, arguments, applied_suggestions_file, patch_generator_dir) elif len(arguments.rollback) > 0: rollback_patches() elif arguments.clear: From a3318e0d5ef96bab30a8d683b1d9c27205e10202 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Thu, 26 Oct 2023 11:13:37 +0200 Subject: [PATCH 05/23] feat(patch_applicator): list and rollback applied suggestions --- .../PatchApplicatorArguments.py | 6 ++ discopop_library/PatchApplicator/__main__.py | 11 +-- discopop_library/PatchApplicator/list.py | 17 ++++ .../PatchApplicator/patch_applicator.py | 5 +- discopop_library/PatchApplicator/rollback.py | 79 ++++++++++++++++++- 5 files changed, 110 insertions(+), 8 deletions(-) create mode 100644 discopop_library/PatchApplicator/list.py diff --git a/discopop_library/PatchApplicator/PatchApplicatorArguments.py b/discopop_library/PatchApplicator/PatchApplicatorArguments.py index a55fc78f3..cc31b85fe 100644 --- a/discopop_library/PatchApplicator/PatchApplicatorArguments.py +++ b/discopop_library/PatchApplicator/PatchApplicatorArguments.py @@ -19,6 +19,7 @@ class PatchApplicatorArguments(object): rollback: List[str] clear: bool load: bool + list: bool def __post_init__(self): self.__validate() @@ -39,6 +40,11 @@ def __validate(self): if self.load and (len(self.apply) > 0 or len(self.rollback)): print("Please use either '--load' or any of '--apply' or '--rollback'.") exit_required = True + if self.list: + self.apply = [] + self.rollback = [] + self.clear = False + self.load = False if exit_required: print("Exiting.") sys.exit(0) diff --git a/discopop_library/PatchApplicator/__main__.py b/discopop_library/PatchApplicator/__main__.py index 17b4dd959..27916cd29 100644 --- a/discopop_library/PatchApplicator/__main__.py +++ b/discopop_library/PatchApplicator/__main__.py @@ -28,12 +28,12 @@ def parse_args() -> PatchApplicatorArguments: "given ids.") parser.add_argument('-r', '--rollback', nargs='+', default=[], help="Roll back the application of the " "parallelization suggestions with the given " - "ids. If this process fails, all applied " - "suggestions until the given ones will be " - "rolled back as well.") - parser.add_argument('-c', '--clear', action="store_true", help="Reset the code to it's original state. Preserves " + "ids.") + parser.add_argument('-C', '--clear', action="store_true", help="Reset the code to it's original state. Preserves " "the list of applied suggestion for loading.") - parser.add_argument('-l', '--load', action="store_true", help="Load a previous state after clearing.") + parser.add_argument('-L', '--load', action="store_true", help="Load a previous state after clearing.") + parser.add_argument('-l', '--list', action="store_true", help="Show the list of applied suggestions." + "If set, nothing else will be done.") # EXPERIMENTAL FLAGS: # fmt: on @@ -45,6 +45,7 @@ def parse_args() -> PatchApplicatorArguments: rollback=arguments.rollback, clear=arguments.clear, load=arguments.load, + list=arguments.list, ) diff --git a/discopop_library/PatchApplicator/list.py b/discopop_library/PatchApplicator/list.py new file mode 100644 index 000000000..113cb7a15 --- /dev/null +++ b/discopop_library/PatchApplicator/list.py @@ -0,0 +1,17 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. +import json +from typing import List, cast + +from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments + + +def list_applied_suggestions(arguments: PatchApplicatorArguments, applied_suggestions_file: str) -> List[str]: + with open(applied_suggestions_file, "r") as f: + applied_suggestions = json.loads(f.read()) + return cast(List[str], applied_suggestions["applied"]) diff --git a/discopop_library/PatchApplicator/patch_applicator.py b/discopop_library/PatchApplicator/patch_applicator.py index 41905fba3..756829b98 100644 --- a/discopop_library/PatchApplicator/patch_applicator.py +++ b/discopop_library/PatchApplicator/patch_applicator.py @@ -11,6 +11,7 @@ from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments from discopop_library.PatchApplicator.apply import apply_patches from discopop_library.PatchApplicator.clear import clear_patches +from discopop_library.PatchApplicator.list import list_applied_suggestions from discopop_library.PatchApplicator.load import load_patches from discopop_library.PatchApplicator.rollback import rollback_patches from discopop_library.PathManagement.PathManagement import load_file_mapping @@ -63,11 +64,13 @@ def run(arguments: PatchApplicatorArguments): if len(arguments.apply) > 0: apply_patches(file_mapping, arguments, applied_suggestions_file, patch_generator_dir) elif len(arguments.rollback) > 0: - rollback_patches() + rollback_patches(file_mapping, arguments, applied_suggestions_file, patch_generator_dir) elif arguments.clear: clear_patches() elif arguments.load: load_patches() + elif arguments.list: + print("Applied suggestions: ", list_applied_suggestions(arguments, applied_suggestions_file)) if arguments.verbose: print("Done.") diff --git a/discopop_library/PatchApplicator/rollback.py b/discopop_library/PatchApplicator/rollback.py index c504322f7..7a527ba12 100644 --- a/discopop_library/PatchApplicator/rollback.py +++ b/discopop_library/PatchApplicator/rollback.py @@ -5,5 +5,80 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -def rollback_patches(): - raise NotImplementedError() +import json +import os +import subprocess +from pathlib import Path +from typing import Dict + +from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments + + +def rollback_patches( + file_mapping: Dict[int, Path], + arguments: PatchApplicatorArguments, + applied_suggestions_file: str, + patch_generator_dir: str, +): + # get list of applicable suggestions + applicable_suggestions = [name for name in os.listdir(patch_generator_dir)] + + # get already applied suggestions + with open(applied_suggestions_file, "r") as f: + applied_suggestions = json.loads(f.read()) + if arguments.verbose: + print("Previously applied suggestions: ", applied_suggestions["applied"]) + + for suggestion_id in arguments.rollback: + if suggestion_id not in applied_suggestions["applied"]: + if arguments.verbose: + print("Skipping rollback of non-applied suggestion: ", suggestion_id) + continue + if suggestion_id in applicable_suggestions: + if arguments.verbose: + print("Applying rollback of suggestion ", suggestion_id) + __rollback_file_patches(file_mapping, suggestion_id, patch_generator_dir, arguments) + applied_suggestions["applied"].remove(suggestion_id) + # write updated applied suggestions to file + with open(applied_suggestions_file, "w") as f: + f.write(json.dumps(applied_suggestions)) + else: + if arguments.verbose: + print("Nothing to rollback for suggestion ", suggestion_id) + + +def __rollback_file_patches( + file_mapping: Dict[int, Path], suggestion_id: str, patch_generator_dir: str, arguments: PatchApplicatorArguments +): + # get a list of patches for the given suggestion + patch_files = os.listdir(os.path.join(patch_generator_dir, suggestion_id)) + if arguments.verbose: + print("\tFound patch files:", patch_files) + + for patch_file_name in patch_files: + patch_file_id = int(patch_file_name.rstrip(".patch")) + patch_target = file_mapping[patch_file_id] + patch_file_path = os.path.join(patch_generator_dir, suggestion_id, patch_file_name) + + command = [ + "patch", + "-r", + patch_target.as_posix(), + patch_file_path, + ] + if arguments.verbose: + print("\tapplying: ", " ".join(command)) + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=os.getcwd(), + ) + if result.returncode != 0: + if arguments.verbose: + print("RESULT: ", result.returncode) + print("STDERR:") + print(result.stderr) + print("STDOUT: ") + print(result.stdout) From bbb11d3ac436976b93fd1cf5c1573e06e8258d42 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Thu, 26 Oct 2023 11:17:47 +0200 Subject: [PATCH 06/23] fix(patch_applicator)[rollback]: patch arguments --- discopop_library/PatchApplicator/rollback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discopop_library/PatchApplicator/rollback.py b/discopop_library/PatchApplicator/rollback.py index 7a527ba12..e30f75dff 100644 --- a/discopop_library/PatchApplicator/rollback.py +++ b/discopop_library/PatchApplicator/rollback.py @@ -62,7 +62,7 @@ def __rollback_file_patches( command = [ "patch", - "-r", + "-R", patch_target.as_posix(), patch_file_path, ] From 464111558db95eb419e8e98e70368193cac3d8cc Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Thu, 26 Oct 2023 11:31:31 +0200 Subject: [PATCH 07/23] feat(patch_applicator)[clear]: implemented functionality --- discopop_library/PatchApplicator/__main__.py | 5 +++- discopop_library/PatchApplicator/apply.py | 5 ++-- discopop_library/PatchApplicator/clear.py | 28 +++++++++++++++++-- .../PatchApplicator/patch_applicator.py | 6 ++-- discopop_library/PatchApplicator/rollback.py | 5 ++-- 5 files changed, 39 insertions(+), 10 deletions(-) diff --git a/discopop_library/PatchApplicator/__main__.py b/discopop_library/PatchApplicator/__main__.py index 27916cd29..6eca64352 100644 --- a/discopop_library/PatchApplicator/__main__.py +++ b/discopop_library/PatchApplicator/__main__.py @@ -30,7 +30,10 @@ def parse_args() -> PatchApplicatorArguments: "parallelization suggestions with the given " "ids.") parser.add_argument('-C', '--clear', action="store_true", help="Reset the code to it's original state. Preserves " - "the list of applied suggestion for loading.") + "the list of applied suggestion for loading." + "Old saves will be overwritten!") + # todo: in the long run, saving configurations to different locations could be easily implemented. + parser.add_argument('-L', '--load', action="store_true", help="Load a previous state after clearing.") parser.add_argument('-l', '--list', action="store_true", help="Show the list of applied suggestions." "If set, nothing else will be done.") diff --git a/discopop_library/PatchApplicator/apply.py b/discopop_library/PatchApplicator/apply.py index 48cc8842a..d0fd75d01 100644 --- a/discopop_library/PatchApplicator/apply.py +++ b/discopop_library/PatchApplicator/apply.py @@ -9,12 +9,13 @@ import os import subprocess from pathlib import Path -from typing import Dict +from typing import Dict, List from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments def apply_patches( + apply: List[str], file_mapping: Dict[int, Path], arguments: PatchApplicatorArguments, applied_suggestions_file: str, @@ -29,7 +30,7 @@ def apply_patches( if arguments.verbose: print("Previously applied suggestions: ", applied_suggestions["applied"]) - for suggestion_id in arguments.apply: + for suggestion_id in apply: if suggestion_id in applied_suggestions["applied"]: if arguments.verbose: print("Skipping already applied suggestion: ", suggestion_id) diff --git a/discopop_library/PatchApplicator/clear.py b/discopop_library/PatchApplicator/clear.py index 584dccf39..fe3ee95ff 100644 --- a/discopop_library/PatchApplicator/clear.py +++ b/discopop_library/PatchApplicator/clear.py @@ -5,5 +5,29 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -def clear_patches(): - raise NotImplementedError() +import json +import os.path +import shutil +from pathlib import Path +from typing import Dict + +from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments +from discopop_library.PatchApplicator.rollback import rollback_patches + + +def clear_patches( + file_mapping: Dict[int, Path], + arguments: PatchApplicatorArguments, + applied_suggestions_file: str, + patch_generator_dir: str, +): + # save the currently applied suggestions. Overwrites old saves + shutil.copyfile(applied_suggestions_file, applied_suggestions_file + ".save") + + # get the list of applied suggestions + with open(applied_suggestions_file, "r") as f: + applied_suggestions = json.loads(f.read())["applied"] + + # rollback all suggestions in inverse order + applied_suggestions.reverse() + rollback_patches(applied_suggestions, file_mapping, arguments, applied_suggestions_file, patch_generator_dir) diff --git a/discopop_library/PatchApplicator/patch_applicator.py b/discopop_library/PatchApplicator/patch_applicator.py index 756829b98..10e3b595d 100644 --- a/discopop_library/PatchApplicator/patch_applicator.py +++ b/discopop_library/PatchApplicator/patch_applicator.py @@ -62,11 +62,11 @@ def run(arguments: PatchApplicatorArguments): # handle arguments if len(arguments.apply) > 0: - apply_patches(file_mapping, arguments, applied_suggestions_file, patch_generator_dir) + apply_patches(arguments.apply, file_mapping, arguments, applied_suggestions_file, patch_generator_dir) elif len(arguments.rollback) > 0: - rollback_patches(file_mapping, arguments, applied_suggestions_file, patch_generator_dir) + rollback_patches(arguments.rollback, file_mapping, arguments, applied_suggestions_file, patch_generator_dir) elif arguments.clear: - clear_patches() + clear_patches(file_mapping, arguments, applied_suggestions_file, patch_generator_dir) elif arguments.load: load_patches() elif arguments.list: diff --git a/discopop_library/PatchApplicator/rollback.py b/discopop_library/PatchApplicator/rollback.py index e30f75dff..5bf1d2ee1 100644 --- a/discopop_library/PatchApplicator/rollback.py +++ b/discopop_library/PatchApplicator/rollback.py @@ -9,12 +9,13 @@ import os import subprocess from pathlib import Path -from typing import Dict +from typing import Dict, List from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments def rollback_patches( + rollback: List[str], file_mapping: Dict[int, Path], arguments: PatchApplicatorArguments, applied_suggestions_file: str, @@ -29,7 +30,7 @@ def rollback_patches( if arguments.verbose: print("Previously applied suggestions: ", applied_suggestions["applied"]) - for suggestion_id in arguments.rollback: + for suggestion_id in rollback: if suggestion_id not in applied_suggestions["applied"]: if arguments.verbose: print("Skipping rollback of non-applied suggestion: ", suggestion_id) From 6a6d69a238a0ef4beb1954f2372925f3ce77b2fd Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Thu, 26 Oct 2023 12:03:00 +0200 Subject: [PATCH 08/23] feat(patch_applicator)[load]: implemented functionality and minor fixes --- discopop_library/PatchApplicator/apply.py | 53 ++++++++++++++++--- discopop_library/PatchApplicator/load.py | 29 +++++++++- .../PatchApplicator/patch_applicator.py | 2 +- discopop_library/PatchApplicator/rollback.py | 53 ++++++++++++++++--- 4 files changed, 120 insertions(+), 17 deletions(-) diff --git a/discopop_library/PatchApplicator/apply.py b/discopop_library/PatchApplicator/apply.py index d0fd75d01..d17322643 100644 --- a/discopop_library/PatchApplicator/apply.py +++ b/discopop_library/PatchApplicator/apply.py @@ -38,11 +38,14 @@ def apply_patches( if suggestion_id in applicable_suggestions: if arguments.verbose: print("Applying suggestion ", suggestion_id) - __apply_file_patches(file_mapping, suggestion_id, patch_generator_dir, arguments) - applied_suggestions["applied"].append(suggestion_id) - # write updated applied suggestions to file - with open(applied_suggestions_file, "w") as f: - f.write(json.dumps(applied_suggestions)) + successful = __apply_file_patches(file_mapping, suggestion_id, patch_generator_dir, arguments) + if successful: + applied_suggestions["applied"].append(suggestion_id) + # write updated applied suggestions to file + with open(applied_suggestions_file, "w") as f: + f.write(json.dumps(applied_suggestions)) + else: + print("Applying suggestion", suggestion_id, "not successful.") else: if arguments.verbose: print("Nothing to apply for suggestion ", suggestion_id) @@ -50,17 +53,18 @@ def apply_patches( def __apply_file_patches( file_mapping: Dict[int, Path], suggestion_id: str, patch_generator_dir: str, arguments: PatchApplicatorArguments -): +) -> bool: # get a list of patches for the given suggestion patch_files = os.listdir(os.path.join(patch_generator_dir, suggestion_id)) if arguments.verbose: print("\tFound patch files:", patch_files) + encountered_error = False + already_patched: List[str] = [] for patch_file_name in patch_files: patch_file_id = int(patch_file_name.rstrip(".patch")) patch_target = file_mapping[patch_file_id] patch_file_path = os.path.join(patch_generator_dir, suggestion_id, patch_file_name) - command = [ "patch", patch_target.as_posix(), @@ -80,5 +84,40 @@ def __apply_file_patches( print("RESULT: ", result.returncode) print("STDERR:") print(result.stderr) + print("STDOUT: ") + print(result.stdout) + encountered_error = True + break + else: + already_patched.append(patch_file_name) + + # cleanup in case of an error + if encountered_error: + for patch_file_name in already_patched: + patch_file_id = int(patch_file_name.rstrip(".patch")) + patch_target = file_mapping[patch_file_id] + patch_file_path = os.path.join(patch_generator_dir, suggestion_id, patch_file_name) + command = [ + "patch", + "-R", + patch_target.as_posix(), + patch_file_path, + ] + if arguments.verbose: + print("\tcleanup: applying: ", " ".join(command)) + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=os.getcwd(), + ) + if result.returncode != 0: + if arguments.verbose: + print("RESULT: ", result.returncode) + print("STDERR:") + print(result.stderr) print("STDOUT: ") print(result.stdout) + + return not encountered_error diff --git a/discopop_library/PatchApplicator/load.py b/discopop_library/PatchApplicator/load.py index f2c770729..fa0bc0d57 100644 --- a/discopop_library/PatchApplicator/load.py +++ b/discopop_library/PatchApplicator/load.py @@ -5,5 +5,30 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -def load_patches(): - raise NotImplementedError() +import json +import os.path +from pathlib import Path +from typing import Dict + +from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments +from discopop_library.PatchApplicator.apply import apply_patches + + +def load_patches( + file_mapping: Dict[int, Path], + arguments: PatchApplicatorArguments, + applied_suggestions_file: str, + patch_generator_dir: str, +): + # check if a saved configuration exists + if not os.path.exists(applied_suggestions_file + ".save"): + raise FileNotFoundError( + "Nothing to be loaded. Exiting." "\nExpected file: ", applied_suggestions_file + ".save" + ) + + # get the list of suggestions to be applied + with open(applied_suggestions_file + ".save", "r") as f: + suggestions_to_be_applied = json.loads(f.read())["applied"] + + # apply all suggestions + apply_patches(suggestions_to_be_applied, file_mapping, arguments, applied_suggestions_file, patch_generator_dir) diff --git a/discopop_library/PatchApplicator/patch_applicator.py b/discopop_library/PatchApplicator/patch_applicator.py index 10e3b595d..40736b8bf 100644 --- a/discopop_library/PatchApplicator/patch_applicator.py +++ b/discopop_library/PatchApplicator/patch_applicator.py @@ -68,7 +68,7 @@ def run(arguments: PatchApplicatorArguments): elif arguments.clear: clear_patches(file_mapping, arguments, applied_suggestions_file, patch_generator_dir) elif arguments.load: - load_patches() + load_patches(file_mapping, arguments, applied_suggestions_file, patch_generator_dir) elif arguments.list: print("Applied suggestions: ", list_applied_suggestions(arguments, applied_suggestions_file)) diff --git a/discopop_library/PatchApplicator/rollback.py b/discopop_library/PatchApplicator/rollback.py index 5bf1d2ee1..9d363fa60 100644 --- a/discopop_library/PatchApplicator/rollback.py +++ b/discopop_library/PatchApplicator/rollback.py @@ -37,12 +37,15 @@ def rollback_patches( continue if suggestion_id in applicable_suggestions: if arguments.verbose: - print("Applying rollback of suggestion ", suggestion_id) - __rollback_file_patches(file_mapping, suggestion_id, patch_generator_dir, arguments) - applied_suggestions["applied"].remove(suggestion_id) - # write updated applied suggestions to file - with open(applied_suggestions_file, "w") as f: - f.write(json.dumps(applied_suggestions)) + print("Rollback suggestion ", suggestion_id) + successul = __rollback_file_patches(file_mapping, suggestion_id, patch_generator_dir, arguments) + if successul: + applied_suggestions["applied"].remove(suggestion_id) + # write updated applied suggestions to file + with open(applied_suggestions_file, "w") as f: + f.write(json.dumps(applied_suggestions)) + else: + print("Rollback of suggestion", suggestion_id, "not successful.") else: if arguments.verbose: print("Nothing to rollback for suggestion ", suggestion_id) @@ -50,12 +53,14 @@ def rollback_patches( def __rollback_file_patches( file_mapping: Dict[int, Path], suggestion_id: str, patch_generator_dir: str, arguments: PatchApplicatorArguments -): +) -> bool: # get a list of patches for the given suggestion patch_files = os.listdir(os.path.join(patch_generator_dir, suggestion_id)) if arguments.verbose: print("\tFound patch files:", patch_files) + encountered_error = False + already_patched: List[str] = [] for patch_file_name in patch_files: patch_file_id = int(patch_file_name.rstrip(".patch")) patch_target = file_mapping[patch_file_id] @@ -81,5 +86,39 @@ def __rollback_file_patches( print("RESULT: ", result.returncode) print("STDERR:") print(result.stderr) + print("STDOUT: ") + print(result.stdout) + encountered_error = True + break + else: + already_patched.append(patch_file_name) + + # cleanup in case of an error + if encountered_error: + for patch_file_name in already_patched: + patch_file_id = int(patch_file_name.rstrip(".patch")) + patch_target = file_mapping[patch_file_id] + patch_file_path = os.path.join(patch_generator_dir, suggestion_id, patch_file_name) + command = [ + "patch", + patch_target.as_posix(), + patch_file_path, + ] + if arguments.verbose: + print("\tcleanup: applying: ", " ".join(command)) + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=os.getcwd(), + ) + if result.returncode != 0: + if arguments.verbose: + print("RESULT: ", result.returncode) + print("STDERR:") + print(result.stderr) print("STDOUT: ") print(result.stdout) + + return not encountered_error From cc272cdbc5585b902aa1e7e8c2166143a77d084b Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Wed, 1 Nov 2023 13:33:10 +0100 Subject: [PATCH 09/23] feat(line_mapping): let discopop_explorer initialize the file --- discopop_explorer/discopop_explorer.py | 6 +++- discopop_library/LineMapping/__init__.py | 0 discopop_library/LineMapping/initialize.py | 36 ++++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 discopop_library/LineMapping/__init__.py create mode 100644 discopop_library/LineMapping/initialize.py diff --git a/discopop_explorer/discopop_explorer.py b/discopop_explorer/discopop_explorer.py index 991e9c401..7758439d2 100644 --- a/discopop_explorer/discopop_explorer.py +++ b/discopop_explorer/discopop_explorer.py @@ -18,7 +18,8 @@ import pstats2 # type:ignore from pluginbase import PluginBase # type:ignore -from discopop_library.PathManagement.PathManagement import get_path +from discopop_library.LineMapping.initialize import initialize_line_mapping +from discopop_library.PathManagement.PathManagement import get_path, load_file_mapping from discopop_library.discopop_optimizer.Microbench.ExtrapInterpolatedMicrobench import ( ExtrapInterpolatedMicrobench, ) @@ -215,6 +216,9 @@ def run(arguments: ExplorerArguments): stats = pstats2.Stats(profile, stream=f).sort_stats("time").reverse_order() stats.print_stats() + # initialize the line_mapping.json + initialize_line_mapping(load_file_mapping(arguments.file_mapping_file), arguments.project_path) + print("Time taken for pattern detection: {0}".format(end - start)) # demonstration of Microbenchmark possibilities diff --git a/discopop_library/LineMapping/__init__.py b/discopop_library/LineMapping/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/discopop_library/LineMapping/initialize.py b/discopop_library/LineMapping/initialize.py new file mode 100644 index 000000000..061d425a6 --- /dev/null +++ b/discopop_library/LineMapping/initialize.py @@ -0,0 +1,36 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. + +import json +import os.path +from pathlib import Path +from typing import Dict + + +def initialize_line_mapping( + file_mapping: Dict[int, Path], + discopop_path: str = ".discopop", +): + """initializes the line mapping dictionary to track line shifts due to inserted pragmas. + The Dictionary will be stored in .discopop/line_mapping.json. + Line ids start with 1.""" + line_mapping_dict: Dict[int, Dict[int, int]] = dict() + + # initialize line mapping (1->1, 2->2, ...) + for file_id in file_mapping: + if file_id not in line_mapping_dict: + line_mapping_dict[file_id] = dict() + line_id = 1 + with open(file_mapping[file_id], "r") as f: + for line in f.readlines(): + line_mapping_dict[file_id][line_id] = line_id + line_id += 1 + + # dump line mapping to json file + with open(os.path.join(discopop_path, "line_mapping.json"), "w+") as f: + json.dump(line_mapping_dict, f) From 9d76cdca2e5f4191f02394a039410b99a16955cd Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Wed, 1 Nov 2023 14:10:04 +0100 Subject: [PATCH 10/23] feat(line_mapping): utilities to save and load added --- discopop_library/LineMapping/initialize.py | 10 +++++++--- discopop_library/LineMapping/load.py | 18 ++++++++++++++++++ discopop_library/LineMapping/save.py | 17 +++++++++++++++++ 3 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 discopop_library/LineMapping/load.py create mode 100644 discopop_library/LineMapping/save.py diff --git a/discopop_library/LineMapping/initialize.py b/discopop_library/LineMapping/initialize.py index 061d425a6..6571451c2 100644 --- a/discopop_library/LineMapping/initialize.py +++ b/discopop_library/LineMapping/initialize.py @@ -11,6 +11,9 @@ from pathlib import Path from typing import Dict +from discopop_library.LineMapping.load import load_line_mapping +from discopop_library.LineMapping.save import save_line_mapping + def initialize_line_mapping( file_mapping: Dict[int, Path], @@ -31,6 +34,7 @@ def initialize_line_mapping( line_mapping_dict[file_id][line_id] = line_id line_id += 1 - # dump line mapping to json file - with open(os.path.join(discopop_path, "line_mapping.json"), "w+") as f: - json.dump(line_mapping_dict, f) + save_line_mapping(line_mapping_dict, discopop_path) + + # debug + load_line_mapping(discopop_path) diff --git a/discopop_library/LineMapping/load.py b/discopop_library/LineMapping/load.py new file mode 100644 index 000000000..9f499cf49 --- /dev/null +++ b/discopop_library/LineMapping/load.py @@ -0,0 +1,18 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. +import json +import os.path +from typing import Dict + + +def load_line_mapping(discopop_path: str = ".discopop") -> Dict[int, Dict[int, int]]: + """Loads and returns the line_mapping dictionary""" + line_mapping_dict: Dict[int, Dict[int, int]] = dict() + with open(os.path.join(discopop_path, "line_mapping.json")) as f: + line_mapping_dict = json.load(f) + return line_mapping_dict diff --git a/discopop_library/LineMapping/save.py b/discopop_library/LineMapping/save.py new file mode 100644 index 000000000..bc07ab074 --- /dev/null +++ b/discopop_library/LineMapping/save.py @@ -0,0 +1,17 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. +import json +import os +from typing import Dict + + +def save_line_mapping(line_mapping_dict: Dict[int, Dict[int, int]], discopop_path: str = ".discopop"): + """dumps line_mapping_dict to line_mapping.json""" + # dump line mapping to json file + with open(os.path.join(discopop_path, "line_mapping.json"), "w+") as f: + json.dump(line_mapping_dict, f) From d5b552abc56dfce3938e25d1458f0fa218904a5a Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Wed, 1 Nov 2023 23:01:32 +0100 Subject: [PATCH 11/23] feat: applied modifications based on diffs & represented in line_mapping --- .../LineMapping/diff_modifications.py | 149 ++++++++++++++++++ discopop_library/LineMapping/initialize.py | 8 +- discopop_library/LineMapping/load.py | 4 +- discopop_library/LineMapping/save.py | 2 +- discopop_library/PatchApplicator/apply.py | 22 ++- discopop_library/PatchApplicator/rollback.py | 22 ++- 6 files changed, 198 insertions(+), 9 deletions(-) create mode 100644 discopop_library/LineMapping/diff_modifications.py diff --git a/discopop_library/LineMapping/diff_modifications.py b/discopop_library/LineMapping/diff_modifications.py new file mode 100644 index 000000000..b13d6f30b --- /dev/null +++ b/discopop_library/LineMapping/diff_modifications.py @@ -0,0 +1,149 @@ +# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de) +# +# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany +# +# This software may be modified and distributed under the terms of +# the 3-Clause BSD License. See the LICENSE file in the package base +# directory for details. +import os.path +import subprocess +from typing import List, Tuple, Dict, Optional, Set + +from discopop_library.LineMapping.load import load_line_mapping +from discopop_library.LineMapping.save import save_line_mapping + + +def apply_line_mapping_modifications_from_files(file_id: int, original_file: str, modified_file: str): + """Calculates diff between original_file and modified_file and applied modifications from the diff to the line_mapping""" + if not os.path.exists(original_file): + raise FileNotFoundError(original_file) + if not os.path.exists(modified_file): + raise FileNotFoundError(original_file) + + command = [ + "diff", + original_file, + modified_file, + ] + result = subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + if result.returncode == 1: + print("RESULT: ", result.returncode) + print("STDERR:") + print(result.stderr) + print("STDOUT: ") + print(result.stdout) + diff = result.stdout + print("DIFF: ", diff) + else: + diff = "" + + apply_line_mapping_modifications_from_diff(file_id, diff) + + +def apply_line_mapping_modifications_from_diff(file_id: int, diff: str): + """parse diff, apply line num modifications according to c,a,d values""" + # get cleaned diff + cleaned_diff: List[str] = [] + for line in diff.split("\n"): + if line.startswith("<") or line.startswith(">") or line.startswith("-") or len(line) == 0: + continue + line = line.replace("\n", "") + cleaned_diff.append(line) + + # get line_mapping + line_mapping: Dict[str, Dict[str, int]] = load_line_mapping() + + deletions: Set[str] = set() + shifts: Dict[str, int] = dict() + + # initialize + for key in line_mapping[str(file_id)]: + shifts[key] = 0 + + # parse diff + for diff_entry in cleaned_diff: + if "a" in diff_entry: + lhs = diff_entry[: diff_entry.index("a")] + if "," in lhs: + base_line = int(lhs.split(",")[0]) + else: + base_line = int(lhs) + rhs = diff_entry[diff_entry.index("a") + 1 :] + if "," in rhs: + added_lines_count = int(rhs.split(",")[1]) - int(rhs.split(",")[0]) + else: + added_lines_count = 1 + for key in line_mapping[str(file_id)]: + if line_mapping[str(file_id)][key] < 0: + # invalid due to deletion + continue + if line_mapping[str(file_id)][key] > base_line: + shifts[key] += added_lines_count + + if "d" in diff_entry: + lhs = diff_entry[: diff_entry.index("d")] + if "," in lhs: + base_line = int(lhs.split(",")[0]) + deleted_lines_count = int(lhs.split(",")[1]) - int(lhs.split(",")[0]) + else: + base_line = int(lhs) + deleted_lines_count = 1 + for key in line_mapping[str(file_id)]: + if line_mapping[str(file_id)][key] < 0: + # invalid due to deletion + continue + if line_mapping[str(file_id)][key] in range(base_line, base_line + deleted_lines_count): + deletions.add(key) + elif line_mapping[str(file_id)][key] >= base_line + deleted_lines_count: + shifts[key] -= deleted_lines_count + + if "c" in diff_entry: + lhs = diff_entry[: diff_entry.index("c")] + if "," in lhs: + base_line = int(lhs.split(",")[0]) + deleted_lines_count = int(lhs.split(",")[1]) - int(lhs.split(",")[0]) + else: + base_line = int(lhs) + deleted_lines_count = 1 + rhs = diff_entry[diff_entry.index("c") + 1 :] + if "," in rhs: + added_lines_count = int(rhs.split(",")[1]) - int(rhs.split(",")[0]) + else: + added_lines_count = 1 + + for key in line_mapping[str(file_id)]: + if line_mapping[str(file_id)][key] < 0: + # invalid due to deletion + continue + if line_mapping[str(file_id)][key] > base_line: + shifts[key] += added_lines_count + + for key in line_mapping[str(file_id)]: + if line_mapping[str(file_id)][key] < 0: + # invalid due to deletion + continue + if line_mapping[str(file_id)][key] in range(base_line, base_line + deleted_lines_count): + deletions.add(key) + elif line_mapping[str(file_id)][key] >= base_line + deleted_lines_count: + shifts[key] -= deleted_lines_count + + # apply deletions + for key in deletions: + line_mapping[str(file_id)][key] = -1 + + # apply shifts + for key in shifts: + if line_mapping[str(file_id)][key] < 0: + # invalid due to deletion + continue + line_mapping[str(file_id)][key] = line_mapping[str(file_id)][key] + shifts[key] + + # save updated line mapping + save_line_mapping(line_mapping) + + pass diff --git a/discopop_library/LineMapping/initialize.py b/discopop_library/LineMapping/initialize.py index 6571451c2..ee18dad96 100644 --- a/discopop_library/LineMapping/initialize.py +++ b/discopop_library/LineMapping/initialize.py @@ -22,16 +22,16 @@ def initialize_line_mapping( """initializes the line mapping dictionary to track line shifts due to inserted pragmas. The Dictionary will be stored in .discopop/line_mapping.json. Line ids start with 1.""" - line_mapping_dict: Dict[int, Dict[int, int]] = dict() + line_mapping_dict: Dict[str, Dict[str, int]] = dict() # initialize line mapping (1->1, 2->2, ...) for file_id in file_mapping: - if file_id not in line_mapping_dict: - line_mapping_dict[file_id] = dict() + if str(file_id) not in line_mapping_dict: + line_mapping_dict[str(file_id)] = dict() line_id = 1 with open(file_mapping[file_id], "r") as f: for line in f.readlines(): - line_mapping_dict[file_id][line_id] = line_id + line_mapping_dict[str(file_id)][str(line_id)] = line_id line_id += 1 save_line_mapping(line_mapping_dict, discopop_path) diff --git a/discopop_library/LineMapping/load.py b/discopop_library/LineMapping/load.py index 9f499cf49..84c248e0e 100644 --- a/discopop_library/LineMapping/load.py +++ b/discopop_library/LineMapping/load.py @@ -10,9 +10,9 @@ from typing import Dict -def load_line_mapping(discopop_path: str = ".discopop") -> Dict[int, Dict[int, int]]: +def load_line_mapping(discopop_path: str = "") -> Dict[str, Dict[str, int]]: """Loads and returns the line_mapping dictionary""" - line_mapping_dict: Dict[int, Dict[int, int]] = dict() + line_mapping_dict: Dict[str, Dict[str, int]] = dict() with open(os.path.join(discopop_path, "line_mapping.json")) as f: line_mapping_dict = json.load(f) return line_mapping_dict diff --git a/discopop_library/LineMapping/save.py b/discopop_library/LineMapping/save.py index bc07ab074..8607d5f7d 100644 --- a/discopop_library/LineMapping/save.py +++ b/discopop_library/LineMapping/save.py @@ -10,7 +10,7 @@ from typing import Dict -def save_line_mapping(line_mapping_dict: Dict[int, Dict[int, int]], discopop_path: str = ".discopop"): +def save_line_mapping(line_mapping_dict: Dict[str, Dict[str, int]], discopop_path: str = ""): """dumps line_mapping_dict to line_mapping.json""" # dump line mapping to json file with open(os.path.join(discopop_path, "line_mapping.json"), "w+") as f: diff --git a/discopop_library/PatchApplicator/apply.py b/discopop_library/PatchApplicator/apply.py index d17322643..2ea3ac2bd 100644 --- a/discopop_library/PatchApplicator/apply.py +++ b/discopop_library/PatchApplicator/apply.py @@ -7,10 +7,12 @@ # directory for details. import json import os +import shutil import subprocess from pathlib import Path from typing import Dict, List +from discopop_library.LineMapping.diff_modifications import apply_line_mapping_modifications_from_files from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments @@ -58,13 +60,14 @@ def __apply_file_patches( patch_files = os.listdir(os.path.join(patch_generator_dir, suggestion_id)) if arguments.verbose: print("\tFound patch files:", patch_files) - encountered_error = False already_patched: List[str] = [] for patch_file_name in patch_files: patch_file_id = int(patch_file_name.rstrip(".patch")) patch_target = file_mapping[patch_file_id] patch_file_path = os.path.join(patch_generator_dir, suggestion_id, patch_file_name) + # save original version to calculate diff to calculate line mapping + shutil.copyfile(patch_target.as_posix(), patch_target.as_posix() + ".line_mapping_tmp") command = [ "patch", patch_target.as_posix(), @@ -87,9 +90,17 @@ def __apply_file_patches( print("STDOUT: ") print(result.stdout) encountered_error = True + # delete temporary file + os.remove(patch_target.as_posix() + ".line_mapping_tmp") break else: already_patched.append(patch_file_name) + # apply modifications to line mapping + apply_line_mapping_modifications_from_files( + patch_file_id, patch_target.as_posix() + ".line_mapping_tmp", patch_target.as_posix() + ) + # delete temporary file + os.remove(patch_target.as_posix() + ".line_mapping_tmp") # cleanup in case of an error if encountered_error: @@ -97,6 +108,8 @@ def __apply_file_patches( patch_file_id = int(patch_file_name.rstrip(".patch")) patch_target = file_mapping[patch_file_id] patch_file_path = os.path.join(patch_generator_dir, suggestion_id, patch_file_name) + # save original version to calculate diff to calculate line mapping + shutil.copyfile(patch_target.as_posix(), patch_target.as_posix() + ".line_mapping_tmp") command = [ "patch", "-R", @@ -120,4 +133,11 @@ def __apply_file_patches( print("STDOUT: ") print(result.stdout) + # apply modifications to line mapping + apply_line_mapping_modifications_from_files( + patch_file_id, patch_target.as_posix() + ".line_mapping_tmp", patch_target.as_posix() + ) + # delete temporary file + os.remove(patch_target.as_posix() + ".line_mapping_tmp") + return not encountered_error diff --git a/discopop_library/PatchApplicator/rollback.py b/discopop_library/PatchApplicator/rollback.py index 9d363fa60..b2bf0a52b 100644 --- a/discopop_library/PatchApplicator/rollback.py +++ b/discopop_library/PatchApplicator/rollback.py @@ -7,10 +7,12 @@ # directory for details. import json import os +import shutil import subprocess from pathlib import Path from typing import Dict, List +from discopop_library.LineMapping.diff_modifications import apply_line_mapping_modifications_from_files from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments @@ -65,7 +67,8 @@ def __rollback_file_patches( patch_file_id = int(patch_file_name.rstrip(".patch")) patch_target = file_mapping[patch_file_id] patch_file_path = os.path.join(patch_generator_dir, suggestion_id, patch_file_name) - + # save original version to calculate diff to calculate line mapping + shutil.copyfile(patch_target.as_posix(), patch_target.as_posix() + ".line_mapping_tmp") command = [ "patch", "-R", @@ -89,9 +92,17 @@ def __rollback_file_patches( print("STDOUT: ") print(result.stdout) encountered_error = True + # delete temporary file + os.remove(patch_target.as_posix() + ".line_mapping_tmp") break else: already_patched.append(patch_file_name) + # apply modifications to line mapping + apply_line_mapping_modifications_from_files( + patch_file_id, patch_target.as_posix() + ".line_mapping_tmp", patch_target.as_posix() + ) + # delete temporary file + os.remove(patch_target.as_posix() + ".line_mapping_tmp") # cleanup in case of an error if encountered_error: @@ -99,6 +110,8 @@ def __rollback_file_patches( patch_file_id = int(patch_file_name.rstrip(".patch")) patch_target = file_mapping[patch_file_id] patch_file_path = os.path.join(patch_generator_dir, suggestion_id, patch_file_name) + # save original version to calculate diff to calculate line mapping + shutil.copyfile(patch_target.as_posix(), patch_target.as_posix() + ".line_mapping_tmp") command = [ "patch", patch_target.as_posix(), @@ -121,4 +134,11 @@ def __rollback_file_patches( print("STDOUT: ") print(result.stdout) + # apply modifications to line mapping + apply_line_mapping_modifications_from_files( + patch_file_id, patch_target.as_posix() + ".line_mapping_tmp", patch_target.as_posix() + ) + # delete temporary file + os.remove(patch_target.as_posix() + ".line_mapping_tmp") + return not encountered_error From cde39895d9b4c4a80d1eca6efa21b1038f75da7d Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Wed, 1 Nov 2023 23:04:12 +0100 Subject: [PATCH 12/23] chore: cleanup debug prints --- discopop_library/LineMapping/diff_modifications.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/discopop_library/LineMapping/diff_modifications.py b/discopop_library/LineMapping/diff_modifications.py index b13d6f30b..873c8caa8 100644 --- a/discopop_library/LineMapping/diff_modifications.py +++ b/discopop_library/LineMapping/diff_modifications.py @@ -32,13 +32,13 @@ def apply_line_mapping_modifications_from_files(file_id: int, original_file: str universal_newlines=True, ) if result.returncode == 1: - print("RESULT: ", result.returncode) - print("STDERR:") - print(result.stderr) - print("STDOUT: ") - print(result.stdout) + # print("RESULT: ", result.returncode) + # print("STDERR:") + # print(result.stderr) + # print("STDOUT: ") + # print(result.stdout) diff = result.stdout - print("DIFF: ", diff) + # print("DIFF: ", diff) else: diff = "" From 83960664498d45ad0815551b67467f73506a2c31 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Mon, 6 Nov 2023 09:17:42 +0100 Subject: [PATCH 13/23] feat(patch_applicator): added return code --- discopop_library/PatchApplicator/__main__.py | 18 ++++++++++++---- discopop_library/PatchApplicator/apply.py | 21 ++++++++++++++++++- discopop_library/PatchApplicator/clear.py | 8 +++++-- discopop_library/PatchApplicator/load.py | 8 +++++-- .../PatchApplicator/patch_applicator.py | 20 +++++++++++++----- discopop_library/PatchApplicator/rollback.py | 21 ++++++++++++++++++- 6 files changed, 81 insertions(+), 15 deletions(-) diff --git a/discopop_library/PatchApplicator/__main__.py b/discopop_library/PatchApplicator/__main__.py index 6eca64352..f5a9f848c 100644 --- a/discopop_library/PatchApplicator/__main__.py +++ b/discopop_library/PatchApplicator/__main__.py @@ -23,7 +23,7 @@ def parse_args() -> PatchApplicatorArguments: # fmt: off parser.add_argument("-v", "--verbose", action="store_true", - help="Enable verbose output.") + help="Enable verbose output.") parser.add_argument('-a', '--apply', nargs='+', default=[], help="Apply the parallelization suggestions with the " "given ids.") parser.add_argument('-r', '--rollback', nargs='+', default=[], help="Roll back the application of the " @@ -52,9 +52,19 @@ def parse_args() -> PatchApplicatorArguments: ) -def main(): - arguments = parse_args() - run(arguments) +def main() -> int: + """Return values:" + "0: Applied successfully" + "1: Nothing applied" + "2: Some changes applied successfully + """ + retval = 0 + try: + arguments = parse_args() + retval = run(arguments) + except FileNotFoundError: + retval = 1 + return retval if __name__ == "__main__": diff --git a/discopop_library/PatchApplicator/apply.py b/discopop_library/PatchApplicator/apply.py index 2ea3ac2bd..b00a70bea 100644 --- a/discopop_library/PatchApplicator/apply.py +++ b/discopop_library/PatchApplicator/apply.py @@ -22,7 +22,12 @@ def apply_patches( arguments: PatchApplicatorArguments, applied_suggestions_file: str, patch_generator_dir: str, -): +) -> int: + """Return values:" + "0: Applied successfully" + "1: Nothing applied" + "2: Some changes applied successfully""" + retval = -1 # -1 -> nothing seen so far # get list of applicable suggestions applicable_suggestions = [name for name in os.listdir(patch_generator_dir)] @@ -46,11 +51,25 @@ def apply_patches( # write updated applied suggestions to file with open(applied_suggestions_file, "w") as f: f.write(json.dumps(applied_suggestions)) + # update return code + if retval == -1: + retval = 0 + if retval == 1: + # no update for retval = 0 necessary + retval = 2 else: print("Applying suggestion", suggestion_id, "not successful.") + # update return code + if retval == -1: + retval = 1 + if retval == 0: + retval = 2 else: if arguments.verbose: print("Nothing to apply for suggestion ", suggestion_id) + if retval == -1: + retval = 0 + return retval def __apply_file_patches( diff --git a/discopop_library/PatchApplicator/clear.py b/discopop_library/PatchApplicator/clear.py index fe3ee95ff..9e6361d07 100644 --- a/discopop_library/PatchApplicator/clear.py +++ b/discopop_library/PatchApplicator/clear.py @@ -20,7 +20,7 @@ def clear_patches( arguments: PatchApplicatorArguments, applied_suggestions_file: str, patch_generator_dir: str, -): +) -> int: # save the currently applied suggestions. Overwrites old saves shutil.copyfile(applied_suggestions_file, applied_suggestions_file + ".save") @@ -30,4 +30,8 @@ def clear_patches( # rollback all suggestions in inverse order applied_suggestions.reverse() - rollback_patches(applied_suggestions, file_mapping, arguments, applied_suggestions_file, patch_generator_dir) + retval = rollback_patches( + applied_suggestions, file_mapping, arguments, applied_suggestions_file, patch_generator_dir + ) + + return retval diff --git a/discopop_library/PatchApplicator/load.py b/discopop_library/PatchApplicator/load.py index fa0bc0d57..b2647ef47 100644 --- a/discopop_library/PatchApplicator/load.py +++ b/discopop_library/PatchApplicator/load.py @@ -19,7 +19,7 @@ def load_patches( arguments: PatchApplicatorArguments, applied_suggestions_file: str, patch_generator_dir: str, -): +) -> int: # check if a saved configuration exists if not os.path.exists(applied_suggestions_file + ".save"): raise FileNotFoundError( @@ -31,4 +31,8 @@ def load_patches( suggestions_to_be_applied = json.loads(f.read())["applied"] # apply all suggestions - apply_patches(suggestions_to_be_applied, file_mapping, arguments, applied_suggestions_file, patch_generator_dir) + retval = apply_patches( + suggestions_to_be_applied, file_mapping, arguments, applied_suggestions_file, patch_generator_dir + ) + + return retval diff --git a/discopop_library/PatchApplicator/patch_applicator.py b/discopop_library/PatchApplicator/patch_applicator.py index 40736b8bf..47e7b943a 100644 --- a/discopop_library/PatchApplicator/patch_applicator.py +++ b/discopop_library/PatchApplicator/patch_applicator.py @@ -17,7 +17,12 @@ from discopop_library.PathManagement.PathManagement import load_file_mapping -def run(arguments: PatchApplicatorArguments): +def run(arguments: PatchApplicatorArguments) -> int: + """Return values:" + "0: Applied successfully" + "1: Nothing applied" + "2: Some changes applied successfully""" + if arguments.verbose: print("Started DiscoPoP Patch Applicator...") print("Working directory: ", os.getcwd()) @@ -61,16 +66,21 @@ def run(arguments: PatchApplicatorArguments): ) # handle arguments + retval = 0 if len(arguments.apply) > 0: - apply_patches(arguments.apply, file_mapping, arguments, applied_suggestions_file, patch_generator_dir) + retval = apply_patches(arguments.apply, file_mapping, arguments, applied_suggestions_file, patch_generator_dir) elif len(arguments.rollback) > 0: - rollback_patches(arguments.rollback, file_mapping, arguments, applied_suggestions_file, patch_generator_dir) + retval = rollback_patches( + arguments.rollback, file_mapping, arguments, applied_suggestions_file, patch_generator_dir + ) elif arguments.clear: - clear_patches(file_mapping, arguments, applied_suggestions_file, patch_generator_dir) + retval = clear_patches(file_mapping, arguments, applied_suggestions_file, patch_generator_dir) elif arguments.load: - load_patches(file_mapping, arguments, applied_suggestions_file, patch_generator_dir) + retval = load_patches(file_mapping, arguments, applied_suggestions_file, patch_generator_dir) elif arguments.list: print("Applied suggestions: ", list_applied_suggestions(arguments, applied_suggestions_file)) if arguments.verbose: print("Done.") + + return retval diff --git a/discopop_library/PatchApplicator/rollback.py b/discopop_library/PatchApplicator/rollback.py index b2bf0a52b..6389f1e87 100644 --- a/discopop_library/PatchApplicator/rollback.py +++ b/discopop_library/PatchApplicator/rollback.py @@ -22,7 +22,12 @@ def rollback_patches( arguments: PatchApplicatorArguments, applied_suggestions_file: str, patch_generator_dir: str, -): +) -> int: + """Return values:" + "0: Applied successfully" + "1: Nothing applied" + "2: Some changes applied successfully""" + retval = -1 # -1 -> nothing seen so far # get list of applicable suggestions applicable_suggestions = [name for name in os.listdir(patch_generator_dir)] @@ -46,11 +51,25 @@ def rollback_patches( # write updated applied suggestions to file with open(applied_suggestions_file, "w") as f: f.write(json.dumps(applied_suggestions)) + # update return code + if retval == -1: + retval = 0 + if retval == 1: + # no update for retval = 0 necessary + retval = 2 else: print("Rollback of suggestion", suggestion_id, "not successful.") + # update return code + if retval == -1: + retval = 1 + if retval == 0: + retval = 2 else: if arguments.verbose: print("Nothing to rollback for suggestion ", suggestion_id) + if retval == -1: + retval = 0 + return retval def __rollback_file_patches( From 4a325066fb1d4912f470e84a15ad9223d43d7363 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Mon, 6 Nov 2023 10:21:23 +0100 Subject: [PATCH 14/23] feat(patch_generator): read CC /CXX from build_config.txt --- .../PatchGenerator/PatchGeneratorArguments.py | 18 ++++++++++ discopop_library/PatchGenerator/__main__.py | 35 +++++++++++++++++-- .../PatchGenerator/patch_generator.py | 4 ++- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/discopop_library/PatchGenerator/PatchGeneratorArguments.py b/discopop_library/PatchGenerator/PatchGeneratorArguments.py index 6887aeb05..8f7eac8e3 100644 --- a/discopop_library/PatchGenerator/PatchGeneratorArguments.py +++ b/discopop_library/PatchGenerator/PatchGeneratorArguments.py @@ -5,6 +5,7 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. +import os.path from dataclasses import dataclass @@ -13,10 +14,27 @@ class PatchGeneratorArguments(object): """Container Class for the arguments passed to the discopop_patch_generator""" verbose: bool + discopop_build_path: str + CC: str + CXX: str def __post_init__(self): self.__validate() def __validate(self): """Validate the arguments passed to the discopop_explorer, e.g check if given files exist""" + if self.verbose: + print("Configuration:") + print("\tDP BUILD PATH: ", self.discopop_build_path) + print("\tCC: ", self.CC) + print("\tCXX: ", self.CXX) + # check if build directory exists + if not os.path.exists(self.discopop_build_path): + raise FileNotFoundError(self.discopop_build_path) + # check if CC and CXX exist + if not os.path.exists(self.CC): + raise FileNotFoundError(self.CC) + if not os.path.exists(self.CXX): + raise FileNotFoundError(self.CXX) + pass diff --git a/discopop_library/PatchGenerator/__main__.py b/discopop_library/PatchGenerator/__main__.py index 8b171928f..b3417aedf 100644 --- a/discopop_library/PatchGenerator/__main__.py +++ b/discopop_library/PatchGenerator/__main__.py @@ -5,8 +5,9 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. - +import os.path from argparse import ArgumentParser +from pathlib import Path from discopop_library.PatchGenerator.PatchGeneratorArguments import PatchGeneratorArguments from discopop_library.PatchGenerator.patch_generator import run @@ -24,12 +25,42 @@ def parse_args() -> PatchGeneratorArguments: # fmt: off parser.add_argument("--verbose", action="store_true", help="Enable verbose output.") + parser.add_argument( + "--dp-build-path", type=str, required=True, + help="Path to DiscoPoP build folder" + ) # EXPERIMENTAL FLAGS: # fmt: on arguments = parser.parse_args() - return PatchGeneratorArguments(verbose=arguments.verbose) + # determine CC and CXX + with open(os.path.join(arguments.dp_build_path, "build_config.txt"), "r") as f: + for line in f.readlines(): + if line.startswith("LLVM_BIN_DIR="): + line = line.replace("\n", "") + llvm_bin_dir = line[len("LLVM_BIN_DIR=") :] + # determine CC + if os.path.exists(os.path.join(llvm_bin_dir, "clang")): + arguments.cc = os.path.join(llvm_bin_dir, "clang") + elif os.path.exists(os.path.join(llvm_bin_dir, "clang-11")): + arguments.cc = os.path.join(llvm_bin_dir, "clang-11") + else: + raise ValueError("Could not determine CC from discopop build path: ", arguments.dp_build_path) + + # determine CXX + if os.path.exists(os.path.join(llvm_bin_dir, "clang++")): + arguments.cxx = os.path.join(llvm_bin_dir, "clang++") + elif os.path.exists(os.path.join(llvm_bin_dir, "clang++-11")): + arguments.cxx = os.path.join(llvm_bin_dir, "clang++-11") + else: + raise ValueError("Could not determine CXX from discopop build path: ", arguments.dp_build_path) + + break + + return PatchGeneratorArguments( + verbose=arguments.verbose, discopop_build_path=arguments.dp_build_path, CC=arguments.cc, CXX=arguments.cxx + ) def main(): diff --git a/discopop_library/PatchGenerator/patch_generator.py b/discopop_library/PatchGenerator/patch_generator.py index 31cd8e675..a8e0b622a 100644 --- a/discopop_library/PatchGenerator/patch_generator.py +++ b/discopop_library/PatchGenerator/patch_generator.py @@ -59,7 +59,9 @@ def run(arguments: PatchGeneratorArguments): for suggestion in patterns_by_type[suggestion_type]: if arguments.verbose: print("Suggestion: ", suggestion) - file_id_to_modified_code: Dict[int, str] = from_json_strings(file_mapping, {suggestion_type: [suggestion]}) + file_id_to_modified_code: Dict[int, str] = from_json_strings( + file_mapping, {suggestion_type: [suggestion]}, CC=arguments.CC, CXX=arguments.CXX + ) # create patches from the modified codes file_id_to_patches: Dict[int, str] = get_diffs_from_modified_code( file_mapping, file_id_to_modified_code, arguments From 1b6aa7899b2c532de5769e106286d50f2562ac3d Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Mon, 6 Nov 2023 10:27:59 +0100 Subject: [PATCH 15/23] fix(patch_generator): disabled compile checks for now --- discopop_library/PatchGenerator/patch_generator.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/discopop_library/PatchGenerator/patch_generator.py b/discopop_library/PatchGenerator/patch_generator.py index a8e0b622a..36b1834a3 100644 --- a/discopop_library/PatchGenerator/patch_generator.py +++ b/discopop_library/PatchGenerator/patch_generator.py @@ -60,7 +60,11 @@ def run(arguments: PatchGeneratorArguments): if arguments.verbose: print("Suggestion: ", suggestion) file_id_to_modified_code: Dict[int, str] = from_json_strings( - file_mapping, {suggestion_type: [suggestion]}, CC=arguments.CC, CXX=arguments.CXX + file_mapping, + {suggestion_type: [suggestion]}, + CC=arguments.CC, + CXX=arguments.CXX, + skip_compilation_check=True, ) # create patches from the modified codes file_id_to_patches: Dict[int, str] = get_diffs_from_modified_code( From 3b6f0cdbf240732c4a22bc201c2042fa1825c73d Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Tue, 7 Nov 2023 12:53:28 +0100 Subject: [PATCH 16/23] fix(patch_generator): check for permission errors and ignore such files --- discopop_library/PatchGenerator/diffs.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/discopop_library/PatchGenerator/diffs.py b/discopop_library/PatchGenerator/diffs.py index c12f5857d..4b316b60f 100644 --- a/discopop_library/PatchGenerator/diffs.py +++ b/discopop_library/PatchGenerator/diffs.py @@ -26,8 +26,11 @@ def get_diffs_from_modified_code( print("Original: ", original_file_path) print("Modified: ", modified_file_path) - with open(modified_file_path, "w") as f: - f.write(file_id_to_modified_code[file_id]) + try: + with open(modified_file_path, "w") as f: + f.write(file_id_to_modified_code[file_id]) + except PermissionError: + continue # calculate diff diff_name = original_file_path.parent / (original_file_path.name + ".discopop_patch_generator.diff") From 0e89b90931c6c78155a8712778d4d27c9266a217 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Tue, 7 Nov 2023 13:42:59 +0100 Subject: [PATCH 17/23] fix: minor fixes and improvements --- discopop_library/PatchApplicator/__main__.py | 7 ++----- discopop_library/PatchGenerator/diffs.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/discopop_library/PatchApplicator/__main__.py b/discopop_library/PatchApplicator/__main__.py index f5a9f848c..10b6ded53 100644 --- a/discopop_library/PatchApplicator/__main__.py +++ b/discopop_library/PatchApplicator/__main__.py @@ -59,11 +59,8 @@ def main() -> int: "2: Some changes applied successfully """ retval = 0 - try: - arguments = parse_args() - retval = run(arguments) - except FileNotFoundError: - retval = 1 + arguments = parse_args() + retval = run(arguments) return retval diff --git a/discopop_library/PatchGenerator/diffs.py b/discopop_library/PatchGenerator/diffs.py index 4b316b60f..291f1d062 100644 --- a/discopop_library/PatchGenerator/diffs.py +++ b/discopop_library/PatchGenerator/diffs.py @@ -36,7 +36,7 @@ def get_diffs_from_modified_code( diff_name = original_file_path.parent / (original_file_path.name + ".discopop_patch_generator.diff") command = [ "diff", - "-Naru", + "-Naru50", original_file_path.as_posix(), modified_file_path.as_posix(), ] From 361cb99a92bce1bea6f46ead1acce040420373d7 Mon Sep 17 00:00:00 2001 From: goerlibe <23436477+goerlibe@users.noreply.github.com> Date: Thu, 9 Nov 2023 15:18:35 +0100 Subject: [PATCH 18/23] fix: -Naru50 --> -Naru --- discopop_library/PatchGenerator/diffs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discopop_library/PatchGenerator/diffs.py b/discopop_library/PatchGenerator/diffs.py index 291f1d062..4b316b60f 100644 --- a/discopop_library/PatchGenerator/diffs.py +++ b/discopop_library/PatchGenerator/diffs.py @@ -36,7 +36,7 @@ def get_diffs_from_modified_code( diff_name = original_file_path.parent / (original_file_path.name + ".discopop_patch_generator.diff") command = [ "diff", - "-Naru50", + "-Naru", original_file_path.as_posix(), modified_file_path.as_posix(), ] From 8a11c29482c8ade1338da8e3e73fee9f23a48c5e Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Thu, 9 Nov 2023 15:32:44 +0100 Subject: [PATCH 19/23] fix: offset number of added lines in case of multiple lines --- discopop_library/LineMapping/diff_modifications.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discopop_library/LineMapping/diff_modifications.py b/discopop_library/LineMapping/diff_modifications.py index 873c8caa8..26f5487f1 100644 --- a/discopop_library/LineMapping/diff_modifications.py +++ b/discopop_library/LineMapping/diff_modifications.py @@ -75,7 +75,7 @@ def apply_line_mapping_modifications_from_diff(file_id: int, diff: str): base_line = int(lhs) rhs = diff_entry[diff_entry.index("a") + 1 :] if "," in rhs: - added_lines_count = int(rhs.split(",")[1]) - int(rhs.split(",")[0]) + added_lines_count = int(rhs.split(",")[1]) - int(rhs.split(",")[0]) + 1 else: added_lines_count = 1 for key in line_mapping[str(file_id)]: From 8de3f0d8299dd4dfe88ef6d085a80f2bcefd830a Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Thu, 9 Nov 2023 15:32:52 +0100 Subject: [PATCH 20/23] fix: offset number of added lines in case of multiple lines --- discopop_library/LineMapping/diff_modifications.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discopop_library/LineMapping/diff_modifications.py b/discopop_library/LineMapping/diff_modifications.py index 26f5487f1..e9a02986c 100644 --- a/discopop_library/LineMapping/diff_modifications.py +++ b/discopop_library/LineMapping/diff_modifications.py @@ -112,7 +112,7 @@ def apply_line_mapping_modifications_from_diff(file_id: int, diff: str): deleted_lines_count = 1 rhs = diff_entry[diff_entry.index("c") + 1 :] if "," in rhs: - added_lines_count = int(rhs.split(",")[1]) - int(rhs.split(",")[0]) + added_lines_count = int(rhs.split(",")[1]) - int(rhs.split(",")[0]) + 1 else: added_lines_count = 1 From bdfaf62e2d15bfef16a1fc1957c5d19712a42878 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Fri, 10 Nov 2023 07:13:00 +0100 Subject: [PATCH 21/23] feat(patch_generator): initialize applied_suggestions.json --- .../PatchGenerator/patch_generator.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/discopop_library/PatchGenerator/patch_generator.py b/discopop_library/PatchGenerator/patch_generator.py index 36b1834a3..207e9583f 100644 --- a/discopop_library/PatchGenerator/patch_generator.py +++ b/discopop_library/PatchGenerator/patch_generator.py @@ -26,6 +26,21 @@ def run(arguments: PatchGeneratorArguments): if not os.path.exists(patch_generator_dir): os.mkdir(patch_generator_dir) + # for compatibility reasons, initialize the file to store applied patches if it doesn't exist already + # create a directory for the patch applicator + patch_applicator_dir = os.path.join(os.getcwd(), "patch_applicator") + if not os.path.exists(patch_applicator_dir): + if arguments.verbose: + print("Creating patch_applicator directory...") + os.mkdir(patch_applicator_dir) + # create a file to store applied suggestions + applied_suggestions_file = os.path.join(patch_applicator_dir, "applied_suggestions.json") + if not os.path.exists(applied_suggestions_file): + if arguments.verbose: + print("Creating applied_suggestions.json file...") + with open(applied_suggestions_file, "w+") as f: + f.write(json.dumps({"applied": []})) + pattern_file_path = os.path.join(os.getcwd(), "explorer", "patterns.json") if not os.path.exists(pattern_file_path): raise FileNotFoundError( From e71193e773cc15c47de161bc088240e8dc88f6f2 Mon Sep 17 00:00:00 2001 From: goerlibe <23436477+goerlibe@users.noreply.github.com> Date: Fri, 10 Nov 2023 10:12:50 +0100 Subject: [PATCH 22/23] feat(code_generator): indentation of pragmas --- .../CodeGenerator/classes/ContentBuffer.py | 11 +++++++++-- discopop_library/CodeGenerator/classes/Line.py | 3 +++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/discopop_library/CodeGenerator/classes/ContentBuffer.py b/discopop_library/CodeGenerator/classes/ContentBuffer.py index a49831fa7..2da03f6e3 100644 --- a/discopop_library/CodeGenerator/classes/ContentBuffer.py +++ b/discopop_library/CodeGenerator/classes/ContentBuffer.py @@ -59,14 +59,16 @@ def get_modified_source_code(self) -> str: result += line.content return result - def append_line_before(self, parent_line_num: int, line: Line): + def append_line_before(self, parent_line_num: int, line: Line, match_indentation: bool = True): """Appends line before the specified parent_line_num""" for idx, potential_parent_line in enumerate(self.lines): if potential_parent_line.line_num == parent_line_num: + if match_indentation: + line.content = potential_parent_line.get_indentation() + line.content self.lines.insert(idx, line) return - def append_line_after(self, parent_line_num: int, line: Line): + def append_line_after(self, parent_line_num: int, line: Line, match_indentation: bool = True): """Appends line after the specified parent_line_num""" for idx, potential_parent_line in enumerate(self.lines): if potential_parent_line.line_num == parent_line_num: @@ -74,6 +76,8 @@ def append_line_after(self, parent_line_num: int, line: Line): self.lines.insert(idx + 1, line) else: self.lines.append(line) + if match_indentation: + line.content = potential_parent_line.get_indentation() + line.content return def add_pragma( @@ -86,6 +90,7 @@ def add_pragma( compile_check_command: Optional[str] = None, CC="clang", CXX="clang++", + match_indentation: bool = True, ) -> bool: """insert pragma into the maintained list of source code lines. Returns True if the pragma resulted in a valid (resp. compilable) code transformation. @@ -106,6 +111,8 @@ def add_pragma( # construct line pragma_line = self.line_type(pragma.start_line) + + # adding as comment if add_as_comment: pragma_line.content = "// " else: diff --git a/discopop_library/CodeGenerator/classes/Line.py b/discopop_library/CodeGenerator/classes/Line.py index eb3b2d199..e43010033 100644 --- a/discopop_library/CodeGenerator/classes/Line.py +++ b/discopop_library/CodeGenerator/classes/Line.py @@ -22,3 +22,6 @@ def __init__(self, parent_line_num: int, line_num=None, content=""): self.owns_region = None self.belongs_to_regions = [] self.belongs_to_original_line = parent_line_num + + def get_indentation(self) -> str: + return self.content[: len(self.content) - len(self.content.lstrip())] From 631673922e084a8d59dc874bb433a4ceedd4c9d9 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Tue, 14 Nov 2023 02:12:23 +0100 Subject: [PATCH 23/23] fix: offset number of removed lines in case of multiple lines --- discopop_library/LineMapping/diff_modifications.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/discopop_library/LineMapping/diff_modifications.py b/discopop_library/LineMapping/diff_modifications.py index e9a02986c..f12cd51c2 100644 --- a/discopop_library/LineMapping/diff_modifications.py +++ b/discopop_library/LineMapping/diff_modifications.py @@ -89,7 +89,7 @@ def apply_line_mapping_modifications_from_diff(file_id: int, diff: str): lhs = diff_entry[: diff_entry.index("d")] if "," in lhs: base_line = int(lhs.split(",")[0]) - deleted_lines_count = int(lhs.split(",")[1]) - int(lhs.split(",")[0]) + deleted_lines_count = int(lhs.split(",")[1]) - int(lhs.split(",")[0]) + 1 else: base_line = int(lhs) deleted_lines_count = 1 @@ -106,7 +106,7 @@ def apply_line_mapping_modifications_from_diff(file_id: int, diff: str): lhs = diff_entry[: diff_entry.index("c")] if "," in lhs: base_line = int(lhs.split(",")[0]) - deleted_lines_count = int(lhs.split(",")[1]) - int(lhs.split(",")[0]) + deleted_lines_count = int(lhs.split(",")[1]) - int(lhs.split(",")[0]) + 1 else: base_line = int(lhs) deleted_lines_count = 1