diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54ce27652..8ec5a873b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,16 +54,16 @@ jobs: run: python -m mypy --config-file=mypy.ini -p discopop_wizard - name: "Check formatting of DiscoPoP Explorer" - run: python -m black -l 100 --check discopop_explorer + run: python -m black -l 120 --check discopop_explorer - name: "Check formatting of DiscoPoP Library" - run: python -m black -l 100 --check discopop_library + run: python -m black -l 120 --check discopop_library - name: "Check formatting of DiscoPoP Profiler" - run: python -m black -l 100 --check discopop_profiler + run: python -m black -l 120 --check discopop_profiler - name: "Check formatting of DiscoPoP Wizard" - run: python -m black -l 100 --check discopop_wizard + run: python -m black -l 120 --check discopop_wizard - name: Test DiscoPop Explorer - DISABLED run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 381b1a351..8ea96a8c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,7 +40,7 @@ repos: rev: 23.7.0 hooks: - id: black # run black fromatter on all files - args: [--line-length=100] + args: [--line-length=120] - repo: local diff --git a/discopop_explorer/PETGraphX.py b/discopop_explorer/PETGraphX.py index 2bdd67631..91421c61f 100644 --- a/discopop_explorer/PETGraphX.py +++ b/discopop_explorer/PETGraphX.py @@ -9,20 +9,18 @@ from __future__ import annotations import copy -import sys -from time import sleep -from typing import Dict, List, Sequence, Tuple, Set, Optional, Type, TypeVar, cast, Union, overload -from enum import IntEnum, Enum import itertools +from enum import IntEnum, Enum +from typing import Dict, List, Sequence, Tuple, Set, Optional, Type, TypeVar, cast, Union, overload, Any import jsonpickle # type:ignore import matplotlib.pyplot as plt # type:ignore import networkx as nx # type:ignore +from alive_progress import alive_bar # type: ignore from lxml.objectify import ObjectifiedElement # type:ignore from .parser import LoopData, readlineToCUIdMap, writelineToCUIdMap, DependenceItem from .variable import Variable -from alive_progress import alive_bar # type: ignore # unused @@ -221,11 +219,7 @@ def __str__(self): return self.id def __eq__(self, other): - return ( - isinstance(other, Node) - and other.file_id == self.file_id - and other.node_id == self.node_id - ) + return isinstance(other, Node) and other.file_id == self.file_id and other.node_id == self.node_id def __hash__(self): return hash(self.id) @@ -409,9 +403,7 @@ def get_immediate_post_dominators(self, pet: PETGraphX) -> Dict[NodeID, NodeID]: # reverse edges immediate_post_dominators: Set[Tuple[NodeID, NodeID]] = set() for exit_cu_id in exit_cu_ids: - immediate_post_dominators.update( - nx.immediate_dominators(copied_graph.reverse(), exit_cu_id).items() - ) + immediate_post_dominators.update(nx.immediate_dominators(copied_graph.reverse(), exit_cu_id).items()) immediate_post_dominators_dict = dict(immediate_post_dominators) # add trivial cases for missing modes @@ -434,10 +426,8 @@ def get_immediate_post_dominators(self, pet: PETGraphX) -> Dict[NodeID, NodeID]: visited = set() use_original = False while ( - pet.node_at(node_id).get_parent_id(pet) - == pet.node_at(post_dom_id).get_parent_id(pet) - and type(pet.node_at(cast(NodeID, pet.node_at(post_dom_id).get_parent_id(pet)))) - != FunctionNode + pet.node_at(node_id).get_parent_id(pet) == pet.node_at(post_dom_id).get_parent_id(pet) + and type(pet.node_at(cast(NodeID, pet.node_at(post_dom_id).get_parent_id(pet)))) != FunctionNode ): if post_dom_id in visited: # cycle detected! @@ -475,9 +465,7 @@ def get_memory_accesses( for mem_reg in writes_by_device[device_id][child_id]: if mem_reg not in self.memory_accesses[device_id]: self.memory_accesses[device_id][mem_reg] = set() - self.memory_accesses[device_id][mem_reg].update( - writes_by_device[device_id][child_id][mem_reg] - ) + self.memory_accesses[device_id][mem_reg].update(writes_by_device[device_id][child_id][mem_reg]) return self.memory_accesses @@ -580,7 +568,7 @@ class PETGraphX(object): g: nx.MultiDiGraph reduction_vars: List[Dict[str, str]] main: Node - pos: Dict + pos: Dict[Any, Any] def __init__(self, g: nx.MultiDiGraph, reduction_vars: List[Dict[str, str]], pos): self.g = g @@ -682,10 +670,7 @@ def from_parsed_input( for var in itertools.chain(source_node.local_vars, source_node.global_vars): vars_in_source_node.add(var.name) - if ( - dep.var_name not in vars_in_sink_node - and dep.var_name not in vars_in_source_node - ): + if dep.var_name not in vars_in_sink_node and dep.var_name not in vars_in_source_node: continue if sink_cu_id and source_cu_id: g.add_edge(sink_cu_id, source_cu_id, data=parse_dependency(dep)) @@ -698,9 +683,7 @@ def map_static_and_dynamic_dependencies(self): mem_reg_mappings: Dict[MemoryRegion, Set[MemoryRegion]] = dict() # initialize mappings for node_id in [n.id for n in self.all_nodes(CUNode)]: - out_deps = [ - (s, t, d) for s, t, d in self.out_edges(node_id) if d.etype == EdgeType.DATA - ] + out_deps = [(s, t, d) for s, t, d in self.out_edges(node_id) if d.etype == EdgeType.DATA] # for outgoing dependencies, the scope must be equal # as a result, comparing variable names to match memory regions is valid @@ -723,9 +706,7 @@ def map_static_and_dynamic_dependencies(self): print("\t\tInstantiating static dependencies...", end=" ") # create copies of static dependency edges for all dynamic mappings for node_id in [n.id for n in self.all_nodes(CUNode)]: - out_deps = [ - (s, t, d) for s, t, d in self.out_edges(node_id) if d.etype == EdgeType.DATA - ] + out_deps = [(s, t, d) for s, t, d in self.out_edges(node_id) if d.etype == EdgeType.DATA] for s, t, d in out_deps: if d.memory_region.startswith("S"): # Static dependency found @@ -733,9 +714,7 @@ def map_static_and_dynamic_dependencies(self): if d.memory_region in mem_reg_mappings: # create instances for all dynamic mappings for dynamic_mapping in [ - mapping - for mapping in mem_reg_mappings[d.memory_region] - if not mapping.startswith("S") + mapping for mapping in mem_reg_mappings[d.memory_region] if not mapping.startswith("S") ]: edge_data = copy.deepcopy(d) edge_data.memory_region = dynamic_mapping @@ -917,9 +896,7 @@ def show(self): self.g, pos, edge_color="orange", - edgelist=[ - e for e in self.g.edges(data="data") if e[2].etype == EdgeType.PRODUCE_CONSUME - ], + edgelist=[e for e in self.g.edges(data="data") if e[2].etype == EdgeType.PRODUCE_CONSUME], ) plt.show() @@ -989,9 +966,7 @@ def subtree_of_type(self, root: Node) -> List[Node]: ... @overload - def subtree_of_type( - self, root: Node, type: Union[Type[NodeT], Tuple[Type[NodeT], ...]] - ) -> List[NodeT]: + def subtree_of_type(self, root: Node, type: Union[Type[NodeT], Tuple[Type[NodeT], ...]]) -> List[NodeT]: ... def subtree_of_type(self, root, type=Node): @@ -1055,10 +1030,7 @@ def direct_children_or_called_nodes(self, root: Node) -> List[Node]: :param root: root node :return: list of direct children """ - return [ - self.node_at(t) - for s, t, d in self.out_edges(root.id, [EdgeType.CHILD, EdgeType.CALLSNODE]) - ] + return [self.node_at(t) for s, t, d in self.out_edges(root.id, [EdgeType.CHILD, EdgeType.CALLSNODE])] def direct_children(self, root: Node) -> List[Node]: """Gets direct children of any type. This includes called nodes! @@ -1075,10 +1047,7 @@ def direct_children_or_called_nodes_of_type(self, root: Node, type: Type[NodeT]) :param type: type of children :return: list of direct children """ - nodes = [ - self.node_at(t) - for s, t, d in self.out_edges(root.id, [EdgeType.CHILD, EdgeType.CALLSNODE]) - ] + nodes = [self.node_at(t) for s, t, d in self.out_edges(root.id, [EdgeType.CHILD, EdgeType.CALLSNODE])] return [t for t in nodes if isinstance(t, type)] @@ -1089,9 +1058,7 @@ def is_reduction_var(self, line: LineID, name: str) -> bool: :param name: variable name :return: true if is reduction variable """ - return any( - rv for rv in self.reduction_vars if rv["loop_line"] == line and rv["name"] == name - ) + return any(rv for rv in self.reduction_vars if rv["loop_line"] == line and rv["name"] == name) def depends_ignore_readonly(self, source: Node, target: Node, root_loop: Node) -> bool: """Detects if source node or one of it's children has a RAW dependency to target node or one of it's children @@ -1155,8 +1122,7 @@ def unused_check_alias(self, s: NodeID, t: NodeID, d: Dependency, root_loop: Nod d_var_name_str = cast(str, d.var_name) if self.unused_is_global(d_var_name_str, sub) and not ( - self.is_passed_by_reference(d, parent_func_sink) - and self.is_passed_by_reference(d, parent_func_source) + self.is_passed_by_reference(d, parent_func_sink) and self.is_passed_by_reference(d, parent_func_source) ): return res return not res @@ -1206,19 +1172,13 @@ def unused_get_first_written_vars_in_loop( for i in raw: if i[2].var_name == var and i[0] in loop_node_ids and i[1] in loop_node_ids: for e in itertools.chain(war, waw): - if ( - e[2].var_name == var - and e[0] in loop_node_ids - and e[1] in loop_node_ids - ): + if e[2].var_name == var and e[0] in loop_node_ids and e[1] in loop_node_ids: if e[2].sink_line == i[2].source_line: fwVars.add(var) return fwVars - def get_dep( - self, node: Node, dep_type: DepType, reversed: bool - ) -> List[Tuple[NodeID, NodeID, Dependency]]: + def get_dep(self, node: Node, dep_type: DepType, reversed: bool) -> List[Tuple[NodeID, NodeID, Dependency]]: """Searches all dependencies of specified type :param node: node @@ -1228,11 +1188,7 @@ def get_dep( """ return [ e - for e in ( - self.in_edges(node.id, EdgeType.DATA) - if reversed - else self.out_edges(node.id, EdgeType.DATA) - ) + for e in (self.in_edges(node.id, EdgeType.DATA) if reversed else self.out_edges(node.id, EdgeType.DATA)) if e[2].dtype == dep_type ] @@ -1244,9 +1200,7 @@ def is_scalar_val(self, allVars: List[Variable], var: str) -> bool: """ for x in allVars: if x.name == var: - return not ( - x.type.endswith("**") or x.type.startswith("ARRAY" or x.type.startswith("[")) - ) + return not (x.type.endswith("**") or x.type.startswith("ARRAY" or x.type.startswith("["))) else: return False raise ValueError("allVars must not be empty.") @@ -1271,9 +1225,7 @@ def get_variables(self, nodes: Sequence[Node]) -> Dict[Variable, Set[MemoryRegio # since the variable name is checked for equality afterwards, # it is safe to consider incoming dependencies at this point as well. # Note that INIT type edges are considered as well! - for _, _, dep in self.out_edges(node.id, EdgeType.DATA) + self.in_edges( - node.id, EdgeType.DATA - ): + for _, _, dep in self.out_edges(node.id, EdgeType.DATA) + self.in_edges(node.id, EdgeType.DATA): if dep.var_name == var_name.name: if dep.memory_region is not None: res[var_name].add(dep.memory_region) @@ -1293,9 +1245,7 @@ def get_undefined_variables_inside_loop( if var.defLine == "LineNotFound" or "0:" in var.defLine: dummyVariables.append(var) elif not include_global_vars: - if var.defLine == "GlobalVar" and not self.is_reduction_var( - root_loop.start_position(), var.name - ): + if var.defLine == "GlobalVar" and not self.is_reduction_var(root_loop.start_position(), var.name): dummyVariables.append(var) # vars = list(set(vars) ^ set(dummyVariables)) @@ -1305,10 +1255,7 @@ def get_undefined_variables_inside_loop( # Exclude variables which are defined inside the loop for var in vars: - if ( - var.defLine >= root_loop.start_position() - and var.defLine <= root_loop.end_position() - ): + if var.defLine >= root_loop.start_position() and var.defLine <= root_loop.end_position(): definedVarsInLoop.append(var) # vars = list(set(vars) ^ set(definedVarsInLoop)) @@ -1361,9 +1308,7 @@ def unused_is_first_written_in_loop(self, dep: Dependency, root_loop: Node): # None may occur because __get_variables doesn't check for actual elements return result - def is_loop_index( - self, var_name: Optional[str], loops_start_lines: List[LineID], children: Sequence[Node] - ) -> bool: + def is_loop_index(self, var_name: Optional[str], loops_start_lines: List[LineID], children: Sequence[Node]) -> bool: """Checks, whether the variable is a loop index. :param var_name: name of the variable @@ -1381,11 +1326,7 @@ def is_loop_index( for s, t, d in self.out_edges(c.id, EdgeType.DATA) if d.dtype == DepType.RAW and d.var_name == var_name ]: - if ( - d.sink_line == d.source_line - and d.source_line in loops_start_lines - and self.node_at(t) in children - ): + if d.sink_line == d.source_line and d.source_line in loops_start_lines and self.node_at(t) in children: return True return False @@ -1417,14 +1358,10 @@ def is_readonly_inside_loop_body( # (sink is always inside loop for waw/war) if dep.memory_region == d.memory_region and not (d.sink_line in loops_start_lines): return False - for t, d in [ - (t, d) for s, t, d in self.in_edges(v.id, EdgeType.DATA) if d.dtype == DepType.RAW - ]: + for t, d in [(t, d) for s, t, d in self.in_edges(v.id, EdgeType.DATA) if d.dtype == DepType.RAW]: # If there is a reverse raw dependency for var, then var is written in loop # (source is always inside loop for reverse raw) - if dep.memory_region == d.memory_region and not ( - d.source_line in loops_start_lines - ): + if dep.memory_region == d.memory_region and not (d.source_line in loops_start_lines): return False return True @@ -1476,9 +1413,7 @@ def get_left_right_subtree( ) else: stack.extend( - self.direct_children(current) - if right_subtree - else reversed(self.direct_children(current)) + self.direct_children(current) if right_subtree else reversed(self.direct_children(current)) ) return res @@ -1545,9 +1480,7 @@ def check_reachability(self, target: Node, source: Node, edge_types: List[EdgeTy cur_node = queue.pop(0) visited.append(cur_node.id) tmp_list = [ - (s, t, e) - for s, t, e in self.in_edges(cur_node.id) - if s not in visited and e.etype in edge_types + (s, t, e) for s, t, e in self.in_edges(cur_node.id) if s not in visited and e.etype in edge_types ] for e in tmp_list: if self.node_at(e[0]) == source: @@ -1564,9 +1497,7 @@ def is_predecessor(self, source_id: NodeID, target_id: NodeID) -> bool: source_parent_function = self.get_parent_function(self.node_at(source_id)) target_parent_function = self.get_parent_function(self.node_at(target_id)) if source_parent_function != target_parent_function: - for callee_id in [ - s for s, _, _ in self.in_edges(source_parent_function.id, EdgeType.CALLSNODE) - ]: + for callee_id in [s for s, _, _ in self.in_edges(source_parent_function.id, EdgeType.CALLSNODE)]: if self.is_predecessor(callee_id, target_id): return True @@ -1585,21 +1516,15 @@ def is_predecessor(self, source_id: NodeID, target_id: NodeID) -> bool: visited.append(current) # add direct successors to queue queue += [ - n.id - for n in self.direct_successors(self.node_at(current)) - if n.id not in visited and n.id not in queue + n.id for n in self.direct_successors(self.node_at(current)) if n.id not in visited and n.id not in queue ] # add children to queue queue += [ - n.id - for n in self.direct_children(self.node_at(current)) - if n.id not in visited and n.id not in queue + n.id for n in self.direct_children(self.node_at(current)) if n.id not in visited and n.id not in queue ] # add called functions to queue queue += [ - t - for _, t, _ in self.out_edges(current, EdgeType.CALLSNODE) - if t not in visited and t not in queue + t for _, t, _ in self.out_edges(current, EdgeType.CALLSNODE) if t not in visited and t not in queue ] return False @@ -1629,9 +1554,7 @@ def check_reachability_and_get_path_nodes( cur_node, cur_path = queue.pop(0) visited.append(cur_node.id) tmp_list = [ - (s, t, e) - for s, t, e in self.in_edges(cur_node.id) - if s not in visited and e.etype in edge_types + (s, t, e) for s, t, e in self.in_edges(cur_node.id) if s not in visited and e.etype in edge_types ] for e in tmp_list: if self.node_at(e[0]) == source: @@ -1691,9 +1614,7 @@ def get_memory_regions(self, nodes: List[CUNode], var_name: str) -> Set[MemoryRe mem_regs.add(d.memory_region) return mem_regs - def get_path_nodes_between( - self, target: CUNode, source: CUNode, edge_types: List[EdgeType] - ) -> List[CUNode]: + def get_path_nodes_between(self, target: CUNode, source: CUNode, edge_types: List[EdgeType]) -> List[CUNode]: """get all nodes of all patch which allow reaching target from source via edges of types edge_type. :param pet: PET graph :param source: CUNode @@ -1712,9 +1633,7 @@ def get_path_nodes_between( cur_node, cur_path = queue.pop(0) visited.append(cur_node.id) tmp_list = [ - (s, t, e) - for s, t, e in self.out_edges(cur_node.id) - if t not in visited and e.etype in edge_types + (s, t, e) for s, t, e in self.out_edges(cur_node.id) if t not in visited and e.etype in edge_types ] for e in tmp_list: if self.node_at(e[1]) == target or self.node_at(e[1]) == source: diff --git a/discopop_explorer/__main__.py b/discopop_explorer/__main__.py index 5e0aea5ad..befb358b3 100644 --- a/discopop_explorer/__main__.py +++ b/discopop_explorer/__main__.py @@ -101,9 +101,7 @@ def parse_args() -> ExplorerArguments: arguments = parser.parse_args() # ensure that --cu-inst-res and --llvm-cxxfilt-path are set if --task-pattern is set - if arguments.task_pattern and ( - arguments.cu_inst_res is None or arguments.llvm_cxxfilt_path is None - ): + if arguments.task_pattern and (arguments.cu_inst_res is None or arguments.llvm_cxxfilt_path is None): parser.error("--task-pattern requires --cu-inst-res and --llvm-cxxfilt-path to be set") # ensure that --cu-xml, --dep-file, --loop-counter, --reduction are set if --generate-data-cu-inst is set @@ -120,14 +118,10 @@ def parse_args() -> ExplorerArguments: arguments.fmap = get_path(arguments.path, arguments.fmap) arguments.json = get_path_or_none(arguments.path, arguments.json) arguments.cu_inst_res = get_path_or_none(arguments.path, arguments.cu_inst_res) - arguments.generate_data_cu_inst = get_path_or_none( - arguments.path, arguments.generate_data_cu_inst - ) + arguments.generate_data_cu_inst = get_path_or_none(arguments.path, arguments.generate_data_cu_inst) arguments.profiling = get_path_or_none(arguments.path, arguments.profiling) arguments.dump_pet = get_path_or_none(arguments.path, arguments.dump_pet) - arguments.dump_detection_result = get_path_or_none( - arguments.path, arguments.dump_detection_result - ) + arguments.dump_detection_result = get_path_or_none(arguments.path, arguments.dump_detection_result) arguments.microbench_file = get_path_or_none(arguments.path, arguments.microbench_file) return ExplorerArguments( diff --git a/discopop_explorer/discopop_explorer.py b/discopop_explorer/discopop_explorer.py index 343011cdd..8156c71b3 100644 --- a/discopop_explorer/discopop_explorer.py +++ b/discopop_explorer/discopop_explorer.py @@ -7,27 +7,26 @@ # directory for details. import cProfile -from dataclasses import dataclass -from pathlib import Path -from pluginbase import PluginBase # type:ignore -from typing import List, Optional - import json import os -import pstats2 # type:ignore import sys import time +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional -from .json_serializer import PatternInfoSerializer -from .parser import parse_inputs -from .pattern_detection import PatternDetectorX -from .PETGraphX import PETGraphX +import pstats2 # type:ignore +from pluginbase import PluginBase # type:ignore +from discopop_library.PathManagement.PathManagement import get_path from discopop_library.discopop_optimizer.Microbench.ExtrapInterpolatedMicrobench import ( ExtrapInterpolatedMicrobench, ) -from discopop_library.PathManagement.PathManagement import get_path from discopop_library.result_classes.DetectionResult import DetectionResult +from .PETGraphX import PETGraphX +from .json_serializer import PatternInfoSerializer +from .parser import parse_inputs +from .pattern_detection import PatternDetectorX @dataclass diff --git a/discopop_explorer/generate_Data_CUInst.py b/discopop_explorer/generate_Data_CUInst.py index 05674b648..e64ac4633 100644 --- a/discopop_explorer/generate_Data_CUInst.py +++ b/discopop_explorer/generate_Data_CUInst.py @@ -20,7 +20,6 @@ EdgeType, ) from .parser import parse_inputs -from discopop_library.PathManagement.PathManagement import get_path def __collect_children_ids(pet: PETGraphX, parent_id: NodeID, children_ids: List[NodeID]): @@ -51,9 +50,7 @@ def __recursive_call_inside_loop(pet: PETGraphX, recursive_function_call: str) - return False -def __recursive_function_called_multiple_times_inside_function( - pet: PETGraphX, recursive_function_call: str -) -> bool: +def __recursive_function_called_multiple_times_inside_function(pet: PETGraphX, recursive_function_call: str) -> bool: """checks if the given recursive function is called multiple times from within a functions body :param pet: PET Graph :param recursive_function_call: string representation of a recursive function call, extracted from cu-xml @@ -143,9 +140,7 @@ def __search_recursive_calls(pet: PETGraphX, output_file, node: Node): # check if recursive function call occurs inside loop (check if line contained in lines of any loop cu) contained_in_loop = __recursive_call_inside_loop(pet, recursive_function_call) # check if recursive function call is called multiple times - called_multiple_times = __recursive_function_called_multiple_times_inside_function( - pet, recursive_function_call - ) + called_multiple_times = __recursive_function_called_multiple_times_inside_function(pet, recursive_function_call) # check if recursive function is called inside loop or multiple times if not (contained_in_loop or called_multiple_times): @@ -163,15 +158,9 @@ def __search_recursive_calls(pet: PETGraphX, output_file, node: Node): # node type is not cu so goto next node if not isinstance(pet.node_at(child_id), CUNode): continue - __output_dependencies_of_type( - pet, child_id, children_ids, output_file, DepType.RAW, "|RAW|" - ) - __output_dependencies_of_type( - pet, child_id, children_ids, output_file, DepType.WAR, "|WAR|" - ) - __output_dependencies_of_type( - pet, child_id, children_ids, output_file, DepType.WAW, "|WAW|" - ) + __output_dependencies_of_type(pet, child_id, children_ids, output_file, DepType.RAW, "|RAW|") + __output_dependencies_of_type(pet, child_id, children_ids, output_file, DepType.WAR, "|WAR|") + __output_dependencies_of_type(pet, child_id, children_ids, output_file, DepType.WAW, "|WAW|") output_file.write("\n") @@ -189,9 +178,7 @@ def cu_instantiation_input_cpp(pet: PETGraphX, output_file: str): def wrapper(cu_xml, dep_file, loop_counter_file, reduction_file, output_file): """Wrapper to generate the Data_CUInst.txt file, required for the generation of CUInstResult.txt""" # 1. generate PET Graph - pet = PETGraphX.from_parsed_input( - *parse_inputs(cu_xml, dep_file, loop_counter_file, reduction_file) - ) + pet = PETGraphX.from_parsed_input(*parse_inputs(cu_xml, dep_file, loop_counter_file, reduction_file)) # 2. Generate Data_CUInst.txt cu_instantiation_input_cpp(pet, output_file) diff --git a/discopop_explorer/json_serializer.py b/discopop_explorer/json_serializer.py index bf7c6f114..c0aee07dc 100644 --- a/discopop_explorer/json_serializer.py +++ b/discopop_explorer/json_serializer.py @@ -7,16 +7,17 @@ # directory for details. from json import JSONEncoder +from typing import Dict, Any -from .PETGraphX import Node from discopop_library.result_classes.DetectionResult import DetectionResult +from .PETGraphX import Node from .pattern_detectors.PatternInfo import PatternInfo from .pattern_detectors.pipeline_detector import PipelineStage -from .variable import Variable from .pattern_detectors.task_parallelism.classes import TPIType +from .variable import Variable -def filter_members(d: dict) -> dict: +def filter_members(d: Dict[Any, Any]) -> Dict[Any, Any]: """Removes private and protected members (which starts with '_') :param d: member dictionary diff --git a/discopop_explorer/parser.py b/discopop_explorer/parser.py index 1bf4a45ad..802229a8f 100644 --- a/discopop_explorer/parser.py +++ b/discopop_explorer/parser.py @@ -6,13 +6,13 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from dataclasses import dataclass -import re import os -from typing import Any, List, Tuple +import re import warnings from collections import defaultdict +from dataclasses import dataclass from os.path import abspath, dirname +from typing import Any, List, Tuple from lxml import objectify # type:ignore @@ -116,8 +116,7 @@ def __parse_dep_file(dep_fd, output_path: str) -> Tuple[List[DependenceItem], Li static_dependency_lines = [] if not os.path.exists(os.path.join(output_path, "static_dependencies.txt")): warnings.warn( - "Static dependencies could not be found under: " - + os.path.join(output_path, "static_dependencies.txt") + "Static dependencies could not be found under: " + os.path.join(output_path, "static_dependencies.txt") ) # todo warnings.warn( @@ -165,9 +164,7 @@ def __parse_dep_file(dep_fd, output_path: str) -> Tuple[List[DependenceItem], Li else: # compatibility with results created without alias analysis var_name = var_str - dependencies_list.append( - DependenceItem(sink, source_fields[0], type, var_name, aa_var_name) - ) + dependencies_list.append(DependenceItem(sink, source_fields[0], type, var_name, aa_var_name)) return dependencies_list, loop_data_list @@ -210,9 +207,7 @@ def parse_inputs(cu_file, dependencies, reduction_file, file_mapping): def is_reduction(reduction_line, fmap_lines, file_mapping): - rex = re.compile( - "FileID : ([0-9]*) Loop Line Number : [0-9]* Reduction Line Number : ([0-9]*) " - ) + rex = re.compile("FileID : ([0-9]*) Loop Line Number : [0-9]* Reduction Line Number : ([0-9]*) ") if not rex: return False res = rex.search(reduction_line) diff --git a/discopop_explorer/pattern_detection.py b/discopop_explorer/pattern_detection.py index bfc01445e..78dc5adf7 100644 --- a/discopop_explorer/pattern_detection.py +++ b/discopop_explorer/pattern_detection.py @@ -5,10 +5,13 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -import os import sys -from typing import Dict, Union +from alive_progress import alive_bar # type: ignore + +from discopop_explorer.pattern_detectors.task_parallelism.task_parallelism_detector import ( + build_preprocessed_graph_and_run_detection as detect_tp, +) from discopop_library.discopop_optimizer.OptimizationGraph import OptimizationGraph from discopop_library.discopop_optimizer.Variables.Experiment import Experiment from discopop_library.discopop_optimizer.classes.system.System import System @@ -20,13 +23,9 @@ from .PETGraphX import DummyNode, LoopNode, PETGraphX, EdgeType from .pattern_detectors.do_all_detector import run_detection as detect_do_all from .pattern_detectors.geometric_decomposition_detector import run_detection as detect_gd -from .pattern_detectors.simple_gpu_patterns.gpu_pattern_detector import run_detection as detect_gpu from .pattern_detectors.pipeline_detector import run_detection as detect_pipeline from .pattern_detectors.reduction_detector import run_detection as detect_reduction -from discopop_explorer.pattern_detectors.task_parallelism.task_parallelism_detector import ( - build_preprocessed_graph_and_run_detection as detect_tp, -) -from alive_progress import alive_bar # type: ignore +from .pattern_detectors.simple_gpu_patterns.gpu_pattern_detector import run_detection as detect_gpu class PatternDetectorX(object): @@ -142,17 +141,11 @@ def __identify_scheduling_clauses( arguments_1, ) arguments_2 = {"--exhaustive-search": False, "--headless-mode": True} - optimization_graph = OptimizationGraph( - project_folder_path, experiment, arguments_2, None, False - ) + optimization_graph = OptimizationGraph(project_folder_path, experiment, arguments_2, None, False) for do_all_suggestion in res.do_all: - for node_id in get_nodes_from_cu_id( - experiment.optimization_graph, do_all_suggestion.node_id - ): - workload_delta, min_workload, max_workload = get_workload_delta_for_cu_node( - experiment, node_id - ) + for node_id in get_nodes_from_cu_id(experiment.optimization_graph, do_all_suggestion.node_id): + workload_delta, min_workload, max_workload = get_workload_delta_for_cu_node(experiment, node_id) print( "DOALL @ ", do_all_suggestion.node_id, diff --git a/discopop_explorer/pattern_detectors/PatternInfo.py b/discopop_explorer/pattern_detectors/PatternInfo.py index bcbd43bc8..126c7537a 100644 --- a/discopop_explorer/pattern_detectors/PatternInfo.py +++ b/discopop_explorer/pattern_detectors/PatternInfo.py @@ -8,8 +8,8 @@ import json from typing import Optional -from ..utils import calculate_workload, calculate_per_iteration_workload_of_loop from ..PETGraphX import LoopNode, Node, NodeID, LineID, PETGraphX +from ..utils import calculate_workload, calculate_per_iteration_workload_of_loop class PatternInfo(object): @@ -25,9 +25,7 @@ class PatternInfo(object): instructions_count: Optional[int] workload: Optional[int] per_iteration_workload: Optional[int] - dp_optimizer_device_id: Optional[ - int - ] = None # used by discopop_optimizer. unused by discopop_explorer. + dp_optimizer_device_id: Optional[int] = None # used by discopop_optimizer. unused by discopop_explorer. def __init__(self, node: Node): """ @@ -43,15 +41,9 @@ def __init__(self, node: Node): else -1 ) self.iterations_count = ( - node.loop_data.total_iteration_count - if (isinstance(node, LoopNode) and node.loop_data is not None) - else -1 - ) - self.entries = ( - node.loop_data.entry_count - if (isinstance(node, LoopNode) and node.loop_data is not None) - else -1 + node.loop_data.total_iteration_count if (isinstance(node, LoopNode) and node.loop_data is not None) else -1 ) + self.entries = node.loop_data.entry_count if (isinstance(node, LoopNode) and node.loop_data is not None) else -1 # TODO self.instructions_count = total_instructions_count(pet, node) self.workload = None @@ -65,9 +57,7 @@ def to_json(self): if key.startswith("_"): del dic[key] - return json.dumps( - dic, indent=2, default=lambda o: o.toJSON() - ) # , default=lambda o: "") + return json.dumps(dic, indent=2, default=lambda o: o.toJSON()) # , default=lambda o: "") def get_workload(self, pet: PETGraphX) -> int: """returns the workload of self._node""" diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/CombinedGPURegions.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/CombinedGPURegions.py index eefec58ac..879b13b92 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/CombinedGPURegions.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/CombinedGPURegions.py @@ -5,11 +5,14 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -import os.path -from typing import List, Tuple, Dict, Set, Type, Optional +import sys +from typing import List, Tuple, Dict, Set from discopop_explorer.PETGraphX import EdgeType, CUNode, PETGraphX, NodeID, MemoryRegion from discopop_explorer.pattern_detectors.PatternInfo import PatternInfo +from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Aliases import ( + VarName, +) from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Dependency import Dependency from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Enums import ( ExitPointPositioning, @@ -18,10 +21,6 @@ EntryPointType, UpdateType, ) -from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Aliases import ( - VarName, -) -from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Update import Update from discopop_explorer.pattern_detectors.combined_gpu_patterns.prepare_metadata import ( get_dependencies_as_metadata, ) @@ -35,21 +34,21 @@ add_memory_regions_to_device_liveness, propagate_memory_regions, convert_liveness, - extend_data_lifespan, calculate_host_liveness, ) from discopop_explorer.pattern_detectors.combined_gpu_patterns.step_3 import ( initialize_writes, - propagate_writes, cleanup_writes, group_writes_by_cu, ) from discopop_explorer.pattern_detectors.combined_gpu_patterns.step_4 import ( - identify_updates, create_circle_free_function_graphs, add_accesses_from_called_functions, identify_updates_in_unrolled_function_graphs, ) +from discopop_explorer.pattern_detectors.combined_gpu_patterns.step_5 import ( + propagate_variable_name_associations, +) from discopop_explorer.pattern_detectors.combined_gpu_patterns.step_6 import ( convert_updates_to_entry_and_exit_points, identify_end_of_life_points, @@ -58,17 +57,8 @@ remove_duplicates, join_elements, ) -from discopop_explorer.pattern_detectors.combined_gpu_patterns.utilities import ( - prepare_liveness_metadata, -) -from discopop_explorer.pattern_detectors.combined_gpu_patterns.step_5 import ( - propagate_variable_name_associations, -) - from discopop_explorer.pattern_detectors.simple_gpu_patterns.GPURegions import GPURegionInfo -import sys - class CombinedGPURegion(PatternInfo): contained_regions: List[GPURegionInfo] @@ -93,9 +83,7 @@ class CombinedGPURegion(PatternInfo): meta_host_liveness: Dict[MemoryRegion, List[str]] project_folder_path: str - def __init__( - self, pet: PETGraphX, contained_regions: List[GPURegionInfo], project_folder_path: str - ): + def __init__(self, pet: PETGraphX, contained_regions: List[GPURegionInfo], project_folder_path: str): self.project_folder_path = project_folder_path node_id = sorted([region.node_id for region in contained_regions])[0] device_cu_ids: List[NodeID] = [] @@ -137,9 +125,7 @@ def __init__( # get memory region and variable associations for each CU cu_and_variable_to_memory_regions: Dict[ NodeID, Dict[VarName, Set[MemoryRegion]] - ] = get_cu_and_varname_to_memory_regions( - self.contained_regions, pet, written_memory_regions_by_cu - ) + ] = get_cu_and_varname_to_memory_regions(self.contained_regions, pet, written_memory_regions_by_cu) print("WRITTEN MEMORY REGIONS BY CU: ", file=sys.stderr) print(written_memory_regions_by_cu, file=sys.stderr) @@ -166,16 +152,12 @@ def __init__( # extend device liveness with memory regions device_liveness_plus_memory_regions: Dict[ VarName, List[Tuple[NodeID, Set[MemoryRegion]]] - ] = add_memory_regions_to_device_liveness( - live_device_variables, cu_and_variable_to_memory_regions - ) + ] = add_memory_regions_to_device_liveness(live_device_variables, cu_and_variable_to_memory_regions) # ### STEP 2.2: CALCULATE LIVE DATA BY PROPAGATING MEMORY REGIONS AND EXTENDING LIFESPAN # propagate memory regions - device_liveness_plus_memory_regions = propagate_memory_regions( - device_liveness_plus_memory_regions - ) + device_liveness_plus_memory_regions = propagate_memory_regions(device_liveness_plus_memory_regions) print("PROPAGATED DEVICE VARIABLES + MEMORY:", file=sys.stderr) print(device_liveness_plus_memory_regions, file=sys.stderr) @@ -188,9 +170,7 @@ def __init__( print(file=sys.stderr) # extend data liveness - extended_memory_region_liveness = ( - memory_region_liveness # extend_data_lifespan(pet, memory_region_liveness) - ) + extended_memory_region_liveness = memory_region_liveness # extend_data_lifespan(pet, memory_region_liveness) print("EXTENDED DEVICE MEMORY REGION LIVENESS:", file=sys.stderr) print(extended_memory_region_liveness, file=sys.stderr) @@ -202,19 +182,17 @@ def __init__( print(host_liveness, file=sys.stderr) print(file=sys.stderr) host_memory_region_liveness = convert_liveness(host_liveness) - extended_host_memory_region_liveness = host_memory_region_liveness # extend_data_lifespan( pet, host_memory_region_liveness ) + extended_host_memory_region_liveness = ( + host_memory_region_liveness # extend_data_lifespan( pet, host_memory_region_liveness ) + ) print("EXTENDED HOST MEMORY REGION LIVENESS:", file=sys.stderr) print(extended_host_memory_region_liveness, file=sys.stderr) print(file=sys.stderr) # ### STEP 3: MARK WRITTEN VARIABLES # initialize writes - device_writes = initialize_writes( - extended_memory_region_liveness, written_memory_regions_by_cu - ) - host_writes = initialize_writes( - extended_host_memory_region_liveness, written_memory_regions_by_cu - ) + device_writes = initialize_writes(extended_memory_region_liveness, written_memory_regions_by_cu) + host_writes = initialize_writes(extended_host_memory_region_liveness, written_memory_regions_by_cu) # propagate writes to parents, successors and the children of successors propagated_device_writes = device_writes # propagate_writes(self, pet, device_writes) @@ -283,9 +261,7 @@ def __init__( # determine variable name for memory regions in update instructions for update in issued_updates: - update.convert_memory_regions_to_variable_names( - pet, memory_regions_to_functions_and_variables - ) + update.convert_memory_regions_to_variable_names(pet, memory_regions_to_functions_and_variables) # ### POTENTIAL STEP 6: CONVERT MEMORY REGIONS TO STRUCTURE INDICES @@ -352,13 +328,11 @@ def __init__( # ) # prepare update instructions self.update_instructions = [ - update.get_as_metadata_using_variable_names(pet, self.project_folder_path) - for update in updates + update.get_as_metadata_using_variable_names(pet, self.project_folder_path) for update in updates ] # prepare entry points self.data_region_entry_points = [ - entry_point.get_as_metadata(pet, self.project_folder_path) - for entry_point in entry_points + entry_point.get_as_metadata(pet, self.project_folder_path) for entry_point in entry_points ] # prepare exit points @@ -375,9 +349,7 @@ def __init__( for exit_point in exit_points: all_dependencies.update(exit_point.dependencies) - self.data_region_depend_in, self.data_region_depend_out = get_dependencies_as_metadata( - pet, all_dependencies - ) + self.data_region_depend_in, self.data_region_depend_out = get_dependencies_as_metadata(pet, all_dependencies) def __str__(self): raise NotImplementedError() # used to identify necessity to call to_string() instead @@ -410,9 +382,9 @@ def find_combined_gpu_regions( combined_gpu_regions.append(CombinedGPURegion(pet, [gpu_region], project_folder_path)) # determine relations between single-element regions - combinable_pairs: List[ - Tuple[CombinedGPURegion, CombinedGPURegion] - ] = find_all_pairwise_gpu_region_combinations(combined_gpu_regions, pet) + combinable_pairs: List[Tuple[CombinedGPURegion, CombinedGPURegion]] = find_all_pairwise_gpu_region_combinations( + combined_gpu_regions, pet + ) intra_function_combinations = find_combinations_within_function_body(pet, combinable_pairs) @@ -424,9 +396,7 @@ def find_combined_gpu_regions( combined_gpu_regions.remove(combinable_1) if combinable_2 in combined_gpu_regions: combined_gpu_regions.remove(combinable_2) - combined_gpu_regions.append( - combine_regions(pet, combinable_1, combinable_2, project_folder_path) - ) + combined_gpu_regions.append(combine_regions(pet, combinable_1, combinable_2, project_folder_path)) # # combine all known regions # combined_region: Optional[CombinedGPURegion] = None @@ -469,12 +439,8 @@ def find_combinations_within_function_body( continue # check reachability in both directions via successor edges # consider children, as loop nodes do not have successors on their own - for region_1_child in pet.direct_children( - pet.node_at(region_1.contained_regions[0].node_id) - ): - for region_2_child in pet.direct_children( - pet.node_at(region_2.contained_regions[0].node_id) - ): + for region_1_child in pet.direct_children(pet.node_at(region_1.contained_regions[0].node_id)): + for region_2_child in pet.direct_children(pet.node_at(region_2.contained_regions[0].node_id)): if region_1_child == region_2_child: continue if pet.check_reachability(region_2_child, region_1_child, [EdgeType.SUCCESSOR]): @@ -496,16 +462,12 @@ def find_true_successor_combinations( # a true successor relation exists, if every successor path outgoing from any child of region_1 arrives at region_2 for region_1, region_2 in intra_function_combinations: true_successors = True - queue: List[CUNode] = pet.direct_children( - pet.node_at(region_1.contained_regions[0].node_id) - ) + queue: List[CUNode] = pet.direct_children(pet.node_at(region_1.contained_regions[0].node_id)) visited: List[CUNode] = [] while queue: current_node: CUNode = queue.pop() visited.append(current_node) - if current_node in pet.direct_children( - pet.node_at(region_2.contained_regions[0].node_id) - ): + if current_node in pet.direct_children(pet.node_at(region_2.contained_regions[0].node_id)): # reached region_2 continue # region_2 not reached diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/EntryPoint.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/EntryPoint.py index 8fbe308e1..76724ca3d 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/EntryPoint.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/EntryPoint.py @@ -6,7 +6,6 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. import os.path -import sys from typing import Set, Tuple, List from discopop_explorer.PETGraphX import PETGraphX, NodeID, MemoryRegion @@ -147,9 +146,7 @@ def get_as_metadata(self, pet: PETGraphX, project_folder_path: str): for vn, t, s in var_names_types_and_sizes ] else: - modified_var_names = [ - (vn + "[:]" if "**" in t else vn) for vn, t, s in var_names_types_and_sizes - ] + modified_var_names = [(vn + "[:]" if "**" in t else vn) for vn, t, s in var_names_types_and_sizes] return [ ",".join(modified_var_names), diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/ExitPoint.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/ExitPoint.py index 60451d95d..423b520f9 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/ExitPoint.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/ExitPoint.py @@ -144,9 +144,7 @@ def get_as_metadata(self, pet: PETGraphX, project_folder_path: str): for vn, t, s in var_names_types_and_sizes ] else: - modified_var_names = [ - (vn + "[:]" if "**" in t else vn) for vn, t, s in var_names_types_and_sizes - ] + modified_var_names = [(vn + "[:]" if "**" in t else vn) for vn, t, s in var_names_types_and_sizes] return [ ",".join(modified_var_names), diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/Update.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/Update.py index a56e6b854..404866454 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/Update.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/Update.py @@ -168,9 +168,7 @@ def get_as_metadata_using_variable_names(self, pet: PETGraphX, project_folder_pa for var_name in self.variable_names: var_obj = pet.get_variable(self.sink_cu_id, var_name) source_cu_id = ( - self.asynchronous_source_cu_id - if self.asynchronous_possible - else self.synchronous_source_cu_id + self.asynchronous_source_cu_id if self.asynchronous_possible else self.synchronous_source_cu_id ) if var_obj is None: var_obj = pet.get_variable(cast(NodeID, source_cu_id), var_name) @@ -195,9 +193,7 @@ def get_as_metadata_using_variable_names(self, pet: PETGraphX, project_folder_pa for vn, t, s in var_names_types_and_sizes ] else: - modified_var_names = [ - (vn + "[:]" if "**" in t else vn) for vn, t, s in var_names_types_and_sizes - ] + modified_var_names = [(vn + "[:]" if "**" in t else vn) for vn, t, s in var_names_types_and_sizes] # determine update position in code # todo consider asynchronous updates @@ -224,17 +220,13 @@ def get_as_metadata_using_variable_names(self, pet: PETGraphX, project_folder_pa update_position, ] - def get_as_metadata_using_variable_names_and_memory_regions( - self, pet: PETGraphX, project_folder_path: str - ): + def get_as_metadata_using_variable_names_and_memory_regions(self, pet: PETGraphX, project_folder_path: str): # get type of mapped variables var_names_types_and_sizes: List[Tuple[VarName, str, int]] = [] for var_name in self.variable_names: var_obj = pet.get_variable(self.sink_cu_id, var_name) source_cu_id = ( - self.asynchronous_source_cu_id - if self.asynchronous_possible - else self.synchronous_source_cu_id + self.asynchronous_source_cu_id if self.asynchronous_possible else self.synchronous_source_cu_id ) if var_obj is None: var_obj = pet.get_variable(cast(NodeID, source_cu_id), var_name) @@ -259,9 +251,7 @@ def get_as_metadata_using_variable_names_and_memory_regions( for vn, t, s in var_names_types_and_sizes ] else: - modified_var_names = [ - (vn + "[:]" if "**" in t else vn) for vn, t, s in var_names_types_and_sizes - ] + modified_var_names = [(vn + "[:]" if "**" in t else vn) for vn, t, s in var_names_types_and_sizes] return [ self.synchronous_source_cu_id, @@ -281,21 +271,13 @@ def convert_memory_regions_to_variable_names( for mem_reg in self.memory_regions: if parent_function_id in memory_regions_to_functions_and_variables[mem_reg]: + self.variable_names.update(memory_regions_to_functions_and_variables[mem_reg][parent_function_id]) + elif self.synchronous_source_cu_id in memory_regions_to_functions_and_variables[mem_reg]: self.variable_names.update( - memory_regions_to_functions_and_variables[mem_reg][parent_function_id] - ) - elif ( - self.synchronous_source_cu_id in memory_regions_to_functions_and_variables[mem_reg] - ): - self.variable_names.update( - memory_regions_to_functions_and_variables[mem_reg][ - self.synchronous_source_cu_id - ] + memory_regions_to_functions_and_variables[mem_reg][self.synchronous_source_cu_id] ) elif self.sink_cu_id in memory_regions_to_functions_and_variables[mem_reg]: - self.variable_names.update( - memory_regions_to_functions_and_variables[mem_reg][self.sink_cu_id] - ) + self.variable_names.update(memory_regions_to_functions_and_variables[mem_reg][self.sink_cu_id]) else: self.variable_names.add(VarName("UNDETERMINED(" + mem_reg + ")")) diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/prepare_metadata.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/prepare_metadata.py index 6b95d9b95..fc957b89a 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/prepare_metadata.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/prepare_metadata.py @@ -6,16 +6,11 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -import sys from typing import Set, List, Tuple from discopop_explorer.PETGraphX import PETGraphX, NodeID from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Aliases import VarName from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Dependency import Dependency -from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Enums import ( - EntryPointPositioning, - ExitPointPositioning, -) def get_dependencies_as_metadata( diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_1.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_1.py index ffe960a81..8e3292b8f 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_1.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_1.py @@ -48,16 +48,12 @@ def get_written_and_read_memory_regions_by_cu( read_memory_regions = [ MemoryRegion(cast(str, d.memory_region)) for s, t, d in in_dep_edges - if (d.dtype == DepType.WAR) - and d.memory_region is not None - and len(d.memory_region) != 0 + if (d.dtype == DepType.WAR) and d.memory_region is not None and len(d.memory_region) != 0 ] read_memory_regions += [ MemoryRegion(cast(str, d.memory_region)) for s, t, d in out_dep_edges - if (d.dtype == DepType.RAW) - and d.memory_region is not None - and len(d.memory_region) != 0 + if (d.dtype == DepType.RAW) and d.memory_region is not None and len(d.memory_region) != 0 ] if cu_id not in written_memory_regions_by_cu_id: diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_2.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_2.py index b7c3a280a..551df2a09 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_2.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_2.py @@ -30,9 +30,7 @@ ) -def populate_live_data( - comb_gpu_reg, pet: PETGraphX, ignore_update_instructions=False -) -> Dict[VarName, List[NodeID]]: +def populate_live_data(comb_gpu_reg, pet: PETGraphX, ignore_update_instructions=False) -> Dict[VarName, List[NodeID]]: """calculate List of cu-id's in the combined region for each variable in which the respective data is live. The gathered information is used for the optimization / creation of data mapping instructions afterwards. """ @@ -168,10 +166,7 @@ def extend_data_lifespan( print("\tparent functions: ", len(cu_ids_by_parent_functions), file=sys.stderr) print( "\t", - [ - (key.name, len(cu_ids_by_parent_functions[key])) - for key in cu_ids_by_parent_functions - ], + [(key.name, len(cu_ids_by_parent_functions[key])) for key in cu_ids_by_parent_functions], file=sys.stderr, ) @@ -191,10 +186,7 @@ def extend_data_lifespan( continue if cu_id in parent_function.reachability_pairs: - reachable = ( - potential_successor_cu_id - in parent_function.reachability_pairs[cu_id] - ) + reachable = potential_successor_cu_id in parent_function.reachability_pairs[cu_id] else: reachable = False @@ -231,9 +223,7 @@ def extend_data_lifespan( # if path_node is located within a loop, add the other loop cus to the path as well to_be_added: List[CUNode] = [] for path_node in path_nodes: - parent_node = [ - pet.node_at(s) for s, t, d in pet.in_edges(path_node.id, EdgeType.CHILD) - ][0] + parent_node = [pet.node_at(s) for s, t, d in pet.in_edges(path_node.id, EdgeType.CHILD)][0] if parent_node.type == NodeType.LOOP: for _, loop_cu_id, _ in pet.out_edges(parent_node.id, EdgeType.CHILD): loop_cu = cast(CUNode, pet.node_at(loop_cu_id)) @@ -249,11 +239,7 @@ def extend_data_lifespan( subtree_without_called_functions = [ cu for cu in pet.direct_children(path_node) - if cu - not in [ - pet.node_at(t) - for s, t, d in pet.out_edges(path_node.id, EdgeType.CALLSNODE) - ] + if cu not in [pet.node_at(t) for s, t, d in pet.out_edges(path_node.id, EdgeType.CALLSNODE)] ] # add path_node itself to the subtree subtree_without_called_functions.append(path_node) @@ -294,13 +280,9 @@ def calculate_host_liveness( all_function_cu_ids: Set[NodeID] = set() for region in comb_gpu_reg.contained_regions: parent_function = pet.get_parent_function(pet.node_at(region.node_id)) - all_function_cu_ids.update( - get_function_body_cus_without_called_functions(pet, parent_function) - ) + all_function_cu_ids.update(get_function_body_cus_without_called_functions(pet, parent_function)) - all_function_host_cu_ids = [ - cu_id for cu_id in all_function_cu_ids if cu_id not in comb_gpu_reg.device_cu_ids - ] + all_function_host_cu_ids = [cu_id for cu_id in all_function_cu_ids if cu_id not in comb_gpu_reg.device_cu_ids] for cu_id in all_function_host_cu_ids: shared_variables: Set[VarName] = set() diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_3.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_3.py index f15bc2b06..fd7f11dd9 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_3.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_3.py @@ -6,8 +6,7 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -import sys -from typing import Dict, List, Tuple, Set, Optional, cast +from typing import Dict, List, Tuple, Set, Optional from discopop_explorer.PETGraphX import PETGraphX, EdgeType, NodeID, CUNode, MemoryRegion @@ -99,9 +98,7 @@ def propagate_writes( modification_found = True # propagated to children of successor # propagate to called functions of successors - for _, called_node_id, _ in pet.out_edges( - successor_id, EdgeType.CALLSNODE - ): + for _, called_node_id, _ in pet.out_edges(successor_id, EdgeType.CALLSNODE): if (called_node_id, write_identifier_1) not in writes[mem_reg]: writes[mem_reg].add((called_node_id, write_identifier_1)) modification_found = True @@ -124,9 +121,7 @@ def propagate_writes( if pet_node.return_instructions_count > 0: # propagate write to calling cus parent_function = pet.get_parent_function(pet_node) - callees = [ - s for s, t, d in pet.in_edges(parent_function.id, EdgeType.CALLSNODE) - ] + callees = [s for s, t, d in pet.in_edges(parent_function.id, EdgeType.CALLSNODE)] for callee_id in callees: if (callee_id, write_identifier) not in writes[mem_reg]: writes[mem_reg].add((callee_id, write_identifier)) diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_4.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_4.py index 3c61d088e..8f22e28c8 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_4.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_4.py @@ -7,7 +7,8 @@ # directory for details. import copy -from typing import Dict, Set, Tuple, Optional, List, cast, Any +import sys +from typing import Dict, Set, Tuple, Optional, List, cast import networkx as nx # type: ignore from networkx import NetworkXNoCycle, MultiDiGraph @@ -22,8 +23,6 @@ Dependency, FunctionNode, ) -import sys - from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Enums import UpdateType from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Update import Update @@ -31,9 +30,7 @@ class Context(object): cu_id: NodeID device_id: int - seen_writes_by_device: Dict[ - int, Dict[MemoryRegion, Set[Tuple[Optional[int], Optional[NodeID]]]] - ] + seen_writes_by_device: Dict[int, Dict[MemoryRegion, Set[Tuple[Optional[int], Optional[NodeID]]]]] def __init__(self, cu_id: NodeID, device_id: int): self.cu_id = cu_id @@ -82,9 +79,7 @@ def update_writes( self.seen_writes_by_device[device_id_to_update][mem_reg] = set() print("==> added ", mem_reg, " to device: ", device_id_to_update, file=sys.stderr) for ident in writes[mem_reg]: - if ident not in [ - t[0] for t in self.seen_writes_by_device[device_id_to_update][mem_reg] - ]: + if ident not in [t[0] for t in self.seen_writes_by_device[device_id_to_update][mem_reg]]: # list of seen writes will be updated updated_memory_regions.add((mem_reg, is_initialization)) print( @@ -99,17 +94,11 @@ def update_writes( print("ADD : ", (ident, origin_cu_id), file=sys.stderr) # todo: do not simply add, but replace current entry with origin_cu_id if it is a successor of the current entry # necessary, since relying on set behavior is not sufficient anymore - if ident not in [ - t[0] for t in self.seen_writes_by_device[device_id_to_update][mem_reg] - ]: - self.seen_writes_by_device[device_id_to_update][mem_reg].add( - (ident, origin_cu_id) - ) + if ident not in [t[0] for t in self.seen_writes_by_device[device_id_to_update][mem_reg]]: + self.seen_writes_by_device[device_id_to_update][mem_reg].add((ident, origin_cu_id)) return updated_memory_regions - def find_required_updates( - self, pet: PETGraphX, new_device_id: int - ) -> Set[Tuple[MemoryRegion, int, int, NodeID]]: + def find_required_updates(self, pet: PETGraphX, new_device_id: int) -> Set[Tuple[MemoryRegion, int, int, NodeID]]: # update required, if seen writes of new device is not a superset of old device id required_updates: Set[Tuple[MemoryRegion, int, int, NodeID]] = set() @@ -127,8 +116,7 @@ def find_required_updates( missing_write_identifiers = [ (ident, origin) for (ident, origin) in self.seen_writes_by_device[device_id_1][mem_reg] - if ident not in [t[0] for t in self.seen_writes_by_device[device_id_2][mem_reg]] - and ident is not None + if ident not in [t[0] for t in self.seen_writes_by_device[device_id_2][mem_reg]] and ident is not None ] if len(missing_write_identifiers) > 0: # get position of the last write @@ -139,9 +127,7 @@ def find_required_updates( if pet.is_predecessor(cast(NodeID, last_write_location), cast(NodeID, origin)): last_write_location = origin - required_updates.add( - (mem_reg, device_id_1, device_id_2, cast(NodeID, last_write_location)) - ) + required_updates.add((mem_reg, device_id_1, device_id_2, cast(NodeID, last_write_location))) return required_updates @@ -159,9 +145,7 @@ def synchronize_states(self, new_device_id: int): print("SYNCHRONIZED: ", mem_reg, (ident, origin), file=sys.stderr) self.seen_writes_by_device[new_device_id][mem_reg].add((ident, origin)) - def request_updates_from_other_devices( - self, pet, new_device_id: int - ) -> Set[Tuple[MemoryRegion, int, int, NodeID]]: + def request_updates_from_other_devices(self, pet, new_device_id: int) -> Set[Tuple[MemoryRegion, int, int, NodeID]]: required_updates: Set[Tuple[MemoryRegion, int, int, NodeID]] = set() to_be_removed = set() for mem_reg in self.seen_writes_by_device[new_device_id]: @@ -175,8 +159,7 @@ def request_updates_from_other_devices( missing_identifiers = [ (ident, origin) for (ident, origin) in self.seen_writes_by_device[other_device_id][mem_reg] - if ident - not in [t[0] for t in self.seen_writes_by_device[new_device_id][mem_reg]] + if ident not in [t[0] for t in self.seen_writes_by_device[new_device_id][mem_reg]] ] if len(missing_identifiers) > 0: # get position of the last write @@ -184,14 +167,10 @@ def request_updates_from_other_devices( for ident, origin in missing_identifiers: if last_write_location is None: last_write_location = origin - if pet.is_predecessor( - cast(NodeID, last_write_location), cast(NodeID, origin) - ): + if pet.is_predecessor(cast(NodeID, last_write_location), cast(NodeID, origin)): last_write_location = origin - required_updates.add( - (mem_reg, other_device_id, new_device_id, cast(NodeID, last_write_location)) - ) + required_updates.add((mem_reg, other_device_id, new_device_id, cast(NodeID, last_write_location))) # remove none identifier for ident, origin in self.seen_writes_by_device[new_device_id][mem_reg]: @@ -237,9 +216,7 @@ def update_to( writes_for_update = dict() if next_cu_id in writes_by_device[new_device_id]: writes_for_update = writes_by_device[new_device_id][next_cu_id] - self_updated_memory_regions = self.update_writes( - new_device_id, writes_for_update, next_cu_id - ) + self_updated_memory_regions = self.update_writes(new_device_id, writes_for_update, next_cu_id) print("UPDATED MEMORY REGIONS: ", self_updated_memory_regions, file=sys.stderr) # identify required updates @@ -326,8 +303,7 @@ def check_validity_of_potential_merge_node(node_id: NodeID): return False if ( "return" in str(potential_merge_node.basic_block_id) - and potential_merge_node.end_position() - == pet.get_parent_function(potential_merge_node).end_position() + and potential_merge_node.end_position() == pet.get_parent_function(potential_merge_node).end_position() ): # do not consider return BB as merge node return False @@ -405,9 +381,7 @@ def identify_updates( print("IDENTIFY UPDATES FOR: ", pet.node_at(parent_function_id).name, file=sys.stderr) # determine entry points entry_points: List[NodeID] = [] - for function_child_id in [ - t for s, t, d in pet.out_edges(parent_function_id, EdgeType.CHILD) - ]: + for function_child_id in [t for s, t, d in pet.out_edges(parent_function_id, EdgeType.CHILD)]: in_successor_edges = pet.in_edges(function_child_id, EdgeType.SUCCESSOR) if len(in_successor_edges) == 0 and pet.node_at(function_child_id).type == NodeType.CU: entry_points.append(function_child_id) @@ -455,10 +429,7 @@ def get_update_type(from_device_id: int, to_device_id: int) -> UpdateType: elif from_device_id != 0 and to_device_id == 0: return UpdateType.FROM_DEVICE raise ValueError( - "Unsupported update type requested for device IDS: " - + str(from_device_id) - + " -> " - + str(to_device_id) + "Unsupported update type requested for device IDS: " + str(from_device_id) + " -> " + str(to_device_id) ) @@ -607,15 +578,11 @@ def create_circle_free_function_graphs(pet: PETGraphX, add_dummy_node=True): entry_nodes.add(node_id) # ==> Identify cycle exits - for potential_exit_node in set( - [s for s, t, d in cycle_edges] + [t for s, t, d in cycle_edges] - ): + for potential_exit_node in set([s for s, t, d in cycle_edges] + [t for s, t, d in cycle_edges]): print("cyc: ", cycle_nodes) print("exit: ", potential_exit_node) potential_cycle_successor_nodes = [ - t - for s, t, d in pet.out_edges(potential_exit_node, EdgeType.SUCCESSOR) - if t not in cycle_nodes + t for s, t, d in pet.out_edges(potential_exit_node, EdgeType.SUCCESSOR) if t not in cycle_nodes ] print("POT: ", potential_cycle_successor_nodes) @@ -628,9 +595,7 @@ def create_circle_free_function_graphs(pet: PETGraphX, add_dummy_node=True): filtered_pcsn = [] for pcsn in potential_cycle_successor_nodes: pcsn_parents = [ - s - for s, t, d in pet.in_edges(pcsn, EdgeType.CHILD) - if type(pet.node_at(s)) != FunctionNode + s for s, t, d in pet.in_edges(pcsn, EdgeType.CHILD) if type(pet.node_at(s)) != FunctionNode ] if len([nid for nid in pen_parents if nid in pcsn_parents]) == 0: # no shared parents @@ -677,9 +642,7 @@ def create_circle_free_function_graphs(pet: PETGraphX, add_dummy_node=True): for cycle_edge in cycle_edges: if cycle_edge[1] == potential_exit_node: print("Redirecting: ", cycle_edge, " TO (", cycle_edge[0], ", ", end="") - unrolled_function_graphs[function].remove_edge( - cycle_edge[0], cycle_edge[1], cycle_edge[2] - ) + unrolled_function_graphs[function].remove_edge(cycle_edge[0], cycle_edge[1], cycle_edge[2]) if add_dummy_node: unrolled_function_graphs[function].add_edge( cycle_edge[0], @@ -697,9 +660,7 @@ def create_circle_free_function_graphs(pet: PETGraphX, add_dummy_node=True): ")", ) if add_dummy_node: - unrolled_function_graphs[function].add_edge( - "dummy:" + potential_exit_node, cycle_successor - ) + unrolled_function_graphs[function].add_edge("dummy:" + potential_exit_node, cycle_successor) else: for buffered_node_id in buffer: unrolled_function_graphs[function].add_edge( @@ -757,14 +718,10 @@ def add_accesses_from_called_functions( if mem_reg not in writes_by_device[device_id][calling_cu_id]: writes_by_device[device_id][calling_cu_id][mem_reg] = set() len_pre = len(writes_by_device[device_id][calling_cu_id][mem_reg]) - writes_by_device[device_id][calling_cu_id][mem_reg].update( - memory_accesses[device_id][mem_reg] - ) + writes_by_device[device_id][calling_cu_id][mem_reg].update(memory_accesses[device_id][mem_reg]) len_post = len(writes_by_device[device_id][calling_cu_id][mem_reg]) values_propagated = ( - (values_propagated or True) - if len_pre < len_post - else (values_propagated or False) + (values_propagated or True) if len_pre < len_post else (values_propagated or False) ) print("cycles: ", cycles) return writes_by_device @@ -786,9 +743,7 @@ def identify_updates_in_unrolled_function_graphs( print("IDENTIFY UPDATES FOR: ", pet.node_at(parent_function_id).name, file=sys.stderr) # determine entry points entry_points: List[NodeID] = [] - for function_child_id in [ - t for s, t, d in pet.out_edges(parent_function_id, EdgeType.CHILD) - ]: + for function_child_id in [t for s, t, d in pet.out_edges(parent_function_id, EdgeType.CHILD)]: in_successor_edges = pet.in_edges(function_child_id, EdgeType.SUCCESSOR) if len(in_successor_edges) == 0 and pet.node_at(function_child_id).type == NodeType.CU: entry_points.append(function_child_id) diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_5.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_5.py index bd59a3a58..7f531b2de 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_5.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_5.py @@ -6,7 +6,7 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from typing import Dict, Set, cast +from typing import Dict, Set from discopop_explorer.PETGraphX import PETGraphX, NodeID, MemoryRegion from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Aliases import ( diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_6.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_6.py index ad44ff22e..5bd80370c 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_6.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/step_6.py @@ -7,7 +7,6 @@ # directory for details. import copy import sys -import typing from typing import Set, Tuple, Dict, List, cast, Optional, Union from networkx import MultiDiGraph # type: ignore @@ -24,14 +23,11 @@ from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Aliases import ( VarName, ) -from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Dependency import Dependency from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.EntryPoint import EntryPoint from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Enums import ( UpdateType, EntryPointType, ExitPointType, - EntryPointPositioning, - ExitPointPositioning, ) from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.ExitPoint import ExitPoint from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Update import Update @@ -47,19 +43,13 @@ def convert_updates_to_entry_and_exit_points( updates: Set[Update] = set() for issued_update in issued_updates: - if ( - issued_update.update_type == UpdateType.TO_DEVICE - or issued_update.update_type == UpdateType.ALLOCATE - ): + if issued_update.update_type == UpdateType.TO_DEVICE or issued_update.update_type == UpdateType.ALLOCATE: # check if the memory region was live on the device before the update qualifies_as_entry = True for mem_reg in issued_update.memory_regions: if mem_reg not in memory_region_liveness_by_device[1]: continue - if ( - issued_update.synchronous_source_cu_id - in memory_region_liveness_by_device[1][mem_reg] - ): + if issued_update.synchronous_source_cu_id in memory_region_liveness_by_device[1][mem_reg]: qualifies_as_entry = False break if qualifies_as_entry: @@ -156,9 +146,7 @@ def add_aliases( memory_regions_to_functions_and_variables: Dict[MemoryRegion, Dict[NodeID, Set[VarName]]], ) -> Set[Update]: for update in issued_updates: - source_parent_function_node = pet.get_parent_function( - pet.node_at(update.synchronous_source_cu_id) - ) + source_parent_function_node = pet.get_parent_function(pet.node_at(update.synchronous_source_cu_id)) sink_parent_function_node = pet.get_parent_function(pet.node_at(update.sink_cu_id)) if source_parent_function_node == sink_parent_function_node: # add alias information from function level @@ -167,9 +155,7 @@ def add_aliases( modification_found = False # add missing variable names for mem_reg in update.memory_regions: - alias_var_names = memory_regions_to_functions_and_variables[mem_reg][ - source_parent_function_node.id - ] + alias_var_names = memory_regions_to_functions_and_variables[mem_reg][source_parent_function_node.id] len_pre = len(update.variable_names) update.variable_names.update(alias_var_names) # add variable name to dependency if required @@ -183,10 +169,7 @@ def add_aliases( # add missing memory regions for var_name in update.variable_names: for mem_reg in memory_regions_to_functions_and_variables: - if ( - source_parent_function_node.id - not in memory_regions_to_functions_and_variables[mem_reg] - ): + if source_parent_function_node.id not in memory_regions_to_functions_and_variables[mem_reg]: continue potential_aliases = memory_regions_to_functions_and_variables[mem_reg][ source_parent_function_node.id @@ -236,10 +219,7 @@ def identify_end_of_life_points( # check if mem_reg is live in all successors of all contained cu's for cu_id in memory_region_liveness_by_device[device_id][mem_reg]: for successor_node in pet.direct_successors(pet.node_at(cu_id)): - if ( - successor_node.id - not in memory_region_liveness_by_device[device_id][mem_reg] - ): + if successor_node.id not in memory_region_liveness_by_device[device_id][mem_reg]: # mem_reg is not live anymore. create an EOL point if mem_reg in mem_reg_aliases: eol_points.add( @@ -281,9 +261,7 @@ def identify_end_of_life_points( var_names: Set[VarName] = set() for mem_reg in eol[2]: if parent_function_node_id in memory_regions_to_functions_and_variables[mem_reg]: - var_names.update( - memory_regions_to_functions_and_variables[mem_reg][parent_function_node_id] - ) + var_names.update(memory_regions_to_functions_and_variables[mem_reg][parent_function_node_id]) memory_regions = set(eol[2]) # check if the exited data is required by another function # if so, mark the exit point as ExitPointType.FROM @@ -300,19 +278,14 @@ def identify_end_of_life_points( for s, t, d in pet.in_edges(path_node.id, EdgeType.DATA) if d.dtype == DepType.RAW and d.memory_region in memory_regions - and pet.get_parent_function(pet.node_at(s)) - != pet.get_parent_function(pet.node_at(t)) + and pet.get_parent_function(pet.node_at(s)) != pet.get_parent_function(pet.node_at(t)) ] if len(in_raw_edges_from_outside) > 0: # value is read -> Copy back to the host so the value does not get discarded - eol_exit_points.add( - ExitPoint(pet, var_names, memory_regions, eol[0], eol[1], ExitPointType.FROM_DEVICE) - ) + eol_exit_points.add(ExitPoint(pet, var_names, memory_regions, eol[0], eol[1], ExitPointType.FROM_DEVICE)) else: # otherwise, mark it as ExitPointType.DELETE - eol_exit_points.add( - ExitPoint(pet, var_names, memory_regions, eol[0], eol[1], ExitPointType.DELETE) - ) + eol_exit_points.add(ExitPoint(pet, var_names, memory_regions, eol[0], eol[1], ExitPointType.DELETE)) print("\tDone.", file=sys.stderr) print(file=sys.stderr) @@ -344,9 +317,7 @@ def extend_region_liveness_using_unrolled_functions( current_node_id, current_mem_reg, visited_nodes = queue.pop() visited_nodes.append(current_node_id) - successors = [ - t for s, t, d in unrolled_function_graph.out_edges(current_node_id, data="data") - ] + successors = [t for s, t, d in unrolled_function_graph.out_edges(current_node_id, data="data")] for successor in successors: # if mem_reg is live in successor, create a new, clean queue entry and set all visited nodes to live diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/utilities.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/utilities.py index 57a004668..b5b739013 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/utilities.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/utilities.py @@ -9,7 +9,6 @@ from typing import List, Set, Dict, Tuple, Optional from discopop_explorer.PETGraphX import ( - CUNode, PETGraphX, EdgeType, NodeID, @@ -23,16 +22,12 @@ def get_contained_lines(start_line: str, end_line: str) -> List[str]: file_id = start_line.split(":")[0] if file_id != end_line.split(":")[0]: raise ValueError("File-ids not equal! ", start_line, end_line) - line_numbers: List[int] = list( - range(int(start_line.split(":")[1]), int(end_line.split(":")[1]) + 1) - ) + line_numbers: List[int] = list(range(int(start_line.split(":")[1]), int(end_line.split(":")[1]) + 1)) result = [file_id + ":" + str(num) for num in line_numbers] return result -def get_function_body_cus_without_called_functions( - pet: PETGraphX, function_node: FunctionNode -) -> List[NodeID]: +def get_function_body_cus_without_called_functions(pet: PETGraphX, function_node: FunctionNode) -> List[NodeID]: queue = [t for s, t, d in pet.out_edges(function_node.id, EdgeType.CHILD)] visited: Set[NodeID] = set() while queue: @@ -43,9 +38,7 @@ def get_function_body_cus_without_called_functions( # add children if they do not result from a call children = [t for s, t, d in pet.out_edges(current, EdgeType.CHILD)] called = [t for s, t, d in pet.out_edges(current, EdgeType.CALLSNODE)] - queue += [ - c for c in children if c not in visited and c not in called - ] # todo add check for call + queue += [c for c in children if c not in visited and c not in called] # todo add check for call return list(visited) @@ -69,13 +62,7 @@ def prepare_liveness_metadata( for line in get_contained_lines(cu_node.start_position(), cu_node.end_position()): if len(write_identifiers) > 0: # is written - line = ( - line - + ":" - + "(" - + ",".join([str(ident) for ident in write_identifiers]) - + ")" - ) + line = line + ":" + "(" + ",".join([str(ident) for ident in write_identifiers]) + ")" else: # is read line = line + ":" diff --git a/discopop_explorer/pattern_detectors/do_all_detector.py b/discopop_explorer/pattern_detectors/do_all_detector.py index 0279a3a4f..ca66fa152 100644 --- a/discopop_explorer/pattern_detectors/do_all_detector.py +++ b/discopop_explorer/pattern_detectors/do_all_detector.py @@ -5,8 +5,11 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. +from multiprocessing import Pool from typing import List, Dict, Set, Tuple, cast + from alive_progress import alive_bar # type: ignore + from .PatternInfo import PatternInfo from ..PETGraphX import ( CUNode, @@ -19,9 +22,8 @@ MemoryRegion, DepType, ) -from ..variable import Variable from ..utils import classify_loop_variables -from multiprocessing import Pool +from ..variable import Variable class DoAllInfo(PatternInfo): @@ -77,9 +79,7 @@ def run_detection(pet: PETGraphX) -> List[DoAllInfo]: param_list = [(node) for node in nodes] with Pool(initializer=__initialize_worker, initargs=(pet,)) as pool: - tmp_result = list( - tqdm.tqdm(pool.imap_unordered(__check_node, param_list), total=len(param_list)) - ) + tmp_result = list(tqdm.tqdm(pool.imap_unordered(__check_node, param_list), total=len(param_list))) for local_result in tmp_result: result += local_result print("GLOBAL RES: ", result) @@ -121,10 +121,7 @@ def __detect_do_all(pet: PETGraphX, root_loop: LoopNode) -> bool: :param root: root node :return: true if do-all """ - subnodes = [ - pet.node_at(t) - for s, t, d in pet.out_edges(root_loop.id, [EdgeType.CHILD, EdgeType.CALLSNODE]) - ] + subnodes = [pet.node_at(t) for s, t, d in pet.out_edges(root_loop.id, [EdgeType.CHILD, EdgeType.CALLSNODE])] # get required metadata loop_start_lines: List[LineID] = [] @@ -193,18 +190,10 @@ def __check_loop_dependencies( deps = set() for n in node_1_children_ids + node_2_children_ids: deps.update( - [ - (s, t, d) - for s, t, d in pet.in_edges(n, EdgeType.DATA) - if s in node_1_children_ids + node_2_children_ids - ] + [(s, t, d) for s, t, d in pet.in_edges(n, EdgeType.DATA) if s in node_1_children_ids + node_2_children_ids] ) deps.update( - [ - (s, t, d) - for s, t, d in pet.out_edges(n, EdgeType.DATA) - if t in node_1_children_ids + node_2_children_ids - ] + [(s, t, d) for s, t, d in pet.out_edges(n, EdgeType.DATA) if t in node_1_children_ids + node_2_children_ids] ) # get memory regions which are defined inside the loop @@ -273,10 +262,7 @@ def __old_detect_do_all(pet: PETGraphX, root_loop: CUNode) -> bool: :param root: root node :return: true if do-all """ - subnodes = [ - pet.node_at(t) - for s, t, d in pet.out_edges(root_loop.id, [EdgeType.CHILD, EdgeType.CALLSNODE]) - ] + subnodes = [pet.node_at(t) for s, t, d in pet.out_edges(root_loop.id, [EdgeType.CHILD, EdgeType.CALLSNODE])] # check if all subnodes are parallelizable for node in pet.subtree_of_type(root_loop, CUNode): diff --git a/discopop_explorer/pattern_detectors/geometric_decomposition_detector.py b/discopop_explorer/pattern_detectors/geometric_decomposition_detector.py index 415e23854..809524420 100644 --- a/discopop_explorer/pattern_detectors/geometric_decomposition_detector.py +++ b/discopop_explorer/pattern_detectors/geometric_decomposition_detector.py @@ -10,11 +10,12 @@ import math from typing import Dict, List, Tuple, Optional +from alive_progress import alive_bar # type: ignore + from .PatternInfo import PatternInfo -from ..PETGraphX import FunctionNode, LoopNode, NodeID, PETGraphX, NodeType, Node, EdgeType -from ..utils import classify_task_vars, get_child_loops, contains +from ..PETGraphX import FunctionNode, LoopNode, NodeID, PETGraphX, Node, EdgeType +from ..utils import classify_task_vars, get_child_loops from ..variable import Variable -from alive_progress import alive_bar # type: ignore __loop_iterations: Dict[NodeID, int] = {} @@ -46,10 +47,8 @@ def __init__(self, pet: PETGraphX, node: Node, min_iter: int): self.num_tasks = math.floor(nt) self.pragma = "for (i = 0; i < num-tasks; i++) #pragma omp task" - lp: List = [] - fp, p, s, in_dep, out_dep, in_out_dep, r = classify_task_vars( - pet, node, "GeometricDecomposition", [], [] - ) + lp: List[Variable] = [] + fp, p, s, in_dep, out_dep, in_out_dep, r = classify_task_vars(pet, node, "GeometricDecomposition", [], []) fp.append(Variable("int", "i", "", sizeInByte=4)) self.first_private = fp @@ -98,9 +97,7 @@ def run_detection(pet: PETGraphX) -> List[GDInfo]: param_list = [(node) for node in nodes] with Pool(initializer=__initialize_worker, initargs=(pet,)) as pool: - tmp_result = list( - tqdm.tqdm(pool.imap_unordered(__check_node, param_list), total=len(param_list)) - ) + tmp_result = list(tqdm.tqdm(pool.imap_unordered(__check_node, param_list), total=len(param_list))) for local_result in tmp_result: result += local_result print("GLOBAL RES: ", result) diff --git a/discopop_explorer/pattern_detectors/pipeline_detector.py b/discopop_explorer/pattern_detectors/pipeline_detector.py index 574721d63..7f0937d08 100644 --- a/discopop_explorer/pattern_detectors/pipeline_detector.py +++ b/discopop_explorer/pattern_detectors/pipeline_detector.py @@ -8,7 +8,9 @@ from typing import List, Tuple, Dict, Set + from alive_progress import alive_bar # type: ignore + from .PatternInfo import PatternInfo from ..PETGraphX import ( CUNode, @@ -16,13 +18,12 @@ LoopNode, NodeID, PETGraphX, - NodeType, Node, EdgeType, DepType, Dependency, ) -from ..utils import correlation_coefficient, classify_task_vars, contains +from ..utils import correlation_coefficient, classify_task_vars __pipeline_threshold = 0.9 @@ -33,9 +34,7 @@ def __init__(self, pet: PETGraphX, node: Node, in_dep, out_dep): self.startsAtLine = node.start_position() self.endsAtLine = node.end_position() - fp, p, s, in_deps, out_deps, in_out_deps, r = classify_task_vars( - pet, node, "Pipeline", in_dep, out_dep - ) + fp, p, s, in_deps, out_deps, in_out_deps, r = classify_task_vars(pet, node, "Pipeline", in_dep, out_dep) self.first_private = fp self.private = p @@ -88,11 +87,7 @@ def __init__(self, pet: PETGraphX, node: Node): def __in_dep(self, node: Node): raw: List[Tuple[NodeID, NodeID, Dependency]] = [] for n in self._pet.subtree_of_type(node, CUNode): - raw.extend( - (s, t, d) - for s, t, d in self._pet.out_edges(n.id, EdgeType.DATA) - if d.dtype == DepType.RAW - ) + raw.extend((s, t, d) for s, t, d in self._pet.out_edges(n.id, EdgeType.DATA) if d.dtype == DepType.RAW) nodes_before = [node] for i in range(self._stages.index(node)): @@ -103,11 +98,7 @@ def __in_dep(self, node: Node): def __out_dep(self, node: Node): raw: List[Tuple[NodeID, NodeID, Dependency]] = [] for n in self._pet.subtree_of_type(node, CUNode): - raw.extend( - (s, t, d) - for s, t, d in self._pet.in_edges(n.id, EdgeType.DATA) - if d.dtype == DepType.RAW - ) + raw.extend((s, t, d) for s, t, d in self._pet.in_edges(n.id, EdgeType.DATA) if d.dtype == DepType.RAW) nodes_after = [node] for i in range(self._stages.index(node) + 1, len(self._stages)): @@ -177,9 +168,7 @@ def run_detection(pet: PETGraphX) -> List[PipelineInfo]: param_list = [(node) for node in nodes] with Pool(initializer=__initialize_worker, initargs=(pet,)) as pool: - tmp_result = list( - tqdm.tqdm(pool.imap_unordered(__check_node, param_list), total=len(param_list)) - ) + tmp_result = list(tqdm.tqdm(pool.imap_unordered(__check_node, param_list), total=len(param_list))) for local_result in tmp_result: result += local_result print("GLOBAL RES: ", result) @@ -232,11 +221,7 @@ def __detect_pipeline(pet: PETGraphX, root: Node, children_cache=None, dep_cache graph_vector = [] for i in range(0, len(loop_subnodes) - 1): - graph_vector.append( - 1.0 - if pet.depends_ignore_readonly(loop_subnodes[i + 1], loop_subnodes[i], root) - else 0.0 - ) + graph_vector.append(1.0 if pet.depends_ignore_readonly(loop_subnodes[i + 1], loop_subnodes[i], root) else 0.0) pipeline_vector = [] for i in range(0, len(loop_subnodes) - 1): diff --git a/discopop_explorer/pattern_detectors/reduction_detector.py b/discopop_explorer/pattern_detectors/reduction_detector.py index a351ad0ef..2f07e77cd 100644 --- a/discopop_explorer/pattern_detectors/reduction_detector.py +++ b/discopop_explorer/pattern_detectors/reduction_detector.py @@ -7,7 +7,9 @@ # directory for details. +from multiprocessing import Pool from typing import List, cast + from alive_progress import alive_bar # type: ignore from .PatternInfo import PatternInfo @@ -21,9 +23,8 @@ DepType, EdgeType, ) +from ..utils import is_reduction_var, classify_loop_variables from ..variable import Variable -from ..utils import is_reduction_var, classify_loop_variables, contains -from multiprocessing import Pool class ReductionInfo(PatternInfo): @@ -76,9 +77,7 @@ def run_detection(pet: PETGraphX) -> List[ReductionInfo]: param_list = [(node) for node in nodes] with Pool(initializer=__initialize_worker, initargs=(pet,)) as pool: - tmp_result = list( - tqdm.tqdm(pool.imap_unordered(__check_node, param_list), total=len(param_list)) - ) + tmp_result = list(tqdm.tqdm(pool.imap_unordered(__check_node, param_list), total=len(param_list))) for local_result in tmp_result: result += local_result print("GLOBAL RES: ", result) @@ -122,12 +121,8 @@ def __detect_reduction(pet: PETGraphX, root: LoopNode) -> bool: # get required metadata loop_start_lines: List[LineID] = [] root_children = pet.subtree_of_type(root, (CUNode, LoopNode)) - root_children_cus: List[CUNode] = [ - cast(CUNode, cu) for cu in root_children if cu.type == NodeType.CU - ] - root_children_loops: List[LoopNode] = [ - cast(LoopNode, cu) for cu in root_children if cu.type == NodeType.LOOP - ] + root_children_cus: List[CUNode] = [cast(CUNode, cu) for cu in root_children if cu.type == NodeType.CU] + root_children_loops: List[LoopNode] = [cast(LoopNode, cu) for cu in root_children if cu.type == NodeType.LOOP] for v in root_children_loops: loop_start_lines.append(v.start_position()) reduction_vars = [ @@ -179,12 +174,8 @@ def __check_loop_dependencies( # get dependency edges between children nodes deps = set() for n in loop_children_ids: - deps.update( - [(s, t, d) for s, t, d in pet.in_edges(n, EdgeType.DATA) if s in loop_children_ids] - ) - deps.update( - [(s, t, d) for s, t, d in pet.out_edges(n, EdgeType.DATA) if t in loop_children_ids] - ) + deps.update([(s, t, d) for s, t, d in pet.in_edges(n, EdgeType.DATA) if s in loop_children_ids]) + deps.update([(s, t, d) for s, t, d in pet.out_edges(n, EdgeType.DATA) if t in loop_children_ids]) for source, target, dep in deps: # check if targeted variable is readonly inside loop diff --git a/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPULoop.py b/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPULoop.py index 7258c3122..65c98270e 100644 --- a/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPULoop.py +++ b/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPULoop.py @@ -6,17 +6,12 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. import os -import sys from enum import IntEnum - from typing import List, Set, Optional, Union, Any, Dict, Tuple, cast -from discopop_library.MemoryRegions.utils import get_sizes_of_memory_regions -from .GPUMemory import getCalledFunctions, map_node, map_type_t, assignMapType from discopop_explorer.PETGraphX import ( PETGraphX, CUNode, - NodeType, parse_id, DepType, NodeID, @@ -26,30 +21,28 @@ Node, LoopNode, ) +from discopop_explorer.pattern_detectors.PatternInfo import PatternInfo +from discopop_explorer.utils import ( + __get_dep_of_type as get_dep_of_type, +) from discopop_explorer.utils import ( is_scalar_val, is_loop_index2, classify_loop_variables, get_initialized_memory_regions_in, ) -from discopop_explorer.utils import ( - __get_variables as get_vars, - __get_dep_of_type as get_dep_of_type, -) from discopop_explorer.utils import ( is_written_in_subtree, is_func_arg, is_readonly, is_global, - is_read_in_subtree, is_read_in_right_subtree, is_first_written, is_read_in_subtree, ) -from discopop_explorer.pattern_detectors.PatternInfo import PatternInfo from discopop_explorer.variable import Variable -from discopop_explorer.pattern_detectors.do_all_detector import DoAllInfo - +from discopop_library.MemoryRegions.utils import get_sizes_of_memory_regions +from .GPUMemory import getCalledFunctions, map_node """ def remove_duplicates(my_list: List) -> List: @@ -145,7 +138,7 @@ def omp_construct_dict( line: LineID, clauses: List[str], positioning: OmpConstructPositioning = OmpConstructPositioning.BEFORE_LINE, -) -> dict: +) -> Dict[str, Union[str, LineID, List[str], OmpConstructPositioning]]: """ :param name: @@ -153,7 +146,7 @@ def omp_construct_dict( :param clauses: :return: """ - result: Dict[str, Union[str, int, List[str]]] = dict() + result: Dict[str, Union[str, LineID, List[str], OmpConstructPositioning]] = dict() result["name"] = name result["line"] = line result["clauses"] = clauses @@ -180,7 +173,7 @@ class GPULoopPattern(PatternInfo): parentLoop: str collapse: int scheduling: str - constructs: List[dict] + constructs: List[Dict[Any, Any]] project_folder_path: str def __init__( @@ -214,9 +207,7 @@ def __init__( self.reduction_vars_str: List[str] = [] self.reduction_vars_ids: List[Variable] = [] else: - self.reduction_vars_str: List[str] = [ - cast(str, v.operation) + ":" + v.name for v in reduction_vars - ] + self.reduction_vars_str: List[str] = [cast(str, v.operation) + ":" + v.name for v in reduction_vars] self.reduction_vars_ids: List[Variable] = reduction_vars self.iteration_count = 0 self.nextLoop = None @@ -283,8 +274,10 @@ def toJson(self, pet: PETGraphX, project_folder_path: str) -> str: json_output += "]}" return json_output - def __get_constructs(self, pet: PETGraphX, project_folder_path: str) -> List[dict]: - constructs: List[dict] = [] + def __get_constructs( + self, pet: PETGraphX, project_folder_path: str + ) -> List[Dict[str, Union[str, LineID, List[str], OmpConstructPositioning]]]: + constructs: List[Dict[str, Union[str, LineID, List[str], OmpConstructPositioning]]] = [] # == default construct == clauses: List[str] = [] @@ -400,9 +393,7 @@ def __get_constructs(self, pet: PETGraphX, project_folder_path: str) -> List[dic tmp_start_line = LineID(str(self._node.file_id) + ":" + str(self.startLine)) constructs.append( - omp_construct_dict( - "#pragma omp target teams distribute parallel for", tmp_start_line, clauses - ) + omp_construct_dict("#pragma omp target teams distribute parallel for", tmp_start_line, clauses) ) # == additional constructs == @@ -411,9 +402,7 @@ def __get_constructs(self, pet: PETGraphX, project_folder_path: str) -> List[dic fn_node: FunctionNode = cast(FunctionNode, map_node(pet, node_id)) fn_node_start_line = LineID(str(fn_node.file_id) + ":" + str(fn_node.start_line)) fn_node_end_line = LineID(str(fn_node.file_id) + ":" + str(fn_node.end_line + 1)) - constructs.append( - omp_construct_dict("#pragma omp declare target", fn_node_start_line, []) - ) + constructs.append(omp_construct_dict("#pragma omp declare target", fn_node_start_line, [])) constructs.append( omp_construct_dict( "#pragma omp end declare target", @@ -440,9 +429,7 @@ def __get_constructs(self, pet: PETGraphX, project_folder_path: str) -> List[dic self.declared_global_variables.update(used_global_vars) for global_var in used_global_vars: constructs.append( - omp_construct_dict( - "#pragma omp declare target // " + global_var.name, global_var.defLine, [] - ) + omp_construct_dict("#pragma omp declare target // " + global_var.name, global_var.defLine, []) ) constructs.append( omp_construct_dict( @@ -512,9 +499,7 @@ def classifyLoopVars(self, pet: PETGraphX, loop: LoopNode) -> None: rev_raw = set() dummyFunctions: Set[NodeID] = set() - self.called_functions.update( - getCalledFunctions(pet, loop, self.called_functions, dummyFunctions) - ) + self.called_functions.update(getCalledFunctions(pet, loop, self.called_functions, dummyFunctions)) for sub_node in sub: raw.update(get_dep_of_type(pet, sub_node, DepType.RAW, False)) @@ -575,9 +560,7 @@ def classifyLoopVars(self, pet: PETGraphX, loop: LoopNode) -> None: pass # use known variables to reconstruct the correct variable names from the classified memory regions - left_subtree_without_called_nodes = pet.get_left_right_subtree( - loop, False, ignore_called_nodes=True - ) + left_subtree_without_called_nodes = pet.get_left_right_subtree(loop, False, ignore_called_nodes=True) prior_known_vars = pet.get_variables(left_subtree_without_called_nodes) # get memory regions which are initialized in the loop and treat them like prior known vars wrt. de-aliasing initilized_in_loop = get_initialized_memory_regions_in(pet, sub) @@ -611,11 +594,7 @@ def __apply_dealiasing( tmp_memory_regions = set() for _, mem_regs in input_list: tmp_memory_regions.update(mem_regs) - cleaned = [ - pkv - for pkv in previously_known - if len(previously_known[pkv].intersection(tmp_memory_regions)) - ] + cleaned = [pkv for pkv in previously_known if len(previously_known[pkv].intersection(tmp_memory_regions))] return cleaned def setParentLoop(self, pl: str) -> None: @@ -671,9 +650,7 @@ def setCollapseClause(self, pet: PETGraphX, node_id: NodeID, res): for cn_id in pet.direct_children(n): if cn_id.type == 2: # check for loop node contained in the loop body - if ( - cn_id.end_line <= n.end_line - ): # todo not true if loop bodies are terminated by braces + if cn_id.end_line <= n.end_line: # todo not true if loop bodies are terminated by braces # only consider child as collapsible, if it is a do-all loop if cn_id.id in do_all_loops: # check for perfect nesting of both loops (i.e. no statements inbetween) diff --git a/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPUMemory.py b/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPUMemory.py index 09e1deb3a..96f232302 100644 --- a/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPUMemory.py +++ b/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPUMemory.py @@ -7,20 +7,21 @@ # directory for details. from enum import Enum -from typing import List, Set, cast -from discopop_explorer.variable import Variable +from typing import List, Set, cast, Tuple + from discopop_explorer.PETGraphX import ( PETGraphX, CUNode, - NodeType, DepType, NodeID, Node, DummyNode, FunctionNode, LoopNode, + Dependency, ) from discopop_explorer.utils import is_func_arg, is_global, __get_dep_of_type as get_dep_of_type +from discopop_explorer.variable import Variable def map_node(pet: PETGraphX, nodeID: NodeID) -> Node: @@ -91,12 +92,12 @@ def getCalledFunctions( def getDeps( cuIDs: List[CUNode], pet: PETGraphX, - RAWDepsOn: Set, - WARDepsOn: Set, - WAWDepsOn: Set, - reverseRAWDepsOn: Set, - reverseWARDepsOn: Set, - reverseWAWDepsOn: Set, + RAWDepsOn: Set[Tuple[NodeID, NodeID, Dependency]], + WARDepsOn: Set[Tuple[NodeID, NodeID, Dependency]], + WAWDepsOn: Set[Tuple[NodeID, NodeID, Dependency]], + reverseRAWDepsOn: Set[Tuple[NodeID, NodeID, Dependency]], + reverseWARDepsOn: Set[Tuple[NodeID, NodeID, Dependency]], + reverseWAWDepsOn: Set[Tuple[NodeID, NodeID, Dependency]], ) -> None: """gather all dependencies of the nodes specified in 'cuIDs' ---doesnt work here because addresses--- @@ -127,8 +128,8 @@ def assignMapType( var: Variable, isScalar: bool, pet: PETGraphX, - RAWDepsOn: Set, - reverseRAWDepsOn: Set, + RAWDepsOn: Set[Tuple[NodeID, NodeID, Dependency]], + reverseRAWDepsOn: Set[Tuple[NodeID, NodeID, Dependency]], ) -> map_type_t: """assigns a map-type to the variable 'var' diff --git a/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPURegions.py b/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPURegions.py index a73e9831f..ef5131cd0 100644 --- a/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPURegions.py +++ b/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPURegions.py @@ -7,6 +7,9 @@ # directory for details. from typing import List, Set, Optional, cast, Dict, Tuple + +from alive_progress import alive_bar # type: ignore + from discopop_explorer.PETGraphX import ( PETGraphX, CUNode, @@ -17,12 +20,10 @@ LoopNode, Node, ) +from discopop_explorer.variable import Variable from .GPULoop import GPULoopPattern from .GPUMemory import map_node -from discopop_explorer.utils import is_loop_index2 -from discopop_explorer.variable import Variable from ..PatternInfo import PatternInfo -from alive_progress import alive_bar # type: ignore class GPURegionInfo(PatternInfo): @@ -207,10 +208,7 @@ def identifyGPURegions(self) -> None: with alive_bar(len(self.gpu_loop_patterns)) as progress_bar: for i in range(0, len(self.gpu_loop_patterns)): if self.gpu_loop_patterns[i].nextLoop is not None: - if ( - map_node(self.pet, cast(NodeID, self.gpu_loop_patterns[i].nextLoop)).type - == 2 - ): + if map_node(self.pet, cast(NodeID, self.gpu_loop_patterns[i].nextLoop)).type == 2: if self.reachableCUs( self.gpu_loop_patterns[i].nodeID, cast(NodeID, self.gpu_loop_patterns[i].nextLoop), @@ -239,18 +237,12 @@ def determineDataMapping(self) -> None: region_loop_patterns: List[GPULoopPattern] = [] for loop_id in region: loop_node: LoopNode = cast(LoopNode, self.pet.node_at(loop_id)) - gpu_lp: GPULoopPattern = [ - p for p in self.gpu_loop_patterns if p.parentLoop == loop_id - ][0] + gpu_lp: GPULoopPattern = [p for p in self.gpu_loop_patterns if p.parentLoop == loop_id][0] region_loop_patterns.append(gpu_lp) - region_cus += [ - cu for cu in self.pet.subtree_of_type(loop_node) if cu not in region_cus - ] + region_cus += [cu for cu in self.pet.subtree_of_type(loop_node) if cu not in region_cus] # add loop initialization to region cus (predecessor of first child of loop, if positions are suitable) loop_entry_cu = self.pet.out_edges(loop_id, EdgeType.CHILD)[0][1] - predecessors = [ - s for s, t, d in self.pet.in_edges(loop_entry_cu, EdgeType.SUCCESSOR) - ] + predecessors = [s for s, t, d in self.pet.in_edges(loop_entry_cu, EdgeType.SUCCESSOR)] for predecessor_id in predecessors: predecessor_node = self.pet.node_at(predecessor_id) if ( @@ -302,16 +294,12 @@ def determineDataMapping(self) -> None: # gather consumed, produced, allocated and deleted variables from mapping information map_to_vars: List[str] = [] for loop_pattern in region_loop_patterns: - map_to_vars += [ - v.name for v in loop_pattern.map_type_to + loop_pattern.map_type_tofrom - ] + map_to_vars += [v.name for v in loop_pattern.map_type_to + loop_pattern.map_type_tofrom] map_to_vars = list(set(map_to_vars)) map_from_vars: List[str] = [] for loop_pattern in region_loop_patterns: - map_from_vars += [ - v.name for v in loop_pattern.map_type_from + loop_pattern.map_type_tofrom - ] + map_from_vars += [v.name for v in loop_pattern.map_type_from + loop_pattern.map_type_tofrom] map_from_vars = list(set(map_from_vars)) map_alloc_vars: List[str] = [] @@ -323,9 +311,7 @@ def determineDataMapping(self) -> None: # allocate unknown variables map_alloc_vars += [ - var - for var in map_from_vars - if var not in consumed_vars + map_to_vars + map_alloc_vars + var for var in map_from_vars if var not in consumed_vars + map_to_vars + map_alloc_vars ] map_alloc_vars = list(set(map_alloc_vars)) @@ -333,9 +319,7 @@ def determineDataMapping(self) -> None: self.map_type_from_by_region[tuple(region)] = [ var for var in map_from_vars if var not in map_to_from_vars ] - self.map_type_to_by_region[tuple(region)] = [ - var for var in map_to_vars if var not in map_to_from_vars - ] + self.map_type_to_by_region[tuple(region)] = [var for var in map_to_vars if var not in map_to_from_vars] self.map_type_tofrom_by_region[tuple(region)] = map_to_from_vars self.map_type_alloc_by_region[tuple(region)] = map_alloc_vars self.map_type_delete_by_region[tuple(region)] = [] @@ -364,9 +348,7 @@ def old_mapData(self) -> None: ln = map_node(self.pet, lastNodeID) start = fn.start_line end = ln.end_line - gpuRegionLoop = GPULoopPattern( - self.pet, firstNodeID, start, end, 1000, self.project_folder_path - ) + gpuRegionLoop = GPULoopPattern(self.pet, firstNodeID, start, end, 1000, self.project_folder_path) visitedVars: Set[Variable] = set() while t >= 0: loopIter = self.findGPULoop( @@ -491,9 +473,7 @@ def get_gpu_region_info(self, pet: PETGraphX, project_folder_path: str) -> List[ print("\tget gpu region info...") with alive_bar(len(self.cascadingLoopsInRegions)) as progress_bar: for region in self.cascadingLoopsInRegions: - contained_loop_patterns = [ - pattern for pattern in self.gpu_loop_patterns if pattern.nodeID in region - ] + contained_loop_patterns = [pattern for pattern in self.gpu_loop_patterns if pattern.nodeID in region] # save OpenMP constructs in GPULoops for the exporting to JSON for loop in contained_loop_patterns: loop.save_omp_constructs(pet, project_folder_path) diff --git a/discopop_explorer/pattern_detectors/simple_gpu_patterns/gpu_pattern_detector.py b/discopop_explorer/pattern_detectors/simple_gpu_patterns/gpu_pattern_detector.py index c9b9a52c6..702944bc3 100644 --- a/discopop_explorer/pattern_detectors/simple_gpu_patterns/gpu_pattern_detector.py +++ b/discopop_explorer/pattern_detectors/simple_gpu_patterns/gpu_pattern_detector.py @@ -8,18 +8,16 @@ from typing import List, cast -from discopop_explorer.PETGraphX import NodeType, PETGraphX, LoopNode +from alive_progress import alive_bar # type: ignore + +from discopop_explorer.PETGraphX import PETGraphX, LoopNode from discopop_explorer.pattern_detectors.PatternInfo import PatternInfo -from discopop_explorer.pattern_detectors.combined_gpu_patterns.CombinedGPURegions import ( - find_combined_gpu_regions, -) from discopop_explorer.pattern_detectors.simple_gpu_patterns.GPULoop import GPULoopPattern from discopop_explorer.pattern_detectors.simple_gpu_patterns.GPURegions import ( GPURegions, GPURegionInfo, ) from discopop_explorer.variable import Variable -from alive_progress import alive_bar # type: ignore def run_detection(pet: PETGraphX, res, project_folder_path: str) -> List[PatternInfo]: diff --git a/discopop_explorer/pattern_detectors/task_parallelism/alias_detection.py b/discopop_explorer/pattern_detectors/task_parallelism/alias_detection.py index f320ddd59..203a8348f 100644 --- a/discopop_explorer/pattern_detectors/task_parallelism/alias_detection.py +++ b/discopop_explorer/pattern_detectors/task_parallelism/alias_detection.py @@ -10,13 +10,12 @@ import pathlib import re import subprocess -from typing import Dict, List, Optional, Match, cast +from typing import Dict, List, Optional, Match, cast, Any + from lxml import objectify # type: ignore -def __prune_statement( - stmt_copy: str, statement: str, var_name: str, var_type: str -) -> Optional[List[str]]: +def __prune_statement(stmt_copy: str, statement: str, var_name: str, var_type: str) -> Optional[List[str]]: """splits a statement and performs multiple alias analyses if necessary (more than one '=' contained in the given statement). :param stmt_copy: cleaned copy of the target statement @@ -51,10 +50,7 @@ def __check_obvious_pointer_type(var_name: str, rhs: str) -> bool: :return: True, of None should be returned by __get_alias_statement. False, otherwise. """ # var_name has index access? - if ( - rhs.index(var_name) + len(var_name) < len(rhs) - and rhs[rhs.index(var_name) + len(var_name)] == "[" - ): + if rhs.index(var_name) + len(var_name) < len(rhs) and rhs[rhs.index(var_name) + len(var_name)] == "[": return True # '*' prior to var_name? if rhs.index(var_name) - 1 >= 0 and rhs[rhs.index(var_name) - 1] == "*": @@ -70,8 +66,7 @@ def __check_obvious_pointer_type(var_name: str, rhs: str) -> bool: # '->' after var_name? if ( rhs.index(var_name) + len(var_name) + 2 > len(rhs) - and rhs[rhs.index(var_name) + len(var_name) : rhs.index(var_name) + len(var_name) + 1] - == "->" + and rhs[rhs.index(var_name) + len(var_name) : rhs.index(var_name) + len(var_name) + 1] == "->" ): return True return False @@ -87,17 +82,10 @@ def __check_possible_pointer_type(var_name: str, rhs: str) -> bool: if rhs.index(var_name) - 1 >= 0 and not rhs[rhs.index(var_name) - 1] == "&": return True # var_name has index access? - if ( - rhs.index(var_name) + len(var_name) < len(rhs) - and rhs[rhs.index(var_name) + len(var_name)] == "[" - ): + if rhs.index(var_name) + len(var_name) < len(rhs) and rhs[rhs.index(var_name) + len(var_name)] == "[": return True # '*' prior to '&'? - if ( - rhs.index(var_name) - 2 >= 0 - and rhs[rhs.index(var_name) - 1] == "&" - and rhs[rhs.index(var_name) - 2] == "*" - ): + if rhs.index(var_name) - 2 >= 0 and rhs[rhs.index(var_name) - 1] == "&" and rhs[rhs.index(var_name) - 2] == "*": return True # '*' and '(' prior, no ')' prior to '&'? if ( @@ -110,8 +98,7 @@ def __check_possible_pointer_type(var_name: str, rhs: str) -> bool: # '->' after var_name? if ( rhs.index(var_name) + len(var_name) + 2 > len(rhs) - and rhs[rhs.index(var_name) + len(var_name) : rhs.index(var_name) + len(var_name) + 1] - == "->" + and rhs[rhs.index(var_name) + len(var_name) : rhs.index(var_name) + len(var_name) + 1] == "->" ): return True return False @@ -154,18 +141,9 @@ def __get_alias_from_statement(var_name: str, var_type: str, statement: str) -> return None # operation is '=' ? stmt_copy = statement - stmt_copy = ( - stmt_copy.replace("<=>", " ") - .replace("<<=", " ") - .replace(">>=", " ") - .replace("==", " ") - ) - stmt_copy = ( - stmt_copy.replace("!=", " ").replace("+=", " ").replace("-=", " ").replace("*=", " ") - ) - stmt_copy = ( - stmt_copy.replace("/=", " ").replace("%=", " ").replace("<=", " ").replace(">=", " ") - ) + stmt_copy = stmt_copy.replace("<=>", " ").replace("<<=", " ").replace(">>=", " ").replace("==", " ") + stmt_copy = stmt_copy.replace("!=", " ").replace("+=", " ").replace("-=", " ").replace("*=", " ") + stmt_copy = stmt_copy.replace("/=", " ").replace("%=", " ").replace("<=", " ").replace(">=", " ") stmt_copy = stmt_copy.replace("&=", " ").replace("^=", " ").replace("|=", " ") if "=" not in stmt_copy: return None @@ -202,8 +180,8 @@ def __get_alias_from_statement(var_name: str, var_type: str, statement: str) -> def __add_alias_information( - function_information_list: List[Dict], statements_file: str -) -> List[Dict]: + function_information_list: List[Dict[str, Any]], statements_file: str +) -> List[Dict[str, Any]]: """Wrapper to gather and append alias information to the entries in function_information as a new field. Aliases can be found up to a depth of 2. Alias detection ignores scopes. @@ -252,14 +230,10 @@ def __add_alias_information( statement_line = statement[statement.index(":") + 1 :] statement_line = statement_line[: statement_line.index(":")] inner_statement_line = inner_statement[inner_statement.index(":") + 1 :] - inner_statement_line = inner_statement_line[ - : inner_statement_line.index(":") - ] + inner_statement_line = inner_statement_line[: inner_statement_line.index(":")] if int(inner_statement_line) < int(statement_line): continue - inner_statement_result = __get_alias_from_statement( - state_res_entry, "*", inner_statement - ) + inner_statement_result = __get_alias_from_statement(state_res_entry, "*", inner_statement) if inner_statement_result is not None: # second level alias found aliases += inner_statement_result @@ -270,7 +244,7 @@ def __add_alias_information( return result_list -def __get_function_information(cu_xml: str) -> List[Dict]: +def __get_function_information(cu_xml: str) -> List[Dict[str, Any]]: """Extracts information on functions from given cu_xml file and stores it in a dictionary representation. :param cu_xml: path to cu_xml file :return: List of dictionaries representing functions from cu_xml""" @@ -384,14 +358,7 @@ def get_alias_information(file_mapping: str, cu_xml: str, temp_file: str, build_ if len(alias_entry) > 0: for alias_name in alias_entry: alias_str += ( - fn_info["id"] - + ";" - + fn_info["name"] - + ";" - + fn_info["args"][idx] - + ";" - + alias_name - + "\n" + fn_info["id"] + ";" + fn_info["name"] + ";" + fn_info["args"][idx] + ";" + alias_name + "\n" ) # cleanup if os.path.exists(temp_file + "_statements"): diff --git a/discopop_explorer/pattern_detectors/task_parallelism/classes.py b/discopop_explorer/pattern_detectors/task_parallelism/classes.py index 0692f00d8..44a6fffb8 100644 --- a/discopop_explorer/pattern_detectors/task_parallelism/classes.py +++ b/discopop_explorer/pattern_detectors/task_parallelism/classes.py @@ -12,6 +12,7 @@ from discopop_explorer.PETGraphX import Node, MWType, PETGraphX from discopop_explorer.pattern_detectors.PatternInfo import PatternInfo + # We decided to omit the information that computes the workload and the relevant codes. For large programs (e.g., ffmpeg), the generated Data.xml file becomes very large. However, we keep the code here because we would like to integrate a hotspot detection algorithm (TODO: Bertin) with the parallelism discovery. Then, we need to retrieve the information to decide which code sections (loops or functions) are worth parallelizing. # from discopop_explorer.utils import total_instructions_count, calculate_workload @@ -55,9 +56,7 @@ def aggregate(self, other: "Task"): self.end_line = other.end_line self.workload += other.workload self.instruction_count += other.instruction_count - self.mw_type = ( - MWType.BARRIER_WORKER if other.mw_type == MWType.BARRIER_WORKER else MWType.WORKER - ) + self.mw_type = MWType.BARRIER_WORKER if other.mw_type == MWType.BARRIER_WORKER else MWType.WORKER class TPIType(Enum): @@ -71,9 +70,7 @@ class TPIType(Enum): class TaskParallelismInfo(PatternInfo): """Class, that contains task parallelism detection result""" - def __init__( - self, node: Node, type: TPIType, pragma, pragma_line, first_private, private, shared - ): + def __init__(self, node: Node, type: TPIType, pragma, pragma_line, first_private, private, shared): """ :param node: node, where task parallelism was detected :param type: type of the suggestion (task, taskwait, taskloop) diff --git a/discopop_explorer/pattern_detectors/task_parallelism/filter.py b/discopop_explorer/pattern_detectors/task_parallelism/filter.py index d9acf58e3..da0d9b660 100644 --- a/discopop_explorer/pattern_detectors/task_parallelism/filter.py +++ b/discopop_explorer/pattern_detectors/task_parallelism/filter.py @@ -38,9 +38,7 @@ def filter_data_sharing_clauses( return suggestions -def __filter_data_sharing_clauses_suppress_shared_loop_index( - pet: PETGraphX, suggestions: List[PatternInfo] -): +def __filter_data_sharing_clauses_suppress_shared_loop_index(pet: PETGraphX, suggestions: List[PatternInfo]): """Removes clauses for shared loop indices. :param pet: PET graph :param suggestions: List[PatternInfo] @@ -52,17 +50,13 @@ def __filter_data_sharing_clauses_suppress_shared_loop_index( if suggestion.type is not TPIType.TASK: continue # get parent loops of suggestion - parent_loops_plus_last_node = get_parent_of_type( - pet, suggestion._node, NodeType.LOOP, EdgeType.CHILD, True - ) + parent_loops_plus_last_node = get_parent_of_type(pet, suggestion._node, NodeType.LOOP, EdgeType.CHILD, True) parent_loops = [e[0] for e in parent_loops_plus_last_node] # consider only loops which enclose the suggestion parent_loops = [ loop for loop in parent_loops - if line_contained_in_region( - suggestion._node.start_position(), loop.start_position(), loop.end_position() - ) + if line_contained_in_region(suggestion._node.start_position(), loop.start_position(), loop.end_position()) ] to_be_removed = [] for var in suggestion.shared: @@ -94,9 +88,7 @@ def __filter_data_sharing_clauses_by_function( if suggestion.type not in [TPIType.TASK, TPIType.TASKLOOP]: continue # get function containing the task cu - parent_function, last_node = get_parent_of_type( - pet, suggestion._node, NodeType.FUNC, EdgeType.CHILD, True - )[0] + parent_function, last_node = get_parent_of_type(pet, suggestion._node, NodeType.FUNC, EdgeType.CHILD, True)[0] # filter firstprivate __filter_firstprivate_clauses(suggestion, parent_function, var_def_line_dict) # filter private @@ -107,9 +99,7 @@ def __filter_data_sharing_clauses_by_function( # remove duplicates and .addr suffix from variable names suggestion.shared = list(set([v.replace(".addr", "") for v in suggestion.shared])) suggestion.private = list(set([v.replace(".addr", "") for v in suggestion.private])) - suggestion.first_private = list( - set([v.replace(".addr", "") for v in suggestion.first_private]) - ) + suggestion.first_private = list(set([v.replace(".addr", "") for v in suggestion.first_private])) # remove duplicates (variable occurring in different classes) remove_from_first_private = [] @@ -124,19 +114,13 @@ def __filter_data_sharing_clauses_by_function( remove_from_first_private.append(var) remove_from_first_private = list(set(remove_from_first_private)) remove_from_private = list(set(remove_from_private)) - remove_from_private = [ - var for var in remove_from_private if var not in remove_from_first_private - ] + remove_from_private = [var for var in remove_from_private if var not in remove_from_first_private] suggestion.private = [var for var in suggestion.private if var not in remove_from_private] - suggestion.first_private = [ - var for var in suggestion.first_private if var not in remove_from_first_private - ] + suggestion.first_private = [var for var in suggestion.first_private if var not in remove_from_first_private] return suggestions -def __filter_shared_clauses( - suggestion: TaskParallelismInfo, parent_function, var_def_line_dict: Dict[str, List[str]] -): +def __filter_shared_clauses(suggestion: TaskParallelismInfo, parent_function, var_def_line_dict: Dict[str, List[str]]): """helper function for filter_data_sharing_clauses_by_function. Filters shared clauses. :param suggestion: Suggestion to be checked @@ -156,9 +140,7 @@ def __filter_shared_clauses( if def_line is None: is_valid = True # check if var is defined in parent function - if line_contained_in_region( - def_line, parent_function.start_position(), parent_function.end_position() - ): + if line_contained_in_region(def_line, parent_function.start_position(), parent_function.end_position()): is_valid = True else: pass @@ -168,14 +150,10 @@ def __filter_shared_clauses( if not is_valid: to_be_removed.append(var) to_be_removed = list(set(to_be_removed)) - suggestion.shared = [ - v for v in suggestion.shared if not v.replace(".addr", "") in to_be_removed - ] + suggestion.shared = [v for v in suggestion.shared if not v.replace(".addr", "") in to_be_removed] -def __filter_private_clauses( - suggestion: TaskParallelismInfo, parent_function, var_def_line_dict: Dict[str, List[str]] -): +def __filter_private_clauses(suggestion: TaskParallelismInfo, parent_function, var_def_line_dict: Dict[str, List[str]]): """helper function for filter_data_sharing_clauses_by_function. Filters private clauses. :param suggestion: Suggestion to be checked @@ -209,9 +187,7 @@ def __filter_private_clauses( if not is_valid: to_be_removed.append(var) to_be_removed = list(set(to_be_removed)) - suggestion.private = [ - v for v in suggestion.private if not v.replace(".addr", "") in to_be_removed - ] + suggestion.private = [v for v in suggestion.private if not v.replace(".addr", "") in to_be_removed] def __filter_firstprivate_clauses( @@ -233,9 +209,7 @@ def __filter_firstprivate_clauses( if defLine is None: is_valid = True # check if var is defined in parent function - if line_contained_in_region( - defLine, parent_function.start_position(), parent_function.end_position() - ): + if line_contained_in_region(defLine, parent_function.start_position(), parent_function.end_position()): is_valid = True else: pass @@ -244,14 +218,10 @@ def __filter_firstprivate_clauses( if not is_valid: to_be_removed.append(var) to_be_removed = list(set(to_be_removed)) - suggestion.first_private = [ - v for v in suggestion.first_private if not v.replace(".addr", "") in to_be_removed - ] + suggestion.first_private = [v for v in suggestion.first_private if not v.replace(".addr", "") in to_be_removed] -def __reverse_reachable_w_o_breaker( - pet: PETGraphX, root: Node, target: Node, breaker_cu: Node, visited: List[Node] -): +def __reverse_reachable_w_o_breaker(pet: PETGraphX, root: Node, target: Node, breaker_cu: Node, visited: List[Node]): """Helper function for filter_data_sharing_clauses_by_scope. Checks if target is reachable by traversing the successor graph in reverse, starting from root, without visiting breaker_cu. @@ -303,9 +273,9 @@ def __filter_data_sharing_clauses_by_scope( if suggestion.type is not TPIType.TASK: continue # get function containing the task cu - parent_function_cu, last_node = get_parent_of_type( - pet, suggestion._node, NodeType.FUNC, EdgeType.CHILD, True - )[0] + parent_function_cu, last_node = get_parent_of_type(pet, suggestion._node, NodeType.FUNC, EdgeType.CHILD, True)[ + 0 + ] # filter firstprivate __filter_sharing_clause(pet, suggestion, var_def_line_dict, parent_function_cu, "FP") # filter private @@ -347,9 +317,7 @@ def __filter_sharing_clause( # get CU which contains var_def_line var_def_cu: Optional[Node] = None for child_cu in get_cus_inside_function(pet, parent_function_cu): - if line_contained_in_region( - var_def_line, child_cu.start_position(), child_cu.end_position() - ): + if line_contained_in_region(var_def_line, child_cu.start_position(), child_cu.end_position()): var_def_cu = child_cu if var_def_cu is None: continue @@ -368,9 +336,7 @@ def __filter_sharing_clause( to_be_removed.append(var) to_be_removed = list(set(to_be_removed)) if target_clause_list == "FP": - suggestion.first_private = [ - v for v in suggestion.first_private if v not in to_be_removed - ] + suggestion.first_private = [v for v in suggestion.first_private if v not in to_be_removed] elif target_clause_list == "PR": suggestion.private = [v for v in suggestion.private if v not in to_be_removed] else: @@ -468,9 +434,7 @@ def __filter_in_dependencies( if defLine is None: is_valid = True # check if var is defined in parent function - if line_contained_in_region( - defLine, parent_function.start_position(), parent_function.end_position() - ): + if line_contained_in_region(defLine, parent_function.start_position(), parent_function.end_position()): # check if var is contained in out_dep_vars and a previous out_dep exists if var in out_dep_vars: for line_num in out_dep_vars[var]: @@ -505,9 +469,7 @@ def __filter_in_dependencies( modification_found = True to_be_removed.append(var) to_be_removed = list(set(to_be_removed)) - suggestion.in_dep = [ - v for v in suggestion.in_dep if not v.replace(".addr", "") in to_be_removed - ] + suggestion.in_dep = [v for v in suggestion.in_dep if not v.replace(".addr", "") in to_be_removed] return modification_found @@ -537,9 +499,7 @@ def __filter_out_dependencies( if defLine is None: is_valid = True # check if var is defined in parent function - if line_contained_in_region( - defLine, parent_function.start_position(), parent_function.end_position() - ): + if line_contained_in_region(defLine, parent_function.start_position(), parent_function.end_position()): # check if var is contained in in_dep_vars and a successive in_dep exists if var in in_dep_vars: for line_num in in_dep_vars[var]: @@ -574,9 +534,7 @@ def __filter_out_dependencies( to_be_removed.append(var) modification_found = True to_be_removed = list(set(to_be_removed)) - suggestion.out_dep = [ - v for v in suggestion.out_dep if not v.replace(".addr", "") in to_be_removed - ] + suggestion.out_dep = [v for v in suggestion.out_dep if not v.replace(".addr", "") in to_be_removed] return modification_found @@ -608,9 +566,7 @@ def __filter_in_out_dependencies( if defLine is None: is_valid = True # check if var is defined in parent function - if line_contained_in_region( - defLine, parent_function.start_position(), parent_function.end_position() - ): + if line_contained_in_region(defLine, parent_function.start_position(), parent_function.end_position()): # check if var occurs more than once as in or out, i.e. at least an actual in or out # dependency exists if len(in_dep_vars[var]) > 1 or len(out_dep_vars[var]) > 1: @@ -685,9 +641,7 @@ def __filter_in_out_dependencies( to_be_removed.append(var) modification_found = True to_be_removed = list(set(to_be_removed)) - suggestion.in_out_dep = [ - v for v in suggestion.in_out_dep if not v.replace(".addr", "") in to_be_removed - ] + suggestion.in_out_dep = [v for v in suggestion.in_out_dep if not v.replace(".addr", "") in to_be_removed] return modification_found @@ -737,9 +691,9 @@ def filter_data_depend_clauses( if suggestion.type not in [TPIType.TASK, TPIType.TASKLOOP]: continue # get function containing the task cu - parent_function, last_node = get_parent_of_type( - pet, suggestion._node, NodeType.FUNC, EdgeType.CHILD, True - )[0] + parent_function, last_node = get_parent_of_type(pet, suggestion._node, NodeType.FUNC, EdgeType.CHILD, True)[ + 0 + ] # filter in_dep modification_found = modification_found or __filter_in_dependencies( pet, suggestion, var_def_line_dict, parent_function, out_dep_vars diff --git a/discopop_explorer/pattern_detectors/task_parallelism/postprocessor.py b/discopop_explorer/pattern_detectors/task_parallelism/postprocessor.py index 834ca63a9..ae998f6e2 100644 --- a/discopop_explorer/pattern_detectors/task_parallelism/postprocessor.py +++ b/discopop_explorer/pattern_detectors/task_parallelism/postprocessor.py @@ -30,24 +30,16 @@ def group_task_suggestions(pet: PETGraphX, suggestions: List[PatternInfo]) -> Li :param pet: PET Graph :param suggestions: Found suggestions :return: Updated suggestions""" - task_suggestions = [ - s - for s in [e for e in suggestions if type(e) == TaskParallelismInfo] - if s.type is TPIType.TASK - ] + task_suggestions = [s for s in [e for e in suggestions if type(e) == TaskParallelismInfo] if s.type is TPIType.TASK] taskwait_suggestions = [ - s - for s in [e for e in suggestions if type(e) == TaskParallelismInfo] - if s.type is TPIType.TASKWAIT + s for s in [e for e in suggestions if type(e) == TaskParallelismInfo] if s.type is TPIType.TASKWAIT ] # mark preceeding suggestions for each taskwait suggestion for task_group_id, tws in enumerate(taskwait_suggestions): # mark taskwait suggestion with own id tws.task_group.append(task_group_id) relatives: List[Node] = [tws._node] - queue: List[Node] = [ - pet.node_at(in_e[0]) for in_e in pet.in_edges(tws._node.id, EdgeType.SUCCESSOR) - ] + queue: List[Node] = [pet.node_at(in_e[0]) for in_e in pet.in_edges(tws._node.id, EdgeType.SUCCESSOR)] while len(queue) > 0: cur = queue.pop(0) if cur.tp_contains_taskwait: @@ -91,10 +83,7 @@ def group_task_suggestions(pet: PETGraphX, suggestions: List[PatternInfo]) -> Li break # execute replacement for sug in task_suggestions + taskwait_suggestions: - sug.task_group = [ - replacements[tg_elem] if tg_elem in replacements else tg_elem - for tg_elem in sug.task_group - ] + sug.task_group = [replacements[tg_elem] if tg_elem in replacements else tg_elem for tg_elem in sug.task_group] # validate and combine results # valid, if all entries in sug.task_group are equal. Replace by single entry if valid. value: Optional[int] = None diff --git a/discopop_explorer/pattern_detectors/task_parallelism/preprocessor.py b/discopop_explorer/pattern_detectors/task_parallelism/preprocessor.py index 07e87e07c..f9682382e 100644 --- a/discopop_explorer/pattern_detectors/task_parallelism/preprocessor.py +++ b/discopop_explorer/pattern_detectors/task_parallelism/preprocessor.py @@ -12,8 +12,7 @@ from lxml import objectify, etree # type: ignore -from discopop_explorer.PETGraphX import LoopNode, NodeType, PETGraphX - +from discopop_explorer.PETGraphX import LoopNode, PETGraphX from discopop_explorer.pattern_detectors.task_parallelism.tp_utils import line_contained_in_region @@ -58,12 +57,8 @@ def cu_xml_preprocessing(cu_xml: str) -> str: for cne_idx, calls_node_entry in enumerate(node.callsNode): # get first matching entry of node.callsNode try: - for rc_idx, rec_call in enumerate( - calls_node_entry.recursiveFunctionCall - ): - rec_call_line = calls_node_entry.nodeCalled[rc_idx].get( - "atLine" - ) + for rc_idx, rec_call in enumerate(calls_node_entry.recursiveFunctionCall): + rec_call_line = calls_node_entry.nodeCalled[rc_idx].get("atLine") if str(rec_call_line) in str(rec_call): tmp_cn_entry = ( rec_call, @@ -82,9 +77,7 @@ def cu_xml_preprocessing(cu_xml: str) -> str: parsed_cu.insert(parsed_cu.index(parent), parent_copy) # Preprocessor Step 2 - generate cu id for new element - __generate_new_cu_id( - parent, parent_copy, used_node_ids, self_added_node_ids - ) + __generate_new_cu_id(parent, parent_copy, used_node_ids, self_added_node_ids) # Preprocessor Step 3 parent_copy.callsNode.clear() @@ -97,12 +90,8 @@ def cu_xml_preprocessing(cu_xml: str) -> str: # delete childrenNodes-entry from parent tmp_cu_id = tmp_cn_entry[1].text - parent.childrenNodes._setText( - parent.childrenNodes.text.replace(tmp_cu_id + ",", "") - ) - parent.childrenNodes._setText( - parent.childrenNodes.text.replace(tmp_cu_id, "") - ) + parent.childrenNodes._setText(parent.childrenNodes.text.replace(tmp_cu_id + ",", "")) + parent.childrenNodes._setText(parent.childrenNodes.text.replace(tmp_cu_id, "")) # set parent_copy.childrenNodes __set_parent_copy_childrennodes(parent_copy) @@ -129,10 +118,7 @@ def cu_xml_preprocessing(cu_xml: str) -> str: for tmp in potential_lines: if tmp == "": continue - if ( - int(tmp[tmp.find(":") + 1 :]) - >= int(separator_line[separator_line.find(":") + 1 :]) + 1 - ): + if int(tmp[tmp.find(":") + 1 :]) >= int(separator_line[separator_line.find(":") + 1 :]) + 1: if parent_new_start_line is None: parent_new_start_line = tmp continue @@ -144,9 +130,7 @@ def cu_xml_preprocessing(cu_xml: str) -> str: if not potential_lines or (potential_lines and not parent_new_start_line): parent_new_start_line = str(separator_line[: separator_line.index(":")]) parent_new_start_line += ":" - parent_new_start_line += str( - int(separator_line[separator_line.index(":") + 1 :]) + 1 - ) + parent_new_start_line += str(int(separator_line[separator_line.index(":") + 1 :]) + 1) parent.set("startsAtLine", parent_new_start_line) parent_copy.set("endsAtLine", separator_line) @@ -179,12 +163,8 @@ def cu_xml_preprocessing(cu_xml: str) -> str: for cne_idx, calls_node_entry in enumerate(parent.callsNode): # get first matching entry of node.callsNode try: - for rc_idx, rec_call in enumerate( - calls_node_entry.recursiveFunctionCall - ): - rec_call_line = calls_node_entry.nodeCalled[rc_idx].get( - "atLine" - ) + for rc_idx, rec_call in enumerate(calls_node_entry.recursiveFunctionCall): + rec_call_line = calls_node_entry.nodeCalled[rc_idx].get("atLine") if str(rec_call_line) in str(rec_call): parent_further_cn_entry = ( rec_call, @@ -232,9 +212,7 @@ def __generate_new_cu_id(parent, parent_copy, used_node_ids, self_added_node_ids # get next free id for specific tmp_file_id parent_copy_id = parent_copy.get("id") tmp_file_id = parent_copy_id[: parent_copy_id.index(":")] - tmp_used_ids = [ - int(s[s.index(":") + 1 :]) for s in used_node_ids if s.startswith(tmp_file_id + ":") - ] + tmp_used_ids = [int(s[s.index(":") + 1 :]) for s in used_node_ids if s.startswith(tmp_file_id + ":")] next_free_id = max(tmp_used_ids) + 1 incremented_id = tmp_file_id + ":" + str(next_free_id) parent.set("id", incremented_id) @@ -250,9 +228,7 @@ def __set_parent_copy_childrennodes(parent_copy): for node_call in calls_node_entry.nodeCalled: try: if node_call.text not in parent_copy.childrenNodes.text: - parent_copy.childrenNodes._setText( - parent_copy.childrenNodes.text + "," + node_call.text - ) + parent_copy.childrenNodes._setText(parent_copy.childrenNodes.text + "," + node_call.text) if parent_copy.childrenNodes.text.startswith(","): parent_copy.childrenNodes._setText(parent_copy.childrenNodes.text[1:]) if parent_copy.childrenNodes.text.endswith(","): @@ -274,12 +250,8 @@ def __remove_overlapping_start_and_end_lines(parent_copy, target_list): """ try: if parent_copy.callsNode.nodeCalled.get("atLine") in target_list.text: - target_list._setText( - target_list.text.replace(parent_copy.callsNode.nodeCalled.get("atLine") + ",", "") - ) - target_list._setText( - target_list.text.replace(parent_copy.callsNode.nodeCalled.get("atLine"), "") - ) + target_list._setText(target_list.text.replace(parent_copy.callsNode.nodeCalled.get("atLine") + ",", "")) + target_list._setText(target_list.text.replace(parent_copy.callsNode.nodeCalled.get("atLine"), "")) target_list.set("count", str(int(target_list.get("count")) - 1)) except TypeError: target_list._setText(parent_copy.callsNode.nodeCalled.get("atLine")) @@ -293,9 +265,7 @@ def __filter_rwi_lines(parent_copy, target_list): :param target_list: eiter readPhaseLines, writePhaseLines or instructionLines of parent_copy""" try: for tmp_line in target_list.text.split(","): - if not line_contained_in_region( - tmp_line, parent_copy.get("startsAtLine"), parent_copy.get("endsAtLine") - ): + if not line_contained_in_region(tmp_line, parent_copy.get("startsAtLine"), parent_copy.get("endsAtLine")): target_list._setText(target_list.text.replace(tmp_line + ",", "")) target_list._setText(target_list.text.replace(tmp_line, "")) if target_list.text.endswith(","): @@ -333,9 +303,7 @@ def __insert_missing_rwi_lines(parent, target_list): target_list._setText(target_list.text[:-1]) target_list.set("count", str(int(target_list.get("count")) + 1)) # increment cur_line by one - cur_line = cur_line[0 : cur_line.rfind(":") + 1] + str( - int(cur_line[cur_line.rfind(":") + 1 :]) + 1 - ) + cur_line = cur_line[0 : cur_line.rfind(":") + 1] + str(int(cur_line[cur_line.rfind(":") + 1 :]) + 1) continue target_list._setText(target_list.text.replace(",,", ",")) @@ -347,9 +315,7 @@ def __remove_unnecessary_return_instructions(target): entries = target.returnInstructions.text.split(",") new_entries = [] for entry in entries: - if line_contained_in_region( - entry, target.get("startsAtLine"), target.get("endsAtLine") - ): + if line_contained_in_region(entry, target.get("startsAtLine"), target.get("endsAtLine")): new_entries.append(entry) target.returnInstructions._setText(",".join(new_entries)) target.returnInstructions.set("count", str(len(new_entries))) @@ -376,9 +342,7 @@ def __add_parent_id_to_children(parsed_cu, parent): if parent_function is None: print("No parent function found for cu node: ", parent.get("id"), ". Ignoring.") else: - parent_function.childrenNodes._setText( - parent_function.childrenNodes.text + "," + parent.get("id") - ) + parent_function.childrenNodes._setText(parent_function.childrenNodes.text + "," + parent.get("id")) if parent_function.childrenNodes.text.startswith(","): parent_function.childrenNodes._setText(parent_function.childrenNodes.text[1:]) @@ -425,15 +389,11 @@ def check_loop_scopes(pet: PETGraphX): :param pet: PET graph""" for loop_cu in pet.all_nodes(LoopNode): for child in pet.direct_children_or_called_nodes(loop_cu): - if not line_contained_in_region( - child.start_position(), loop_cu.start_position(), loop_cu.end_position() - ): + if not line_contained_in_region(child.start_position(), loop_cu.start_position(), loop_cu.end_position()): # expand loop_cu start_position upwards if child.start_line < loop_cu.start_line and loop_cu.file_id == child.file_id: loop_cu.start_line = child.start_line - if not line_contained_in_region( - child.end_position(), loop_cu.start_position(), loop_cu.end_position() - ): + if not line_contained_in_region(child.end_position(), loop_cu.start_position(), loop_cu.end_position()): # expand loop_cu end_position downwards if child.end_line > loop_cu.end_line and loop_cu.file_id == child.file_id: loop_cu.end_line = child.end_line diff --git a/discopop_explorer/pattern_detectors/task_parallelism/suggesters/auxiliary.py b/discopop_explorer/pattern_detectors/task_parallelism/suggesters/auxiliary.py index 83bba1c20..7125fdfe7 100644 --- a/discopop_explorer/pattern_detectors/task_parallelism/suggesters/auxiliary.py +++ b/discopop_explorer/pattern_detectors/task_parallelism/suggesters/auxiliary.py @@ -14,7 +14,6 @@ from discopop_explorer.pattern_detectors.task_parallelism.classes import ( TaskParallelismInfo, ParallelRegionInfo, - Task, OmittableCuInfo, TPIType, ) @@ -24,9 +23,7 @@ ) -def suggest_parallel_regions( - pet: PETGraphX, suggestions: List[TaskParallelismInfo] -) -> List[ParallelRegionInfo]: +def suggest_parallel_regions(pet: PETGraphX, suggestions: List[TaskParallelismInfo]) -> List[ParallelRegionInfo]: """create suggestions for parallel regions based on suggested tasks. Parallel regions are suggested aroung each outer-most function call possibly leading to the creation of tasks. @@ -66,9 +63,7 @@ def suggest_parallel_regions( continue last_node = last_node region_suggestions.append( - ParallelRegionInfo( - parent, TPIType.PARALLELREGION, last_node.start_position(), last_node.end_position() - ) + ParallelRegionInfo(parent, TPIType.PARALLELREGION, last_node.start_position(), last_node.end_position()) ) return region_suggestions @@ -116,9 +111,7 @@ def set_task_contained_lines(suggestions: List[TaskParallelismInfo]) -> List[Tas return output -def detect_taskloop_reduction( - pet: PETGraphX, suggestions: List[TaskParallelismInfo] -) -> List[TaskParallelismInfo]: +def detect_taskloop_reduction(pet: PETGraphX, suggestions: List[TaskParallelismInfo]) -> List[TaskParallelismInfo]: """detect suggested tasks which can and should be replaced by taskloop reduction. return the modified list of suggestions. @@ -193,12 +186,8 @@ def combine_omittable_cus(pet: PETGraphX, suggestions: List[PatternInfo]) -> Lis # successor graph of a node containing a task suggestion useful_omittable_suggestions = [] for oms in omittable_suggestions: - in_succ_edges = [ - (s, t, e) for s, t, e in pet.in_edges(oms._node.id) if e.etype == EdgeType.SUCCESSOR - ] - parent_task_nodes = [ - pet.node_at(e[0]) for e in in_succ_edges if pet.node_at(e[0]).tp_contains_task is True - ] + in_succ_edges = [(s, t, e) for s, t, e in pet.in_edges(oms._node.id) if e.etype == EdgeType.SUCCESSOR] + parent_task_nodes = [pet.node_at(e[0]) for e in in_succ_edges if pet.node_at(e[0]).tp_contains_task is True] if len(parent_task_nodes) != 0: useful_omittable_suggestions.append(oms) else: @@ -246,37 +235,26 @@ def combine_omittable_cus(pet: PETGraphX, suggestions: List[PatternInfo]) -> Lis for omit_out_var in omit_s.out_dep: if omit_out_var is None: continue - task_suggestions_dict[omit_s.combine_with_node][ - omit_target_task_idx - ].out_dep.append(omit_out_var) + task_suggestions_dict[omit_s.combine_with_node][omit_target_task_idx].out_dep.append(omit_out_var) # omit_s.combine_with_node.out_dep.append(omit_out_var) # process in dependencies of omit_s for omit_in_var in omit_s.in_dep: # note: only dependencies to target node allowed - if ( - omit_in_var - in task_suggestions_dict[omit_s.combine_with_node][ - omit_target_task_idx - ].out_dep - ): - task_suggestions_dict[omit_s.combine_with_node][ - omit_target_task_idx - ].out_dep.remove(omit_in_var) + if omit_in_var in task_suggestions_dict[omit_s.combine_with_node][omit_target_task_idx].out_dep: + task_suggestions_dict[omit_s.combine_with_node][omit_target_task_idx].out_dep.remove( + omit_in_var + ) # omit_s.combine_with_node.out_dep.remove(omit_in_var) # increase size of pragma region if needed if ":" not in cast( str, - task_suggestions_dict[omit_s.combine_with_node][ - omit_target_task_idx - ].region_end_line, + task_suggestions_dict[omit_s.combine_with_node][omit_target_task_idx].region_end_line, ): if int(omit_s.end_line[omit_s.end_line.index(":") + 1 :]) > int( cast( str, - task_suggestions_dict[omit_s.combine_with_node][ - omit_target_task_idx - ].region_end_line, + task_suggestions_dict[omit_s.combine_with_node][omit_target_task_idx].region_end_line, ) ): task_suggestions_dict[omit_s.combine_with_node][ @@ -285,14 +263,10 @@ def combine_omittable_cus(pet: PETGraphX, suggestions: List[PatternInfo]) -> Lis else: cut_region_end_line = cast( str, - task_suggestions_dict[omit_s.combine_with_node][ - omit_target_task_idx - ].region_end_line, + task_suggestions_dict[omit_s.combine_with_node][omit_target_task_idx].region_end_line, ) cut_region_end_line = cut_region_end_line[cut_region_end_line.index(":") + 1 :] - if int(omit_s.end_line[omit_s.end_line.index(":") + 1 :]) > int( - cut_region_end_line - ): + if int(omit_s.end_line[omit_s.end_line.index(":") + 1 :]) > int(cut_region_end_line): task_suggestions_dict[omit_s.combine_with_node][ omit_target_task_idx ].region_end_line = omit_s.end_line diff --git a/discopop_explorer/pattern_detectors/task_parallelism/suggesters/barriers.py b/discopop_explorer/pattern_detectors/task_parallelism/suggesters/barriers.py index 05478e930..d7137ccea 100644 --- a/discopop_explorer/pattern_detectors/task_parallelism/suggesters/barriers.py +++ b/discopop_explorer/pattern_detectors/task_parallelism/suggesters/barriers.py @@ -59,9 +59,7 @@ def detect_barrier_suggestions(pet: PETGraphX, suggestions: List[PatternInfo]) - v = queue.pop(0) # check step 1 out_dep_edges = [ - (s, t, e) - for s, t, e in pet.out_edges(v.id) - if e.etype == EdgeType.DATA and pet.node_at(t) != v + (s, t, e) for s, t, e in pet.out_edges(v.id) if e.etype == EdgeType.DATA and pet.node_at(t) != v ] # ignore cyclic dependencies on the same variable to_remove = [] @@ -69,9 +67,7 @@ def detect_barrier_suggestions(pet: PETGraphX, suggestions: List[PatternInfo]) - targets_cyclic_dep_edges = [ (s, t, e) for s, t, e in pet.out_edges(dep_edge[1]) - if e.etype == EdgeType.DATA - and t == dep_edge[0] - and e.var_name == dep_edge[2].var_name + if e.etype == EdgeType.DATA and t == dep_edge[0] and e.var_name == dep_edge[2].var_name ] if len(targets_cyclic_dep_edges) != 0: to_remove.append(dep_edge) @@ -88,9 +84,7 @@ def detect_barrier_suggestions(pet: PETGraphX, suggestions: List[PatternInfo]) - if not v.tp_omittable: # actual change v.tp_omittable = True - combine_with_node_list = [ - pet.node_at(e[1]) for e in out_dep_edges if pet.node_at(e[1]) in task_nodes - ] + combine_with_node_list = [pet.node_at(e[1]) for e in out_dep_edges if pet.node_at(e[1]) in task_nodes] if len(combine_with_node_list) < 1: raise ValueError("length combine_with_node < 1!") combine_with_node = combine_with_node_list[0] @@ -99,14 +93,8 @@ def detect_barrier_suggestions(pet: PETGraphX, suggestions: List[PatternInfo]) - transformation_happened = True elif barrier_count != 0 and task_count != 0: # check if child barrier(s) cover each child task - child_barriers = [ - e[1] for e in out_dep_edges if pet.node_at(e[1]).tp_contains_taskwait is True - ] - child_tasks = [ - pet.node_at(e[1]) - for e in out_dep_edges - if pet.node_at(e[1]).tp_contains_task is True - ] + child_barriers = [e[1] for e in out_dep_edges if pet.node_at(e[1]).tp_contains_taskwait is True] + child_tasks = [pet.node_at(e[1]) for e in out_dep_edges if pet.node_at(e[1]).tp_contains_task is True] uncovered_task_exists = False for ct in child_tasks: ct_start_line = ct.start_position() @@ -129,24 +117,18 @@ def detect_barrier_suggestions(pet: PETGraphX, suggestions: List[PatternInfo]) - v.tp_contains_taskwait = True barrier_nodes.append(v) transformation_happened = True - tmp_suggestion = TaskParallelismInfo( - v, TPIType.TASKWAIT, ["taskwait"], v_first_line, [], [], [] - ) + tmp_suggestion = TaskParallelismInfo(v, TPIType.TASKWAIT, ["taskwait"], v_first_line, [], [], []) suggestions.append(tmp_suggestion) else: # no barrier needed pass - elif ( - omittable_count == 0 and task_count > 1 - ): # connected to at least two distinct task nodes + elif omittable_count == 0 and task_count > 1: # connected to at least two distinct task nodes if v.tp_contains_taskwait is False: # actual change v.tp_contains_taskwait = True barrier_nodes.append(v) transformation_happened = True - tmp_suggestion = TaskParallelismInfo( - v, TPIType.TASKWAIT, ["taskwait"], v_first_line, [], [], [] - ) + tmp_suggestion = TaskParallelismInfo(v, TPIType.TASKWAIT, ["taskwait"], v_first_line, [], [], []) suggestions.append(tmp_suggestion) if omittable_count == 1 and v.tp_contains_task is False and v.tp_contains_taskwait is False: # omittable node appended to prior omittable node @@ -178,9 +160,7 @@ def detect_barrier_suggestions(pet: PETGraphX, suggestions: List[PatternInfo]) - # append neighbors of modified node to queue if transformation_happened: in_dep_edges = [ - (s, t, e) - for s, t, e in pet.in_edges(v.id) - if e.etype == EdgeType.DATA and pet.node_at(s) != v + (s, t, e) for s, t, e in pet.in_edges(v.id) if e.etype == EdgeType.DATA and pet.node_at(s) != v ] for e in out_dep_edges: queue.append(pet.node_at(e[1])) @@ -195,9 +175,9 @@ def __count_adjacent_nodes( pet: PETGraphX, suggestions: List[PatternInfo], out_dep_edges: List[Tuple[Any, Any, Any]], - task_nodes: List, - barrier_nodes: List, - omittable_nodes: List, + task_nodes: List[Node], + barrier_nodes: List[Node], + omittable_nodes: List[Tuple[Node, List[Node]]], ) -> Tuple[int, int, int, int]: """Checks the types of nodes pointed to by out_dep_edges and increments the respective counters. :param pet: PET Graph @@ -235,9 +215,7 @@ def __count_adjacent_nodes( tmp_omit_suggestions: List[OmittableCuInfo] = cast( List[OmittableCuInfo], [s for s in suggestions if type(s) == OmittableCuInfo] ) - parent_task = [tos for tos in tmp_omit_suggestions if tos._node == pet.node_at(e[1])][ - 0 - ].combine_with_node + parent_task = [tos for tos in tmp_omit_suggestions if tos._node == pet.node_at(e[1])][0].combine_with_node if parent_task.id not in omittable_parent_buffer: omittable_parent_buffer.append(parent_task.id) omittable_count += 1 @@ -271,9 +249,7 @@ def __check_dependences_and_predecessors( else: violation = True # check if node is a direct successor of an omittable node or a task node - in_succ_edges = [ - (s, t, e) for (s, t, e) in pet.in_edges(cur_cu.id) if e.etype == EdgeType.SUCCESSOR - ] + in_succ_edges = [(s, t, e) for (s, t, e) in pet.in_edges(cur_cu.id) if e.etype == EdgeType.SUCCESSOR] is_successor = False for e in in_succ_edges: if pet.node_at(e[0]).tp_omittable is True: @@ -328,9 +304,7 @@ def suggest_barriers_for_uncovered_tasks_before_return( covered_by_parallel_region = False for tmp in suggestions: if type(tmp) == ParallelRegionInfo: - if line_contained_in_region( - suggestion.start_line, tmp.region_start_line, tmp.region_end_line - ): + if line_contained_in_region(suggestion.start_line, tmp.region_start_line, tmp.region_end_line): covered_by_parallel_region = True break if covered_by_parallel_region: @@ -360,9 +334,7 @@ def suggest_barriers_for_uncovered_tasks_before_return( cu.tp_contains_taskwait = True pragma_line = cu.end_position() # since return has to be the last statement in a CU pragma_line = LineID(pragma_line[pragma_line.index(":") + 1 :]) - tmp_suggestion = TaskParallelismInfo( - cu, TPIType.TASKWAIT, ["taskwait"], pragma_line, [], [], [] - ) + tmp_suggestion = TaskParallelismInfo(cu, TPIType.TASKWAIT, ["taskwait"], pragma_line, [], [], []) print( "TPDet:suggest_barriers_for_uncovered_tasks_before_return: added taskwait suggestion at line: ", cu.end_position(), @@ -447,9 +419,7 @@ def validate_barriers(pet: PETGraphX, suggestions: List[PatternInfo]) -> List[Pa return result -def suggest_missing_barriers_for_global_vars( - pet: PETGraphX, suggestions: List[PatternInfo] -) -> List[PatternInfo]: +def suggest_missing_barriers_for_global_vars(pet: PETGraphX, suggestions: List[PatternInfo]) -> List[PatternInfo]: """Suggests a barrier if a node is a successor of a task CU which is not covered by an existing barrier and the set of global variables of the CU and the task are overlapping @@ -465,10 +435,7 @@ def suggest_missing_barriers_for_global_vars( taskwait_suggestions = [] task_suggestions = [] for single_suggestion in suggestions: - if ( - type(single_suggestion) == ParallelRegionInfo - or type(single_suggestion) == OmittableCuInfo - ): + if type(single_suggestion) == ParallelRegionInfo or type(single_suggestion) == OmittableCuInfo: continue if type(single_suggestion) == TaskParallelismInfo: if single_suggestion.type is TPIType.TASKWAIT: diff --git a/discopop_explorer/pattern_detectors/task_parallelism/suggesters/data_sharing_clauses.py b/discopop_explorer/pattern_detectors/task_parallelism/suggesters/data_sharing_clauses.py index 56c21dc56..78495d0e3 100644 --- a/discopop_explorer/pattern_detectors/task_parallelism/suggesters/data_sharing_clauses.py +++ b/discopop_explorer/pattern_detectors/task_parallelism/suggesters/data_sharing_clauses.py @@ -6,9 +6,9 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from typing import List, cast +from typing import List -from discopop_explorer.PETGraphX import EdgeType, FunctionNode, NodeType, PETGraphX +from discopop_explorer.PETGraphX import EdgeType, FunctionNode, PETGraphX from discopop_explorer.pattern_detectors.PatternInfo import PatternInfo from discopop_explorer.pattern_detectors.task_parallelism.classes import ( TaskParallelismInfo, @@ -26,11 +26,7 @@ def suggest_shared_clauses_for_all_tasks_in_function_body( :param: suggestions: List[PatternInfo] :return: List[PatternInfo] """ - task_suggestions = [ - s - for s in [t for t in suggestions if type(t) == TaskParallelismInfo] - if s.type is TPIType.TASK - ] + task_suggestions = [s for s in [t for t in suggestions if type(t) == TaskParallelismInfo] if s.type is TPIType.TASK] for ts in task_suggestions: if ts.shared: # iterate over parent function(s) diff --git a/discopop_explorer/pattern_detectors/task_parallelism/suggesters/dependency_clauses.py b/discopop_explorer/pattern_detectors/task_parallelism/suggesters/dependency_clauses.py index 2a013bb24..46cf44bac 100644 --- a/discopop_explorer/pattern_detectors/task_parallelism/suggesters/dependency_clauses.py +++ b/discopop_explorer/pattern_detectors/task_parallelism/suggesters/dependency_clauses.py @@ -7,23 +7,23 @@ # directory for details. import os -from typing import List, Dict, Tuple, Optional, Union, cast +from typing import List, Dict, Tuple, Optional, cast, Any from discopop_explorer.PETGraphX import ( CUNode, - DummyNode, EdgeType, FunctionNode, - NodeType, Node, PETGraphX, NodeID, LineID, ) from discopop_explorer.pattern_detectors.PatternInfo import PatternInfo +from discopop_explorer.pattern_detectors.task_parallelism.alias_detection import ( + get_alias_information as get_alias_detection_result, +) from discopop_explorer.pattern_detectors.task_parallelism.classes import ( TaskParallelismInfo, - OmittableCuInfo, TPIType, ) from discopop_explorer.pattern_detectors.task_parallelism.tp_utils import ( @@ -33,9 +33,6 @@ demangle, get_called_functions_recursively, ) -from discopop_explorer.pattern_detectors.task_parallelism.alias_detection import ( - get_alias_information as get_alias_detection_result, -) def detect_dependency_clauses_alias_based( @@ -165,9 +162,7 @@ def get_dict_from_cu_inst_result_file( return res_dict -def get_alias_information( - pet: PETGraphX, suggestions: List[PatternInfo], source_code_files: Dict[str, str] -): +def get_alias_information(pet: PETGraphX, suggestions: List[PatternInfo], source_code_files: Dict[str, str]): """Generate and return alias information dictionary. :param pet: PET Graph :param suggestions: List[PatternInfo] @@ -175,14 +170,10 @@ def get_alias_information( :return: alias information dictionary """ # iterate over task suggestions - task_suggestions = [ - s - for s in [e for e in suggestions if type(e) == TaskParallelismInfo] - if s.type is TPIType.TASK - ] + task_suggestions = [s for s in [e for e in suggestions if type(e) == TaskParallelismInfo] if s.type is TPIType.TASK] # collect alias information aliases: Dict[TaskParallelismInfo, List[List[Tuple[str, str, LineID, LineID]]]] = dict() - called_function_cache: Dict = dict() + called_function_cache: Dict[Any, Any] = dict() for ts in task_suggestions: current_alias_entry = [] potential_parent_functions = __get_potential_parent_functions(pet, ts) @@ -304,9 +295,9 @@ def get_function_internal_parameter_aliases( def identify_dependencies_for_different_functions( pet: PETGraphX, suggestions: List[PatternInfo], - aliases: Dict, - source_code_files: Dict, - raw_dependency_information: Dict, + aliases: Dict[TaskParallelismInfo, List[List[Tuple[str, str, LineID, LineID]]]], + source_code_files: Dict[str, str], + raw_dependency_information: Dict[str, List[Tuple[str, str]]], ) -> List[PatternInfo]: """Identify dependency clauses for all combinations of suggested tasks concerning different called functions and supplement the suggestions. @@ -382,10 +373,7 @@ def identify_dependencies_for_different_functions( ) # check if task suggestion_1 occurs prior to task_suggestion_2 - if ( - ts_1._node.start_position().split(":")[0] - == ts_2._node.start_position().split(":")[0] - ): + if ts_1._node.start_position().split(":")[0] == ts_2._node.start_position().split(":")[0]: # same file id ts_1_pragma_line = ( int(ts_1.pragma_line) @@ -474,18 +462,12 @@ def __get_potential_children_of_function(pet: PETGraphX, parent_function: Node) ): potential_children.append(cur_potential_child) for tmp_child in pet.direct_children_or_called_nodes(cur_potential_child): - if ( - tmp_child not in queue - and tmp_child not in potential_children - and tmp_child not in visited - ): + if tmp_child not in queue and tmp_child not in potential_children and tmp_child not in visited: queue.append(tmp_child) return potential_children -def __get_recursive_calls_from_function( - potential_children: List[Node], parent_function: Node -) -> List[str]: +def __get_recursive_calls_from_function(potential_children: List[Node], parent_function: Node) -> List[str]: """Helper function for identify_dependencies_for_same_functions. Creates a list of recursive function calls located inside the body of the given function. :param potential_children: List of CUs contained in function's body @@ -499,9 +481,7 @@ def __get_recursive_calls_from_function( if line_contained_in_region( c.start_position(), parent_function.start_position(), parent_function.end_position() ) - and line_contained_in_region( - c.end_position(), parent_function.start_position(), parent_function.end_position() - ) + and line_contained_in_region(c.end_position(), parent_function.start_position(), parent_function.end_position()) ]: for e in child.recursive_function_calls: if e is not None: @@ -513,8 +493,8 @@ def __get_recursive_calls_from_function( def identify_dependencies_for_same_functions( pet: PETGraphX, suggestions: List[PatternInfo], - source_code_files: Dict, - cu_inst_result_dict: Dict, + source_code_files: Dict[str, str], + cu_inst_result_dict: Dict[str, List[Dict[str, Optional[str]]]], function_parameter_alias_dict: Dict[str, List[Tuple[str, str]]], ) -> List[PatternInfo]: """Identify dependency clauses for all combinations of suggested tasks concerning equal called functions @@ -559,29 +539,21 @@ def identify_dependencies_for_same_functions( cur_potential_parent_function = potential_parent_functions_1.pop() # get recursive function call from original source code try: - get_function_call_from_source_code( - source_code_files, int(ts_1.pragma_line), ts_1.node_id.split(":")[0] - ) + get_function_call_from_source_code(source_code_files, int(ts_1.pragma_line), ts_1.node_id.split(":")[0]) except IndexError: continue # get potential children by dfs enumerating children nodes inside cur_potential_parent_functions scope - potential_children = __get_potential_children_of_function( - pet, cur_potential_parent_function - ) + potential_children = __get_potential_children_of_function(pet, cur_potential_parent_function) cppf_recursive_function_calls = __get_recursive_calls_from_function( potential_children, cur_potential_parent_function ) outer_breaker = False - for rfce_idx_1, recursive_function_call_entry_1 in enumerate( - cppf_recursive_function_calls - ): + for rfce_idx_1, recursive_function_call_entry_1 in enumerate(cppf_recursive_function_calls): if outer_breaker: break # 3. get R/W information for cf's parameters based on CUInstResult.txt - called_function_name_1, call_line_1 = recursive_function_call_entry_1.split(",")[ - 0 - ].split(" ") + called_function_name_1, call_line_1 = recursive_function_call_entry_1.split(",")[0].split(" ") lower_line_num_1 = ts_1.pragma_line if ":" in lower_line_num_1: lower_line_num_1 = lower_line_num_1.split(":")[1] @@ -609,17 +581,13 @@ def identify_dependencies_for_same_functions( function_raw_information_cache, ) = ret_val_1 # 4. iterate over successive calls to cf, named scf - for rfce_idx_2, recursive_function_call_entry_2 in enumerate( - cppf_recursive_function_calls - ): + for rfce_idx_2, recursive_function_call_entry_2 in enumerate(cppf_recursive_function_calls): if rfce_idx_2 == rfce_idx_1: continue if recursive_function_call_entry_2 is None: continue # 5. get R/W Information for scf - called_function_name_2, call_line_2 = recursive_function_call_entry_2.split( - "," - )[0].split(" ") + called_function_name_2, call_line_2 = recursive_function_call_entry_2.split(",")[0].split(" ") lower_line_num_2 = ts_1.pragma_line if ":" in lower_line_num_2: lower_line_num_2 = lower_line_num_2.split(":")[1] @@ -669,9 +637,7 @@ def identify_dependencies_for_same_functions( continue # 6.3 If intersecting parameter of cf is RAW, add dependency (scf:in, cf:out) - for intersection_var, is_pessimistic in [ - (e[0], e[2]) for e in intersection if e[1] - ]: + for intersection_var, is_pessimistic in [(e[0], e[2]) for e in intersection if e[1]]: if ts_1 not in out_dep_updates: out_dep_updates[ts_1] = [] out_dep_updates[ts_1].append((intersection_var, is_pessimistic)) @@ -680,9 +646,7 @@ def identify_dependencies_for_same_functions( in_dep_updates[ts_2].append((intersection_var, is_pessimistic)) outer_breaker = True # perform updates of in and out dependencies - return __perform_dependency_updates( - task_suggestions, in_dep_updates, out_dep_updates, result_suggestions - ) + return __perform_dependency_updates(task_suggestions, in_dep_updates, out_dep_updates, result_suggestions) def __perform_dependency_updates( @@ -733,7 +697,7 @@ def get_alias_for_parameter_at_position( parameter_position: int, source_code_files: Dict[str, str], visited: List[Tuple[Node, int]], - called_function_cache: Dict, + called_function_cache: Dict[Any, Any], ) -> List[Tuple[str, str, LineID, LineID]]: """Returns alias information for a parameter at a specific position. :param pet: PET Graph @@ -760,9 +724,7 @@ def get_alias_for_parameter_at_position( # read line from source code (iterate over lines of CU to search for function call) for line in range(cu.start_line, cu.end_line + 1): try: - source_code_line = get_function_call_from_source_code( - source_code_files, line, cu.id.split(":")[0] - ) + source_code_line = get_function_call_from_source_code(source_code_files, line, cu.id.split(":")[0]) except IndexError: continue # get parameter names from call @@ -791,8 +753,8 @@ def get_alias_for_parameter_at_position( def check_dependence_of_task_pair( - aliases: Dict, - raw_dependency_information: Dict, + aliases: Dict[TaskParallelismInfo, List[List[Tuple[str, str, LineID, LineID]]]], + raw_dependency_information: Dict[str, List[Tuple[str, str]]], task_suggestion_1: TaskParallelismInfo, param_names_1: List[Optional[str]], task_suggestion_2: TaskParallelismInfo, @@ -849,10 +811,7 @@ def check_dependence_of_task_pair( dependencies.append(parameter) dependencies = list(set(dependencies)) # check if task suggestion_1 occurs prior to task_suggestion_2 - if ( - task_suggestion_1._node.start_position().split(":")[0] - == task_suggestion_2._node.start_position().split(":")[0] - ): + if task_suggestion_1._node.start_position().split(":")[0] == task_suggestion_2._node.start_position().split(":")[0]: # same file id ts_1_pragma_line = ( int(task_suggestion_1.pragma_line) @@ -884,9 +843,7 @@ def get_function_call_parameter_rw_information( function_parameter_alias_dict: Dict[str, List[Tuple[str, str]]], called_cu_id: Optional[str] = None, called_function_name: Optional[str] = None, -) -> Optional[ - Tuple[str, List[Tuple[str, bool, bool]], List[Node], Dict[str, List[Tuple[bool, bool]]]] -]: +) -> Optional[Tuple[str, List[Tuple[str, bool, bool]], List[Node], Dict[str, List[Tuple[bool, bool]]]]]: """Retrieves the call_position and information whether the parameters of the target function are modified within the respective function, based on the contents of cu_inst_result_dict. Either called_cu_id, called_function_name or both need to be set. @@ -1025,15 +982,11 @@ def get_function_call_parameter_rw_information( function_parameter_alias_dict, source_code_files, ) - function_raw_information_cache[ - called_function_cu.name - ] = res_called_function_raw_information + function_raw_information_cache[called_function_cu.name] = res_called_function_raw_information else: # read cache if called_function_cu.name in function_raw_information_cache: - res_called_function_raw_information = function_raw_information_cache[ - called_function_cu.name - ] + res_called_function_raw_information = function_raw_information_cache[called_function_cu.name] # 5.5 match parameter_names with gathered R/W information of argument positions if len(raw_reported_for_param_positions) != len(parameter_names): @@ -1043,10 +996,7 @@ def get_function_call_parameter_rw_information( for idx in range(0, len(parameter_names)): tmp = ( parameter_names[idx], - ( - raw_reported_for_param_positions[idx][0] - or res_called_function_raw_information[idx][0] - ), + (raw_reported_for_param_positions[idx][0] or res_called_function_raw_information[idx][0]), res_called_function_raw_information[idx][1], ) if tmp[0] is None: @@ -1125,11 +1075,7 @@ def get_function_call_parameter_rw_information_recursion_step( if cur_potential_child not in potential_children: potential_children.append(cur_potential_child) for tmp_child in pet.direct_children_or_called_nodes(cur_potential_child): - if ( - tmp_child not in queue_1 - and tmp_child not in potential_children - and tmp_child not in visited - ): + if tmp_child not in queue_1 and tmp_child not in potential_children and tmp_child not in visited: queue_1.append(tmp_child) called_function_args_raw_information = [] @@ -1180,9 +1126,7 @@ def get_function_call_parameter_rw_information_recursion_step( child_raw_info, child_is_pessimistic, ) in parameter_names_raw_information: - for idx, (var_name, raw_info, is_pessimistic) in enumerate( - called_function_args_raw_information - ): + for idx, (var_name, raw_info, is_pessimistic) in enumerate(called_function_args_raw_information): if var_name == child_var_name: called_function_args_raw_information[idx] = ( var_name, @@ -1227,17 +1171,13 @@ def get_function_call_parameter_rw_information_recursion_step( break if var_name_is_modified: # update RAW information - for idx, (old_var_name, raw_info, _) in enumerate( - called_function_args_raw_information - ): + for idx, (old_var_name, raw_info, _) in enumerate(called_function_args_raw_information): if old_var_name == var_name: if not raw_info: called_function_args_raw_information[idx] = (old_var_name, True, True) # second True denotes the pessimistic nature of a potential created dependency # remove names from called_function_args_raw_information - called_function_args_raw_information_bools = [ - (e[1], e[2]) for e in called_function_args_raw_information - ] + called_function_args_raw_information_bools = [(e[1], e[2]) for e in called_function_args_raw_information] return ( recursively_visited, called_function_cu.name, diff --git a/discopop_explorer/pattern_detectors/task_parallelism/suggesters/tasks.py b/discopop_explorer/pattern_detectors/task_parallelism/suggesters/tasks.py index 9afee54c2..64888b1e2 100644 --- a/discopop_explorer/pattern_detectors/task_parallelism/suggesters/tasks.py +++ b/discopop_explorer/pattern_detectors/task_parallelism/suggesters/tasks.py @@ -13,7 +13,6 @@ FunctionNode, LoopNode, MWType, - NodeType, EdgeType, Node, PETGraphX, @@ -74,9 +73,7 @@ def detect_task_suggestions(pet: PETGraphX) -> List[PatternInfo]: dep_line_number = dep_line[dep_line.index(":") + 1 :] if dep_line_number < first_dependency_line_number: first_dependency_line = dep_line - tmp_suggestion = TaskParallelismInfo( - v, TPIType.TASKWAIT, ["taskwait"], first_dependency_line, [], [], [] - ) + tmp_suggestion = TaskParallelismInfo(v, TPIType.TASKWAIT, ["taskwait"], first_dependency_line, [], [], []) if v.start_position() not in suggestions: # no entry for source code line contained in suggestions suggestions[v.start_position()] = [] @@ -93,9 +90,7 @@ def detect_task_suggestions(pet: PETGraphX) -> List[PatternInfo]: function_call_string = vx.recursive_function_calls[i] if not type(function_call_string) == str: continue - contained_in = recursive_function_call_contained_in_worker_cu( - function_call_string, worker_cus - ) + contained_in = recursive_function_call_contained_in_worker_cu(function_call_string, worker_cus) if contained_in is not None: current_suggestions = None # recursive Function call contained in worker cu @@ -146,9 +141,7 @@ def detect_task_suggestions(pet: PETGraphX) -> List[PatternInfo]: return result -def correct_task_suggestions_in_loop_body( - pet: PETGraphX, suggestions: List[PatternInfo] -) -> List[PatternInfo]: +def correct_task_suggestions_in_loop_body(pet: PETGraphX, suggestions: List[PatternInfo]) -> List[PatternInfo]: """Separate treatment of task suggestions at loop increment CUs. If regular loop: move taskwait suggested at loop increment line to end of loop body. If do-all loop: move taskwait suggested at loop increment line outside of loop body. @@ -159,19 +152,13 @@ def correct_task_suggestions_in_loop_body( :param pet: PET graph :param suggestions: Found suggestions :return: Updated suggestions""" - task_suggestions = [ - s - for s in [e for e in suggestions if type(e) == TaskParallelismInfo] - if s.type is TPIType.TASK - ] + task_suggestions = [s for s in [e for e in suggestions if type(e) == TaskParallelismInfo] if s.type is TPIType.TASK] for ts in task_suggestions: found_critical_cus: List[Node] = [] found_atomic_cus: List[Node] = [] for loop_cu in pet.all_nodes(LoopNode): # check if task suggestion inside do-all loop exists - if line_contained_in_region( - ts._node.start_position(), loop_cu.start_position(), loop_cu.end_position() - ): + if line_contained_in_region(ts._node.start_position(), loop_cu.start_position(), loop_cu.end_position()): def find_taskwaits(cu_node: Node, visited: List[Node]): if cu_node.tp_contains_taskwait: @@ -243,36 +230,22 @@ def find_taskwaits(cu_node: Node, visited: List[Node]): # containing such cases for loop_cu_child in pet.direct_children_or_called_nodes(loop_cu): for in_dep_var_name in list( - set( - [ - e[2].var_name - for e in pet.in_edges(loop_cu_child.id, EdgeType.DATA) - ] - ) + set([e[2].var_name for e in pet.in_edges(loop_cu_child.id, EdgeType.DATA)]) ): if in_dep_var_name in ts.shared: # check if the found dependency occurs in the scope of the suggested task if ( loop_cu_child.file_id == ts._node.file_id - and loop_cu_child.start_line - >= int(ts.region_start_line) + and loop_cu_child.start_line >= int(ts.region_start_line) and loop_cu_child.end_line <= int(ts.region_end_line) ): # seperate between critical and atomic CUs if contains_reduction(pet, loop_cu_child): # split cu lines on reduction, mark surrounding lines as critical - file_idx = loop_cu_child.start_position().split( - ":" - )[0] - start_line = int( - loop_cu_child.start_position().split(":")[1] - ) - end_line = int( - loop_cu_child.end_position().split(":")[1] - ) - critical_lines_range = range( - start_line, end_line + 1 - ) + file_idx = loop_cu_child.start_position().split(":")[0] + start_line = int(loop_cu_child.start_position().split(":")[1]) + end_line = int(loop_cu_child.end_position().split(":")[1]) + critical_lines_range = range(start_line, end_line + 1) atomic_lines = [] # seperate between critical and atomic lines for red_var in pet.reduction_vars: @@ -282,46 +255,29 @@ def find_taskwaits(cu_node: Node, visited: List[Node]): loop_cu_child.end_position(), ): atomic_lines.append( - int( - red_var["reduction_line"].split( - ":" - )[1] - ) + int(red_var["reduction_line"].split(":")[1]) ) critical_lines = [ - e - for e in critical_lines_range - if e not in atomic_lines + e for e in critical_lines_range if e not in atomic_lines ] # combine successive critical lines if possible combined_critical_lines = [ (e, e) for e in critical_lines ] # (start, end) found_combination = True - while ( - found_combination - and len(combined_critical_lines) > 1 - ): + while found_combination and len(combined_critical_lines) > 1: found_combination = False - for outer_idx, outer in enumerate( - combined_critical_lines - ): - for inner_idx, inner in enumerate( - combined_critical_lines - ): + for outer_idx, outer in enumerate(combined_critical_lines): + for inner_idx, inner in enumerate(combined_critical_lines): if inner_idx == outer_idx: continue if outer[1] + 1 == inner[0]: # inner is direct successor of outer - combined_critical_lines[ - outer_idx - ] = ( + combined_critical_lines[outer_idx] = ( outer[0], inner[1], ) - combined_critical_lines.pop( - inner_idx - ) + combined_critical_lines.pop(inner_idx) found_combination = True break if found_combination: @@ -360,7 +316,7 @@ def find_taskwaits(cu_node: Node, visited: List[Node]): def __identify_atomic_or_critical_sections( - pet: PETGraphX, ts: TaskParallelismInfo, found_cus: List, selector: bool + pet: PETGraphX, ts: TaskParallelismInfo, found_cus: List[Node], selector: bool ): """Identifies and marks atomic or critical sections. :param pet: PET Graph @@ -385,9 +341,7 @@ def __identify_atomic_or_critical_sections( break if parent_idx == child_idx: continue - if combinations[child_idx][0] in pet.direct_successors( - combinations[parent_idx][-1] - ): + if combinations[child_idx][0] in pet.direct_successors(combinations[parent_idx][-1]): combinations[parent_idx] += combinations[child_idx] combinations.pop(child_idx) found_combination = True diff --git a/discopop_explorer/pattern_detectors/task_parallelism/task_parallelism_detector.py b/discopop_explorer/pattern_detectors/task_parallelism/task_parallelism_detector.py index f529f754c..2bf13f596 100644 --- a/discopop_explorer/pattern_detectors/task_parallelism/task_parallelism_detector.py +++ b/discopop_explorer/pattern_detectors/task_parallelism/task_parallelism_detector.py @@ -9,7 +9,7 @@ from typing import List, Optional, cast -from discopop_explorer.PETGraphX import DummyNode, PETGraphX, NodeType, MWType +from discopop_explorer.PETGraphX import DummyNode, PETGraphX, MWType from discopop_explorer.parser import parse_inputs from discopop_explorer.pattern_detectors.PatternInfo import PatternInfo from discopop_explorer.pattern_detectors.do_all_detector import run_detection as detect_do_all @@ -33,6 +33,12 @@ cu_xml_preprocessing, check_loop_scopes, ) +from discopop_explorer.pattern_detectors.task_parallelism.suggesters.auxiliary import ( + suggest_parallel_regions, + set_task_contained_lines, + detect_taskloop_reduction, + combine_omittable_cus, +) from discopop_explorer.pattern_detectors.task_parallelism.suggesters.barriers import ( detect_barrier_suggestions, suggest_barriers_for_uncovered_tasks_before_return, @@ -49,12 +55,6 @@ detect_task_suggestions, correct_task_suggestions_in_loop_body, ) -from discopop_explorer.pattern_detectors.task_parallelism.suggesters.auxiliary import ( - suggest_parallel_regions, - set_task_contained_lines, - detect_taskloop_reduction, - combine_omittable_cus, -) from discopop_explorer.pattern_detectors.task_parallelism.tp_utils import ( create_task_tree, __forks, @@ -160,20 +160,14 @@ def run_detection( for fork in fs: if fork.child_tasks: - result.append( - TaskParallelismInfo(fork.nodes[0], TPIType.DUMMY, ["dummy_fork"], [], [], [], []) - ) + result.append(TaskParallelismInfo(fork.nodes[0], TPIType.DUMMY, ["dummy_fork"], [], [], [], [])) # Preprocessing check_loop_scopes(pet) # Suggestion generation result += detect_task_suggestions(pet) result += suggest_parallel_regions(pet, cast(List[TaskParallelismInfo], result)) - result = cast( - List[PatternInfo], set_task_contained_lines(cast(List[TaskParallelismInfo], result)) - ) - result = cast( - List[PatternInfo], detect_taskloop_reduction(pet, cast(List[TaskParallelismInfo], result)) - ) + result = cast(List[PatternInfo], set_task_contained_lines(cast(List[TaskParallelismInfo], result))) + result = cast(List[PatternInfo], detect_taskloop_reduction(pet, cast(List[TaskParallelismInfo], result))) result = cast( List[PatternInfo], remove_useless_barrier_suggestions(pet, cast(List[TaskParallelismInfo], result)), diff --git a/discopop_explorer/pattern_detectors/task_parallelism/tp_utils.py b/discopop_explorer/pattern_detectors/task_parallelism/tp_utils.py index 80ab115bb..8f30a96e8 100644 --- a/discopop_explorer/pattern_detectors/task_parallelism/tp_utils.py +++ b/discopop_explorer/pattern_detectors/task_parallelism/tp_utils.py @@ -7,9 +7,10 @@ # directory for details. import subprocess -from typing import Union, cast, IO, Dict, List, Tuple, Optional +from typing import Union, cast, Dict, List, Tuple, Optional, Any from lxml import objectify # type: ignore + from discopop_explorer.PETGraphX import ( DummyNode, FunctionNode, @@ -56,10 +57,7 @@ def demangle(mangled_name: str) -> str: return out except FileNotFoundError: raise ValueError( - "Executable '" - + llvm_cxxfilt_path - + "' not found." - + " Check or supply --llvm-cxxfilt-path parameter." + "Executable '" + llvm_cxxfilt_path + "' not found." + " Check or supply --llvm-cxxfilt-path parameter." ) raise ValueError("Demangling of " + mangled_name + " not possible!") @@ -108,9 +106,7 @@ def get_parent_of_type( last_node = cur_node visited.append(cur_node) tmp_list = [ - (s, t, e) - for s, t, e in pet.in_edges(cur_node.id) - if pet.node_at(s) not in visited and e.etype == edge_type + (s, t, e) for s, t, e in pet.in_edges(cur_node.id) if pet.node_at(s) not in visited and e.etype == edge_type ] for e in tmp_list: if pet.node_at(e[0]).type == parent_type: @@ -142,9 +138,7 @@ def get_cus_inside_function(pet: PETGraphX, function_cu: Node) -> List[Node]: return result_list -def check_reachability( - pet: PETGraphX, target: Node, source: Node, edge_types: List[EdgeType] -) -> bool: +def check_reachability(pet: PETGraphX, target: Node, source: Node, edge_types: List[EdgeType]) -> bool: """check if target is reachable from source via edges of types edge_type. :param pet: PET graph :param source: CUNode @@ -158,11 +152,7 @@ def check_reachability( while len(queue) > 0: cur_node = queue.pop(0) visited.append(cur_node.id) - tmp_list = [ - (s, t, e) - for s, t, e in pet.in_edges(cur_node.id) - if s not in visited and e.etype in edge_types - ] + tmp_list = [(s, t, e) for s, t, e in pet.in_edges(cur_node.id) if s not in visited and e.etype in edge_types] for e in tmp_list: if pet.node_at(e[0]) == source: return True @@ -172,9 +162,7 @@ def check_reachability( return False -def get_predecessor_nodes( - pet: PETGraphX, root: Node, visited_nodes: List[Node] -) -> Tuple[List[Node], List[Node]]: +def get_predecessor_nodes(pet: PETGraphX, root: Node, visited_nodes: List[Node]) -> Tuple[List[Node], List[Node]]: """return a list of reachable predecessor nodes. generate list recursively. stop recursion if a node of type "function" is found or root is a barrier @@ -192,9 +180,7 @@ def get_predecessor_nodes( in_succ_edges = [ (s, t, e) for s, t, e in pet.in_edges(root.id) - if e.etype == EdgeType.SUCCESSOR - and pet.node_at(s) != root - and pet.node_at(s) not in visited_nodes + if e.etype == EdgeType.SUCCESSOR and pet.node_at(s) != root and pet.node_at(s) not in visited_nodes ] for e in in_succ_edges: tmp, visited_nodes = get_predecessor_nodes(pet, pet.node_at(e[0]), visited_nodes) @@ -295,9 +281,7 @@ def create_task_tree_helper(pet: PETGraphX, current: Node, root: Task, visited_f create_task_tree_helper(pet, child, root, visited_func) -def recursive_function_call_contained_in_worker_cu( - function_call_string: str, worker_cus: List[Node] -) -> Node: +def recursive_function_call_contained_in_worker_cu(function_call_string: str, worker_cus: List[Node]) -> Node: """check if submitted function call is contained in at least one WORKER cu. Returns the vertex identifier of the containing cu. If no cu contains the function call, None is returned. @@ -318,9 +302,7 @@ def recursive_function_call_contained_in_worker_cu( # function_call_string looks now like like: 'fib 7:52' # split String into function_name. file_id and line_number - file_id = function_call_string[ - function_call_string.index(" ") + 1 : function_call_string.index(":") - ] + file_id = function_call_string[function_call_string.index(" ") + 1 : function_call_string.index(":")] line_number = function_call_string[function_call_string.index(":") + 1 :] # get tightest surrounding cu @@ -333,9 +315,7 @@ def recursive_function_call_contained_in_worker_cu( # check if file_id is equal if file_id == cur_w_file_id: # trim to line numbers only - cur_w_starts_at_line = LineID( - cur_w_starts_at_line[cur_w_starts_at_line.index(":") + 1 :] - ) + cur_w_starts_at_line = LineID(cur_w_starts_at_line[cur_w_starts_at_line.index(":") + 1 :]) cur_w_ends_at_line = LineID(cur_w_ends_at_line[cur_w_ends_at_line.index(":") + 1 :]) # check if line_number is contained if int(cur_w_starts_at_line) <= int(line_number) <= int(cur_w_ends_at_line): @@ -446,11 +426,7 @@ def __get_word_prior_to_bracket(string): # if word prior to ( is "while", "for" or "if", cut away until ( word_prior_to_bracket = __get_word_prior_to_bracket(function_call_string) if word_prior_to_bracket is not None: - if ( - word_prior_to_bracket == "while" - or word_prior_to_bracket == "for" - or word_prior_to_bracket == "if" - ): + if word_prior_to_bracket == "while" or word_prior_to_bracket == "for" or word_prior_to_bracket == "if": function_call_string = function_call_string[function_call_string.index("(") + 1 :] # check if called_function_name is contained in function_call_string if not called_function_name_contained: @@ -463,9 +439,7 @@ def __get_word_prior_to_bracket(string): # if called_function_name is set and contained more than once in function_call_string, split function_call_string if called_function_name is not None: while function_call_string.count(called_function_name) > 1: - function_call_string = function_call_string[ - : function_call_string.rfind(called_function_name) - ] + function_call_string = function_call_string[: function_call_string.rfind(called_function_name)] return function_call_string @@ -516,29 +490,16 @@ def get_called_function_and_parameter_names_from_function_call( result_parameters: List[Optional[str]] = [] for param in parameters: param = param.replace("\t", "") - if ( - "+" in param - or "-" in param - or "*" in param - or "/" in param - or "(" in param - or ")" in param - ): + if "+" in param or "-" in param or "*" in param or "/" in param or "(" in param or ")" in param: param_expression = param.replace("+", "$$").replace("-", "$$").replace("/", "$$") - param_expression = ( - param_expression.replace("*", "$$").replace("(", "$$").replace(")", "$$") - ) + param_expression = param_expression.replace("*", "$$").replace("(", "$$").replace(")", "$$") split_param_expression = param_expression.split("$$") split_param_expression = [ex.replace(" ", "") for ex in split_param_expression] # check if any of the parameters is in list of known variables split_param_expression = [ ex for ex in split_param_expression - if ex - in [ - var.replace(".addr", "") - for var in [v.name for v in node.local_vars + node.global_vars] - ] + if ex in [var.replace(".addr", "") for var in [v.name for v in node.local_vars + node.global_vars]] ] # check if type of any of them contains * (i.e. is a pointer) found_entry = False @@ -568,7 +529,7 @@ def set_global_llvm_cxxfilt_path(value: str): def get_called_functions_recursively( - pet: PETGraphX, root: Node, visited: List[Node], cache: Dict + pet: PETGraphX, root: Node, visited: List[Node], cache: Dict[Any, Any] ) -> List[Union[FunctionNode, DummyNode]]: """returns a recursively generated list of called functions, started at root.""" visited.append(root) @@ -600,9 +561,7 @@ def contains_reduction(pet: PETGraphX, node: Node) -> bool: :param node: CUNode :return: bool""" for red_var in pet.reduction_vars: - if line_contained_in_region( - red_var["reduction_line"], node.start_position(), node.end_position() - ): + if line_contained_in_region(red_var["reduction_line"], node.start_position(), node.end_position()): return True return False diff --git a/discopop_explorer/plugins/pipeline.py b/discopop_explorer/plugins/pipeline.py index bc4b073eb..6b3549e02 100644 --- a/discopop_explorer/plugins/pipeline.py +++ b/discopop_explorer/plugins/pipeline.py @@ -9,7 +9,7 @@ from copy import deepcopy from typing import List -from ..PETGraphX import LineID, LoopNode, PETGraphX, NodeType, Node, EdgeType +from ..PETGraphX import LineID, LoopNode, PETGraphX, Node, EdgeType from ..utils import correlation_coefficient total = 0 @@ -133,9 +133,7 @@ def get_matrix(pet: PETGraphX, root: Node, loop_subnodes: List[Node]) -> List[Li for i in range(0, len(loop_subnodes)): res.append([]) for j in range(0, len(loop_subnodes)): - res[i].append( - int(pet.depends_ignore_readonly(loop_subnodes[i], loop_subnodes[j], root)) - ) + res[i].append(int(pet.depends_ignore_readonly(loop_subnodes[i], loop_subnodes[j], root))) return res diff --git a/discopop_explorer/utils.py b/discopop_explorer/utils.py index 6a9d64bc8..0b8bfada3 100644 --- a/discopop_explorer/utils.py +++ b/discopop_explorer/utils.py @@ -5,8 +5,6 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -import sys -import time import itertools from typing import List, Sequence, Set, Dict, Tuple, cast @@ -98,9 +96,7 @@ def is_loop_index2(pet: PETGraphX, root_loop: Node, var_name: str) -> bool: # NOTE: left old code as it may become relevant again in the near future # We decided to omit the information that computes the workload and the relevant codes. For large programs (e.g., ffmpeg), the generated Data.xml file becomes very large. However, we keep the code here because we would like to integrate a hotspot detection algorithm (TODO: Bertin) with the parallelism discovery. Then, we need to retrieve the information to decide which code sections (loops or functions) are worth parallelizing. -def calculate_workload( - pet: PETGraphX, node: Node, ignore_function_calls_and_cached_values: bool = False -) -> int: +def calculate_workload(pet: PETGraphX, node: Node, ignore_function_calls_and_cached_values: bool = False) -> int: """Calculates and stores the workload for a given node The workload is the number of instructions multiplied by respective number of iterations @@ -236,11 +232,7 @@ def __get_dep_of_type( """ return [ e - for e in ( - pet.in_edges(node.id, EdgeType.DATA) - if reversed - else pet.out_edges(node.id, EdgeType.DATA) - ) + for e in (pet.in_edges(node.id, EdgeType.DATA) if reversed else pet.out_edges(node.id, EdgeType.DATA)) if e[2].dtype == dep_type ] @@ -273,9 +265,7 @@ def is_reduction_var(line: LineID, name: str, reduction_vars: List[Dict[str, str return any(rv for rv in reduction_vars if rv["loop_line"] == line and rv["name"] == name) -def is_reduction_any( - possible_lines: List[LineID], var_name: str, reduction_vars: List[Dict[str, str]] -) -> bool: +def is_reduction_any(possible_lines: List[LineID], var_name: str, reduction_vars: List[Dict[str, str]]) -> bool: """Determines, whether or not the given variable is reduction variable :param possible_lines: possible loop line number @@ -502,10 +492,7 @@ def is_depend_in_out( """ for in_dep in in_deps: for out_dep in out_deps: - if ( - in_dep[2].memory_region in mem_regs - and in_dep[2].memory_region == out_dep[2].memory_region - ): + if in_dep[2].memory_region in mem_regs and in_dep[2].memory_region == out_dep[2].memory_region: return True return False @@ -610,9 +597,7 @@ def get_child_loops(pet: PETGraphX, node: Node) -> Tuple[List[Node], List[Node]] return do_all, reduction -def get_initialized_memory_regions_in( - pet: PETGraphX, cu_nodes: List[CUNode] -) -> Dict[Variable, Set[MemoryRegion]]: +def get_initialized_memory_regions_in(pet: PETGraphX, cu_nodes: List[CUNode]) -> Dict[Variable, Set[MemoryRegion]]: initialized_memory_regions: Dict[Variable, Set[MemoryRegion]] = dict() for cu in cu_nodes: for s, t, d in pet.out_edges(cu.id, EdgeType.DATA): @@ -661,9 +646,7 @@ def classify_loop_variables( # only consider memory regions which are know at the current code location. # ignore memory regions which stem from called functions. - left_subtree_without_called_nodes = pet.get_left_right_subtree( - loop, False, ignore_called_nodes=True - ) + left_subtree_without_called_nodes = pet.get_left_right_subtree(loop, False, ignore_called_nodes=True) prior_known_vars = pet.get_variables(left_subtree_without_called_nodes) prior_known_mem_regs = set() for pkv in prior_known_vars: @@ -692,11 +675,7 @@ def classify_loop_variables( elif loop.reduction and pet.is_reduction_var(loop.start_position(), var.name): var.operation = pet.get_reduction_sign(loop.start_position(), var.name) reduction.append(var) - elif ( - is_written_in_subtree(vars[var], raw, waw, lst) - or is_func_arg(pet, var.name, loop) - and is_scalar_val(var) - ): + elif is_written_in_subtree(vars[var], raw, waw, lst) or is_func_arg(pet, var.name, loop) and is_scalar_val(var): if is_readonly(vars[var], war, waw, rev_raw): if is_global(var.name, sub): shared.append(var) @@ -742,15 +721,7 @@ def classify_task_vars( in_deps: List[Tuple[NodeID, NodeID, Dependency]], out_deps: List[Tuple[NodeID, NodeID, Dependency]], used_in_task_parallelism_detection=False, -) -> Tuple[ - List[Variable], - List[Variable], - List[Variable], - List[Variable], - List[Variable], - List[Variable], - List[str], -]: +) -> Tuple[List[Variable], List[Variable], List[Variable], List[Variable], List[Variable], List[Variable], List[str],]: """Classify task variables :param pet: CU graph @@ -868,8 +839,7 @@ def classify_task_vars( subtree, ): if is_scalar_val(var) and ( - not used_in_task_parallelism_detection - or not __is_written_prior_to_task(pet, var, task) + not used_in_task_parallelism_detection or not __is_written_prior_to_task(pet, var, task) ): if is_read_in( vars[var], @@ -886,9 +856,7 @@ def classify_task_vars( shared.append((var, vars[var])) # use known variables to reconstruct the correct variable names from the classified memory regions - left_subtree_without_called_nodes = pet.get_left_right_subtree( - task, False, ignore_called_nodes=True - ) + left_subtree_without_called_nodes = pet.get_left_right_subtree(task, False, ignore_called_nodes=True) prior_known_vars = pet.get_variables(left_subtree_without_called_nodes) return ( @@ -909,11 +877,7 @@ def __apply_dealiasing( tmp_memory_regions = set() for _, mem_regs in input_list: tmp_memory_regions.update(mem_regs) - cleaned = [ - pkv - for pkv in previously_known - if len(previously_known[pkv].intersection(tmp_memory_regions)) - ] + cleaned = [pkv for pkv in previously_known if len(previously_known[pkv].intersection(tmp_memory_regions))] return cleaned diff --git a/discopop_explorer/variable.py b/discopop_explorer/variable.py index ce43e5f8e..e93ebf390 100644 --- a/discopop_explorer/variable.py +++ b/discopop_explorer/variable.py @@ -5,7 +5,6 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -import json from typing import Optional diff --git a/discopop_library/CodeGenerator/__main__.py b/discopop_library/CodeGenerator/__main__.py index 80133dc84..826cf8d52 100644 --- a/discopop_library/CodeGenerator/__main__.py +++ b/discopop_library/CodeGenerator/__main__.py @@ -31,8 +31,8 @@ from discopop_library.CodeGenerator.CodeGenerator import ( from_json_strings as generate_code_from_json_strings, ) -from discopop_library.PathManagement.PathManagement import load_file_mapping, get_path from discopop_library.JSONHandler.JSONHandler import read_patterns_from_json_to_json +from discopop_library.PathManagement.PathManagement import load_file_mapping, get_path docopt_schema = Schema( { @@ -63,18 +63,12 @@ def main(): relevant_patterns: List[str] = ( [] if arguments["--patterns"] == "None" - else ( - arguments["--patterns"].split(",") - if "," in arguments["--patterns"] - else [arguments["--patterns"]] - ) + else (arguments["--patterns"].split(",") if "," in arguments["--patterns"] else [arguments["--patterns"]]) ) # validate patterns for pattern in relevant_patterns: if pattern not in ["reduction", "do_all", "simple_gpu", "combined_gpu"]: - raise ValueError( - "Unsupported pattern name: ", pattern, " given in '--patterns' argument!" - ) + raise ValueError("Unsupported pattern name: ", pattern, " given in '--patterns' argument!") for file in [file_mapping_file, json_file]: if not os.path.isfile(file): diff --git a/discopop_library/CodeGenerator/classes/ContentBuffer.py b/discopop_library/CodeGenerator/classes/ContentBuffer.py index 3811f0c33..a49831fa7 100644 --- a/discopop_library/CodeGenerator/classes/ContentBuffer.py +++ b/discopop_library/CodeGenerator/classes/ContentBuffer.py @@ -11,9 +11,8 @@ import re import shlex import subprocess -import sys from pathlib import Path -from typing import List, Dict, Sequence, Any, Optional +from typing import List, Dict, Any, Optional from discopop_library.CodeGenerator.classes.Enums import PragmaPosition from discopop_library.CodeGenerator.classes.Line import Line @@ -93,9 +92,7 @@ def add_pragma( Returns False if compilation of the modified code was not possible. In this case, no changes will be applied to the ContentBuffer Object.""" if pragma.start_line is None or pragma.end_line is None: - raise ValueError( - "Unsupported start or end line: ", pragma.start_line, "-", pragma.end_line - ) + raise ValueError("Unsupported start or end line: ", pragma.start_line, "-", pragma.end_line) if pragma.file_id is None: raise ValueError("Unsupported file_id: ", pragma.file_id) if pragma.file_id != self.file_id: @@ -135,18 +132,14 @@ def add_pragma( # update belonging information of contained lines tmp_start_line = ( - pragma.start_line - if pragma.pragma_position == PragmaPosition.BEFORE_START - else pragma.start_line + 1 + pragma.start_line if pragma.pragma_position == PragmaPosition.BEFORE_START else pragma.start_line + 1 ) tmp_end_line = pragma.end_line + 1 for line_num in range(tmp_start_line, tmp_end_line): for line in self.lines: if line.line_num == line_num: line.belongs_to_regions += [ - n - for n in pragma_line.belongs_to_regions - if n not in line.belongs_to_regions + n for n in pragma_line.belongs_to_regions if n not in line.belongs_to_regions ] # append children to lines (mark as contained in region) @@ -247,9 +240,7 @@ def add_pragma( self.max_line_num = backup_max_line_num self.compile_result_buffer += result.stdout.decode("utf-8") + "\n" self.compile_result_buffer += result.stderr.decode("utf-8") + "\n" - self.compile_result_buffer += ( - "==> Skipped pragma insertion due to potential compilation errors!\n" - ) + self.compile_result_buffer += "==> Skipped pragma insertion due to potential compilation errors!\n" print(self.compile_result_buffer) return False return True diff --git a/discopop_library/CodeGenerator/classes/Pragma.py b/discopop_library/CodeGenerator/classes/Pragma.py index 0e2e2c9bd..973475021 100644 --- a/discopop_library/CodeGenerator/classes/Pragma.py +++ b/discopop_library/CodeGenerator/classes/Pragma.py @@ -21,6 +21,4 @@ class Pragma(object): children: List[Any] = [] def __init__(self): - self.children = ( - [] - ) # create individual list for each Pragma object to prevent bugs due to mutability + self.children = [] # create individual list for each Pragma object to prevent bugs due to mutability diff --git a/discopop_library/CodeGenerator/classes/UnpackedSuggestion.py b/discopop_library/CodeGenerator/classes/UnpackedSuggestion.py index f5dca63e9..20fe5fbbe 100644 --- a/discopop_library/CodeGenerator/classes/UnpackedSuggestion.py +++ b/discopop_library/CodeGenerator/classes/UnpackedSuggestion.py @@ -6,7 +6,7 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from typing import Dict, List +from typing import Dict, List, Any from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Enums import ( EntryPointType, @@ -24,12 +24,12 @@ class UnpackedSuggestion(object): type: str - values: dict + values: Dict[str, Any] file_id: int start_line: int end_line: int - def __init__(self, type_str: str, values: dict): + def __init__(self, type_str: str, values: Dict[str, Any]): self.type = type_str self.values = values @@ -153,9 +153,7 @@ def __get_do_all_and_reduction_pragmas(self, is_gpu_pragma: bool) -> List[Pragma reductions_dict[red_type] = [] reductions_dict[red_type].append(var) for red_type in reductions_dict: - pragma.pragma_str += ( - "reduction(" + red_type + ":" + ",".join(reductions_dict[red_type]) + ") " - ) + pragma.pragma_str += "reduction(" + red_type + ":" + ",".join(reductions_dict[red_type]) + ") " pragma.file_id = self.file_id pragma.start_line = self.start_line @@ -188,9 +186,7 @@ def __get_pipeline_pragmas(self) -> List[Pragma]: reductions_dict[red_type] = [] reductions_dict[red_type].append(var) for red_type in reductions_dict: - pragma.pragma_str += ( - "reduction(" + red_type + ":" + ",".join(reductions_dict[red_type]) + ") " - ) + pragma.pragma_str += "reduction(" + red_type + ":" + ",".join(reductions_dict[red_type]) + ") " if len(stage["in_deps"]) > 0: pragma.pragma_str += "depends(in:" + ",".join(stage["in_deps"]) + ") " if len(stage["out_deps"]) > 0: @@ -238,9 +234,7 @@ def __get_simple_gpu_pragmas( construct_start_line = int(construct_start.split(":")[1]) child_pragma = Pragma() if mark_invalid: - child_pragma.pragma_str = ( - "// INVALID - MISSING POSITION:: " + construct["name"] + " " - ) + child_pragma.pragma_str = "// INVALID - MISSING POSITION:: " + construct["name"] + " " else: child_pragma.pragma_str = construct["name"] + " " if loop["collapse"] > 1: @@ -270,9 +264,7 @@ def __get_simple_gpu_pragmas( elif construct["positioning"] == OmpConstructPositioning.AFTER_LINE: child_pragma.pragma_position = PragmaPosition.AFTER_START else: - raise ValueError( - "Unsupported positioning information: ", construct["positioning"] - ) + raise ValueError("Unsupported positioning information: ", construct["positioning"]) # create pragma for visualization child_pragma.start_line = start_line child_pragma.end_line = end_line diff --git a/discopop_library/JSONHandler/JSONHandler.py b/discopop_library/JSONHandler/JSONHandler.py index 42703c273..c4728dede 100644 --- a/discopop_library/JSONHandler/JSONHandler.py +++ b/discopop_library/JSONHandler/JSONHandler.py @@ -11,9 +11,7 @@ from typing import Dict, List -def read_patterns_from_json_to_json( - json_path: str, relevant_patterns: List[str] -) -> Dict[str, List[str]]: +def read_patterns_from_json_to_json(json_path: str, relevant_patterns: List[str]) -> Dict[str, List[str]]: """relevant_patterns specifies the names of patterns which shall be returned. An empty list acts as a wildcard.""" pattern_json_strings_by_type: Dict[str, List[str]] = dict() diff --git a/discopop_library/discopop_optimizer/CostModels/CostModel.py b/discopop_library/discopop_optimizer/CostModels/CostModel.py index 6e8f81788..95206a76f 100644 --- a/discopop_library/discopop_optimizer/CostModels/CostModel.py +++ b/discopop_library/discopop_optimizer/CostModels/CostModel.py @@ -7,8 +7,6 @@ # directory for details. import random import sys -import warnings -from functools import cmp_to_key from typing import List, Dict, Tuple, Optional import numpy as np @@ -136,9 +134,7 @@ def __lt__(self, other): """Compare both models. The comparison is based on random sampling and may not be correct in all cases! """ - decision_tendency = ( - 0 # positive -> self was evaluated to be smaller more often than the other way around - ) + decision_tendency = 0 # positive -> self was evaluated to be smaller more often than the other way around decided = False counter = 0 # Sampling parameters @@ -168,15 +164,11 @@ def __lt__(self, other): normalized_pick = random.weibullvariate(alpha, beta) if self.free_symbol_distributions[symbol] == FreeSymbolDistribution.LEFT_HEAVY: # calculate sampling point using the range starting from minimum - sampling_point[symbol] = ( - range_min + (range_max - range_min) * normalized_pick - ) + sampling_point[symbol] = range_min + (range_max - range_min) * normalized_pick else: # simulate a right heavy distribution # calculate sampling point using the range starting from maximum - sampling_point[symbol] = ( - range_max - (range_max - range_min) * normalized_pick - ) + sampling_point[symbol] = range_max - (range_max - range_min) * normalized_pick # evaluate both functions at the sampling point substituted_model_1_1 = self.parallelizable_costs.xreplace(sampling_point) diff --git a/discopop_library/discopop_optimizer/CostModels/DataTransfer/DataTransferCosts.py b/discopop_library/discopop_optimizer/CostModels/DataTransfer/DataTransferCosts.py index d0049cdc8..e0693859a 100644 --- a/discopop_library/discopop_optimizer/CostModels/DataTransfer/DataTransferCosts.py +++ b/discopop_library/discopop_optimizer/CostModels/DataTransfer/DataTransferCosts.py @@ -5,7 +5,7 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from typing import Dict, List, Tuple, Set +from typing import Dict, List, Tuple import networkx as nx # type: ignore from sympy import Integer # type: ignore @@ -16,7 +16,6 @@ from discopop_library.discopop_optimizer.classes.context.ContextObjectUtils import ( get_transfer_costs, ) -from discopop_library.discopop_optimizer.classes.context.Update import Update from discopop_library.discopop_optimizer.classes.nodes.FunctionRoot import FunctionRoot diff --git a/discopop_library/discopop_optimizer/CostModels/utilities.py b/discopop_library/discopop_optimizer/CostModels/utilities.py index c14bb2c26..dd9bb9bf2 100644 --- a/discopop_library/discopop_optimizer/CostModels/utilities.py +++ b/discopop_library/discopop_optimizer/CostModels/utilities.py @@ -16,8 +16,6 @@ from discopop_library.discopop_optimizer.CostModels.CostModel import CostModel from discopop_library.discopop_optimizer.Variables.Experiment import Experiment from discopop_library.discopop_optimizer.classes.nodes.FunctionRoot import FunctionRoot -from discopop_library.discopop_optimizer.classes.nodes.GenericNode import GenericNode -from discopop_library.discopop_optimizer.classes.nodes.Loop import Loop from discopop_library.discopop_optimizer.classes.system.devices.GPU import GPU from discopop_library.discopop_optimizer.utilities.MOGUtilities import ( get_successors, @@ -55,9 +53,7 @@ def get_performance_models_for_functions( # filter out NaN - Models performance_models[node_data] = [ - model - for model in performance_models[node_data] - if model.parallelizable_costs != sympy.nan + model for model in performance_models[node_data] if model.parallelizable_costs != sympy.nan ] return performance_models @@ -195,11 +191,7 @@ def get_node_performance_models( for seq in sequential_version_ids: for option in get_out_options(graph, seq): # 2.3 - for visited_req in [ - req - for req in get_requirements(graph, option) - if req in visited_nodes - ]: + for visited_req in [req for req in get_requirements(graph, option) if req in visited_nodes]: # 2.4 if visited_req != successor: path_invalid = True @@ -235,19 +227,14 @@ def get_node_performance_models( # check if the current decision invalidates decision requirements, if some are specified if restrict_to_decisions is not None: - if not ( - successor in restrict_to_decisions - or data_at(graph, successor).suggestion is None - ): + if not (successor in restrict_to_decisions or data_at(graph, successor).suggestion is None): path_invalid = True if not path_invalid: if data_at(graph, successor).suggestion is None: # if the sequential "fallback" has been used, check if a different option is specifically # mentioned in restrict_to_decisions. If so, the sequential fallback shall be ignored. options = get_out_options(graph, successor) - restricted_options = [ - opt for opt in options if opt in restrict_to_decisions - ] + restricted_options = [opt for opt in options if opt in restrict_to_decisions] if len(restricted_options) != 0: # do not use he sequential fallback since a required option exists path_invalid = True @@ -282,9 +269,7 @@ def get_node_performance_models( ignore_node_costs=ignore_node_costs, ): tmp = combined_model.parallelizable_plus_combine(model) - tmp.path_decisions += [ - d for d in model.path_decisions if d not in tmp.path_decisions - ] + tmp.path_decisions += [d for d in model.path_decisions if d not in tmp.path_decisions] result_list.append(tmp) if len(result_list) >= 1: return result_list diff --git a/discopop_library/discopop_optimizer/DataTransfers/DataTransfers.py b/discopop_library/discopop_optimizer/DataTransfers/DataTransfers.py index 9a947f9bd..93ed6ccc1 100644 --- a/discopop_library/discopop_optimizer/DataTransfers/DataTransfers.py +++ b/discopop_library/discopop_optimizer/DataTransfers/DataTransfers.py @@ -5,16 +5,14 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from typing import Dict, List, Tuple, Set, cast +from typing import Dict, List, Tuple, cast import networkx as nx # type: ignore from discopop_library.discopop_optimizer.CostModels.CostModel import CostModel from discopop_library.discopop_optimizer.classes.context.ContextObject import ContextObject -from discopop_library.discopop_optimizer.classes.context.Update import Update from discopop_library.discopop_optimizer.classes.nodes.ContextNode import ContextNode from discopop_library.discopop_optimizer.classes.nodes.FunctionRoot import FunctionRoot -from discopop_library.discopop_optimizer.classes.types.Aliases import DeviceID from discopop_library.discopop_optimizer.utilities.MOGUtilities import ( get_successors, data_at, @@ -37,9 +35,7 @@ def calculate_data_transfers( return result_dict -def get_path_context( - node_id: int, graph: nx.DiGraph, model: CostModel, context: ContextObject -) -> ContextObject: +def get_path_context(node_id: int, graph: nx.DiGraph, model: CostModel, context: ContextObject) -> ContextObject: """passes the context Object along the path and returns the context once the end has been reached""" # push device id to stack if necessary node_data = data_at(graph, node_id) @@ -91,9 +87,7 @@ def get_path_context( return get_path_context(suitable_successors[0], graph, model, context) -def __check_current_node( - node_id: int, graph: nx.DiGraph, model: CostModel, context: ContextObject -) -> ContextObject: +def __check_current_node(node_id: int, graph: nx.DiGraph, model: CostModel, context: ContextObject) -> ContextObject: """Check if the given node results in modifications to the given context. Return a modified version of the context which contains the required updates.""" # due to the Read-Compute-Write paradigm used to create the Computational Units, @@ -117,16 +111,12 @@ def __check_current_node( ) # add the writes performed by the given node to the context - context = context.add_writes( - node_data.written_memory_regions, cast(int, context.last_seen_device_ids[-1]) - ) + context = context.add_writes(node_data.written_memory_regions, cast(int, context.last_seen_device_ids[-1])) return context -def __check_children( - node_id: int, graph: nx.DiGraph, model: CostModel, context: ContextObject -) -> ContextObject: +def __check_children(node_id: int, graph: nx.DiGraph, model: CostModel, context: ContextObject) -> ContextObject: # pass context to all children for child in get_children(graph, node_id): # reset last_visited_node_id inbetween visiting children diff --git a/discopop_library/discopop_optimizer/Microbench/DelaunayInterpolatedMicrobench.py b/discopop_library/discopop_optimizer/Microbench/DelaunayInterpolatedMicrobench.py index 3ae4d3b6c..ecea207d8 100644 --- a/discopop_library/discopop_optimizer/Microbench/DelaunayInterpolatedMicrobench.py +++ b/discopop_library/discopop_optimizer/Microbench/DelaunayInterpolatedMicrobench.py @@ -5,18 +5,18 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from .PureDataMicrobench import PureDataMicrobench +from typing import Dict, List, Tuple, Union + +import numpy as np +from scipy.interpolate import LinearNDInterpolator # type: ignore + from .Microbench import ( Microbench, MicrobenchType, MicrobenchDimension, MicrobenchCoordinate, ) - -from typing import Dict, List, Tuple, Union - -import numpy as np -from scipy.interpolate import LinearNDInterpolator # type: ignore +from .PureDataMicrobench import PureDataMicrobench # This class uses Delaunay Interpolation to create a microbench model from measurements. No extrapolation is possible @@ -99,6 +99,4 @@ def evaluateInterpolation( if not self.__isInterpolated: self.__interpolate() self.__isInterpolated = True - return ( - self.interpolator[benchType][benchDim](benchCoord) / 1000000.0 - ) # convert microseconds to seconds + return self.interpolator[benchType][benchDim](benchCoord) / 1000000.0 # convert microseconds to seconds diff --git a/discopop_library/discopop_optimizer/Microbench/ExtrapInterpolatedMicrobench.py b/discopop_library/discopop_optimizer/Microbench/ExtrapInterpolatedMicrobench.py index 7f0e1076b..960549456 100644 --- a/discopop_library/discopop_optimizer/Microbench/ExtrapInterpolatedMicrobench.py +++ b/discopop_library/discopop_optimizer/Microbench/ExtrapInterpolatedMicrobench.py @@ -5,15 +5,9 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from .Microbench import ( - Microbench, - MicrobenchType, - MicrobenchDimension, - MicrobenchCoordinate, -) - from typing import Dict, Tuple, Union +import sympy from extrap.entities.callpath import Callpath # type: ignore from extrap.entities.metric import Metric # type: ignore from extrap.entities.model import Model # type: ignore @@ -22,7 +16,13 @@ from extrap.modelers.model_generator import ModelGenerator # type: ignore from extrap.modelers.multi_parameter.multi_parameter_modeler import MultiParameterModeler # type: ignore from sympy.parsing.sympy_parser import parse_expr # type: ignore -import sympy + +from .Microbench import ( + Microbench, + MicrobenchType, + MicrobenchDimension, + MicrobenchCoordinate, +) # This class uses extrap to extrapolate microbench measurements. @@ -86,6 +86,4 @@ def evaluateInterpolation( benchDim: MicrobenchDimension, benchCoord: Union[MicrobenchCoordinate, Tuple[int, float, float]], ): - return self.models[(Callpath(benchType), Metric(benchDim))].hypothesis.function.evaluate( - [*benchCoord] - ) + return self.models[(Callpath(benchType), Metric(benchDim))].hypothesis.function.evaluate([*benchCoord]) diff --git a/discopop_library/discopop_optimizer/Microbench/Microbench.py b/discopop_library/discopop_optimizer/Microbench/Microbench.py index 6963eb8f8..b2f2567d0 100644 --- a/discopop_library/discopop_optimizer/Microbench/Microbench.py +++ b/discopop_library/discopop_optimizer/Microbench/Microbench.py @@ -6,14 +6,15 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. from __future__ import annotations + +import logging from abc import ABC, abstractmethod from enum import Enum -import logging from pathlib import Path -from typing import Dict, List, NamedTuple, Optional, Tuple, Union, cast, Any +from typing import Dict, List, NamedTuple, Optional, Tuple, Union -import numpy as np import matplotlib # type: ignore +import numpy as np matplotlib.use("Agg") from matplotlib import cm # type: ignore @@ -89,11 +90,7 @@ def plotInterpolation( # y axis: workloads # z axis: measurement values # (iterations are fixed based on input parameter) - z = [ - self.evaluateInterpolation(type, dim, (i, j, iterations)) - for i in threads - for j in workloads - ] # results + z = [self.evaluateInterpolation(type, dim, (i, j, iterations)) for i in threads for j in workloads] # results X, Y = np.meshgrid(threads, workloads) Z = np.array(z).reshape(len(threads), len(workloads)).transpose() diff --git a/discopop_library/discopop_optimizer/Microbench/MicrobenchParser.py b/discopop_library/discopop_optimizer/Microbench/MicrobenchParser.py index 579c05a39..3c11c7ccd 100644 --- a/discopop_library/discopop_optimizer/Microbench/MicrobenchParser.py +++ b/discopop_library/discopop_optimizer/Microbench/MicrobenchParser.py @@ -5,9 +5,8 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from dataclasses import dataclass import json -import os +from dataclasses import dataclass from typing import Dict, List, Tuple from .Microbench import ( @@ -16,7 +15,6 @@ MicrobenchDimension, ) - # threads/workload/iterations ParsedCoordinate = Tuple[int, int, int] diff --git a/discopop_library/discopop_optimizer/Microbench/MixedMicrobench.py b/discopop_library/discopop_optimizer/Microbench/MixedMicrobench.py index fbb0c9f7a..bf64b6113 100644 --- a/discopop_library/discopop_optimizer/Microbench/MixedMicrobench.py +++ b/discopop_library/discopop_optimizer/Microbench/MixedMicrobench.py @@ -6,6 +6,7 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. from typing import Dict, List, Tuple, Union + from .Microbench import Microbench from .Microbench import MicrobenchCoordinate, MicrobenchDimension, MicrobenchType diff --git a/discopop_library/discopop_optimizer/Microbench/PureDataMicrobench.py b/discopop_library/discopop_optimizer/Microbench/PureDataMicrobench.py index 7c53e3e88..bf0cb1f86 100644 --- a/discopop_library/discopop_optimizer/Microbench/PureDataMicrobench.py +++ b/discopop_library/discopop_optimizer/Microbench/PureDataMicrobench.py @@ -6,9 +6,10 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. from __future__ import annotations + +import json from dataclasses import dataclass from typing import Dict, List, Tuple, TypeVar, Union, overload -import json import numpy as np @@ -51,21 +52,15 @@ class PureDataMicrobench(Microbench): ] @overload - def __getitem__( - self, key: MicrobenchType - ) -> Dict[MicrobenchDimension, Dict[MicrobenchCoordinate, List[float]]]: + def __getitem__(self, key: MicrobenchType) -> Dict[MicrobenchDimension, Dict[MicrobenchCoordinate, List[float]]]: ... @overload - def __getitem__( - self, key: Tuple[MicrobenchType, MicrobenchDimension] - ) -> Dict[MicrobenchCoordinate, List[float]]: + def __getitem__(self, key: Tuple[MicrobenchType, MicrobenchDimension]) -> Dict[MicrobenchCoordinate, List[float]]: ... @overload - def __getitem__( - self, key: Tuple[MicrobenchType, MicrobenchDimension, MicrobenchCoordinate] - ) -> List[float]: + def __getitem__(self, key: Tuple[MicrobenchType, MicrobenchDimension, MicrobenchCoordinate]) -> List[float]: ... # allow to use this class like a dictionary @@ -108,9 +103,7 @@ def __setitem__(self, key, value): ): if not isinstance(value, Dict): raise ValueError( - "Invalid value type: " - + str(type(value)) - + " expected: Dict[MicrobenchCoordinate, List[float]]" + "Invalid value type: " + str(type(value)) + " expected: Dict[MicrobenchCoordinate, List[float]]" ) self.measurements[key[0]][key[1]] = value elif ( @@ -165,8 +158,7 @@ def clampValues( if d in dimMap: for coord, values in dimMap[d].items(): self.measurements[type][d][coord] = [ - v if v >= min and v <= max else (min if v < min else max) - for v in values + v if v >= min and v <= max else (min if v < min else max) for v in values ] def merge(self, other: PureDataMicrobench): diff --git a/discopop_library/discopop_optimizer/Microbench/utils.py b/discopop_library/discopop_optimizer/Microbench/utils.py index ccc4d5473..6427c052b 100644 --- a/discopop_library/discopop_optimizer/Microbench/utils.py +++ b/discopop_library/discopop_optimizer/Microbench/utils.py @@ -10,9 +10,7 @@ from sympy import Expr, Integer, Max -def convert_discopop_to_microbench_workload( - discopop_per_iteration_workload: Expr, iteration_count: Expr -) -> Expr: +def convert_discopop_to_microbench_workload(discopop_per_iteration_workload: Expr, iteration_count: Expr) -> Expr: """Converts the estimated workload into the workload measurement used by Microbench. According to Formula: @@ -39,9 +37,7 @@ def convert_discopop_to_microbench_workload( return Max(Wm, Integer(1)) -def convert_microbench_to_discopop_workload( - microbench_workload: Expr, iteration_count: Expr -) -> Expr: +def convert_microbench_to_discopop_workload(microbench_workload: Expr, iteration_count: Expr) -> Expr: """Converts the estimated workload into the workload measurement used by DiscoPoP. According to Formula: diff --git a/discopop_library/discopop_optimizer/OptimizationGraph.py b/discopop_library/discopop_optimizer/OptimizationGraph.py index 2338485f2..9455a5d6e 100644 --- a/discopop_library/discopop_optimizer/OptimizationGraph.py +++ b/discopop_library/discopop_optimizer/OptimizationGraph.py @@ -5,12 +5,12 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from typing import Dict, cast, List, Tuple, Optional +import tkinter as tk +from typing import Dict, cast, List, Tuple, Optional, Any import jsonpickle # type: ignore import networkx as nx # type: ignore import sympy # type: ignore -import tkinter as tk from spb import plot3d, MB # type: ignore from sympy import Integer, Expr, Symbol, lambdify, plot, Float, init_printing, simplify, diff # type: ignore @@ -48,7 +48,7 @@ def __init__( self, project_folder_path, experiment: Experiment, - arguments: Dict, + arguments: Dict[str, Any], parent_frame: Optional[tk.Frame], destroy_window_after_execution: bool, ): @@ -122,23 +122,19 @@ def __init__( experiment.substitutions[symbol] = Float(value) else: free_symbol_ranges[symbol] = (cast(float, start_value), cast(float, end_value)) - free_symbol_distributions[symbol] = cast( - FreeSymbolDistribution, symbol_distribution - ) + free_symbol_distributions[symbol] = cast(FreeSymbolDistribution, symbol_distribution) # by default, select the sequential version of each function for substitution for function in sequential_complete_performance_models: - experiment.selected_paths_per_function[ - function - ] = sequential_complete_performance_models[function][0] + experiment.selected_paths_per_function[function] = sequential_complete_performance_models[function][0] # add function symbols to list of substitutions # collect substitutions for function in experiment.selected_paths_per_function: # register substitution - experiment.substitutions[ - cast(Symbol, function.sequential_costs) - ] = experiment.selected_paths_per_function[function][0].sequential_costs + experiment.substitutions[cast(Symbol, function.sequential_costs)] = experiment.selected_paths_per_function[ + function + ][0].sequential_costs experiment.substitutions[ cast(Symbol, function.parallelizable_costs) ] = experiment.selected_paths_per_function[function][0].parallelizable_costs diff --git a/discopop_library/discopop_optimizer/PETParser/DataAccesses/FromCUs.py b/discopop_library/discopop_optimizer/PETParser/DataAccesses/FromCUs.py index 17545cdbe..ce8122fa8 100644 --- a/discopop_library/discopop_optimizer/PETParser/DataAccesses/FromCUs.py +++ b/discopop_library/discopop_optimizer/PETParser/DataAccesses/FromCUs.py @@ -5,7 +5,7 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from typing import Tuple, List, Set, cast, Dict, Optional +from typing import Tuple, Set, cast from discopop_explorer.PETGraphX import NodeID, PETGraphX, MemoryRegion, CUNode, EdgeType, DepType from discopop_library.discopop_optimizer.classes.types.DataAccessType import ( @@ -23,9 +23,7 @@ def get_next_free_unique_write_id() -> int: return buffer -def get_data_accesses_for_cu( - pet: PETGraphX, cu_id: NodeID -) -> Tuple[Set[WriteDataAccess], Set[ReadDataAccess]]: +def get_data_accesses_for_cu(pet: PETGraphX, cu_id: NodeID) -> Tuple[Set[WriteDataAccess], Set[ReadDataAccess]]: """Calculates and returns the sets of accessed memory regions for the given cu node. The first element contains write accesses, the second element contains read accesses.""" parent_function = pet.get_parent_function(pet.node_at(cu_id)) @@ -35,18 +33,14 @@ def get_data_accesses_for_cu( out_dep_edges = pet.out_edges(cu_id, EdgeType.DATA) written_memory_regions = [ - WriteDataAccess( - MemoryRegion(cast(str, d.memory_region)), get_next_free_unique_write_id(), d.var_name - ) + WriteDataAccess(MemoryRegion(cast(str, d.memory_region)), get_next_free_unique_write_id(), d.var_name) for s, t, d in in_dep_edges if (d.dtype == DepType.RAW or d.dtype == DepType.WAW) and d.memory_region is not None and len(d.memory_region) != 0 ] written_memory_regions += [ - WriteDataAccess( - MemoryRegion(cast(str, d.memory_region)), get_next_free_unique_write_id(), d.var_name - ) + WriteDataAccess(MemoryRegion(cast(str, d.memory_region)), get_next_free_unique_write_id(), d.var_name) for s, t, d in out_dep_edges if (d.dtype == DepType.WAR or d.dtype == DepType.WAW) and d.memory_region is not None diff --git a/discopop_library/discopop_optimizer/PETParser/PETParser.py b/discopop_library/discopop_optimizer/PETParser/PETParser.py index a83811aa5..9daefce33 100644 --- a/discopop_library/discopop_optimizer/PETParser/PETParser.py +++ b/discopop_library/discopop_optimizer/PETParser/PETParser.py @@ -41,7 +41,6 @@ add_temporary_edge, redirect_edge, convert_temporary_edges, - show, get_all_function_nodes, get_read_and_written_data_from_subgraph, ) @@ -89,9 +88,7 @@ def __parse_branched_sections(self): 'between' the different branches""" visited_nodes: Set[int] = set() for function_node in self.pet.all_nodes(FunctionNode): - _, _ = self.__parse_raw_node( - self.cu_id_to_graph_node_id[function_node.id], visited_nodes - ) + _, _ = self.__parse_raw_node(self.cu_id_to_graph_node_id[function_node.id], visited_nodes) # remove visited nodes, since duplicates exist now for node_id in visited_nodes: @@ -164,9 +161,7 @@ def __parse_branching_point( # Step 2: create and connect context snapshot context_snapshot_id = self.get_new_node_id() - self.graph.add_node( - context_snapshot_id, data=ContextSnapshot(context_snapshot_id, self.experiment) - ) + self.graph.add_node(context_snapshot_id, data=ContextSnapshot(context_snapshot_id, self.experiment)) add_temporary_edge(self.graph, duplicate_node_id, context_snapshot_id) # Step 3: parse branches @@ -185,9 +180,7 @@ def __parse_branching_point( # Step 3.3: create context save node branch_context_save_id = self.get_new_node_id() - self.graph.add_node( - branch_context_save_id, data=ContextSave(branch_context_save_id, self.experiment) - ) + self.graph.add_node(branch_context_save_id, data=ContextSave(branch_context_save_id, self.experiment)) # Step 3.3: connect restore and save node to branch entry and exit add_temporary_edge(self.graph, branch_context_restore_id, branch_entry) @@ -198,9 +191,7 @@ def __parse_branching_point( # step 4: create and connect context merge node context_merge_node_id = self.get_new_node_id() - self.graph.add_node( - context_merge_node_id, data=ContextMerge(context_merge_node_id, self.experiment) - ) + self.graph.add_node(context_merge_node_id, data=ContextMerge(context_merge_node_id, self.experiment)) add_temporary_edge(self.graph, last_added_node_id, context_merge_node_id) # Step 5: create and connect context snapshot pop @@ -225,9 +216,7 @@ def __add_cu_nodes(self): new_node_id = self.get_new_node_id() self.cu_id_to_graph_node_id[cu_node.id] = new_node_id # calculate accessed data - written_memory_regions, read_memory_regions = get_data_accesses_for_cu( - self.pet, cu_node.id - ) + written_memory_regions, read_memory_regions = get_data_accesses_for_cu(self.pet, cu_node.id) self.graph.add_node( new_node_id, data=Workload( @@ -346,9 +335,7 @@ def __add_functions(self): def __add_pet_successor_edges(self): for cu_node in self.pet.all_nodes(CUNode): - for successor_cu_id in [ - t for s, t, d in self.pet.out_edges(cu_node.id, EdgeType.SUCCESSOR) - ]: + for successor_cu_id in [t for s, t, d in self.pet.out_edges(cu_node.id, EdgeType.SUCCESSOR)]: add_successor_edge( self.graph, self.cu_id_to_graph_node_id[cu_node.id], @@ -415,9 +402,7 @@ def inlined_data_flow_calculation(current_node, current_last_writes): # continue to successor successors = get_successors(self.graph, current_node) if len(successors) > 1: - raise ValueError( - "only a single successor should exist at this stage in the process!" - ) + raise ValueError("only a single successor should exist at this stage in the process!") elif len(successors) == 1: current_node = successors[0] else: diff --git a/discopop_library/discopop_optimizer/Variables/Experiment.py b/discopop_library/discopop_optimizer/Variables/Experiment.py index 5a5e68e30..229f1e5e2 100644 --- a/discopop_library/discopop_optimizer/Variables/Experiment.py +++ b/discopop_library/discopop_optimizer/Variables/Experiment.py @@ -7,20 +7,20 @@ # directory for details. import os from pathlib import Path -from typing import Dict, Tuple, Set, Optional, List +from typing import Dict, Tuple, Set, Optional, List, Any import networkx as nx # type: ignore from sympy import Integer, Symbol, Expr, Float # type: ignore from discopop_explorer.PETGraphX import MemoryRegion -from discopop_library.result_classes.DetectionResult import DetectionResult -from discopop_library.PathManagement.PathManagement import load_file_mapping from discopop_library.MemoryRegions.utils import get_sizes_of_memory_regions +from discopop_library.PathManagement.PathManagement import load_file_mapping from discopop_library.discopop_optimizer.CostModels.CostModel import CostModel from discopop_library.discopop_optimizer.classes.context.ContextObject import ContextObject from discopop_library.discopop_optimizer.classes.enums.Distributions import FreeSymbolDistribution from discopop_library.discopop_optimizer.classes.nodes.FunctionRoot import FunctionRoot from discopop_library.discopop_optimizer.classes.system.System import System +from discopop_library.result_classes.DetectionResult import DetectionResult class Experiment(object): @@ -78,7 +78,7 @@ def __init__( file_mapping_path: str, system: System, detection_result: DetectionResult, - arguments: Dict, + arguments: Dict[str, Any], ): self.__system = system self.detection_result = detection_result diff --git a/discopop_library/discopop_optimizer/Variables/ExperimentUtils.py b/discopop_library/discopop_optimizer/Variables/ExperimentUtils.py index 25c4f9869..08d3a58da 100644 --- a/discopop_library/discopop_optimizer/Variables/ExperimentUtils.py +++ b/discopop_library/discopop_optimizer/Variables/ExperimentUtils.py @@ -7,8 +7,9 @@ # directory for details. import os import pickle -from tkinter import Tk, Button -from typing import List, Optional, cast, Set +import tkinter as tk +from tkinter import Button +from typing import List, Optional, cast import jsonpickle # type: ignore import jsons # type: ignore @@ -22,7 +23,6 @@ ) from discopop_library.discopop_optimizer.gui.widgets.ScrollableFrame import ScrollableFrameWidget from discopop_library.discopop_optimizer.utilities.MOGUtilities import data_at -import tkinter as tk def show_function_models( @@ -31,9 +31,7 @@ def show_function_models( destroy_window_after_execution: bool, show_functions: Optional[List[FunctionRoot]] = None, ): - considered_functions = ( - show_functions if show_functions is not None else experiment.function_models - ) + considered_functions = show_functions if show_functions is not None else experiment.function_models # show function selection dialogue parent_frame.rowconfigure(0, weight=1) parent_frame.rowconfigure(1, weight=1) @@ -141,9 +139,7 @@ def export_to_json(experiment: Experiment): for k2, v in to_be_added: experiment.function_models[k2] = v # type: ignore - experiment_dump_path: str = os.path.join( - experiment.discopop_optimizer_path, "last_experiment.pickle" - ) + experiment_dump_path: str = os.path.join(experiment.discopop_optimizer_path, "last_experiment.pickle") if not os.path.exists(experiment.discopop_optimizer_path): os.makedirs(experiment.discopop_optimizer_path) pickle.dump(experiment, open(experiment_dump_path, "wb")) diff --git a/discopop_library/discopop_optimizer/__main__.py b/discopop_library/discopop_optimizer/__main__.py index 3c2b5ff67..0c2c59061 100644 --- a/discopop_library/discopop_optimizer/__main__.py +++ b/discopop_library/discopop_optimizer/__main__.py @@ -64,7 +64,6 @@ import pstats2 # type:ignore from docopt import docopt # type:ignore from schema import Schema, Use, SchemaError # type:ignore -from sympy import Symbol, Integer from discopop_library.result_classes.DetectionResult import DetectionResult from discopop_library.discopop_optimizer.Microbench.ExtrapInterpolatedMicrobench import ( @@ -135,33 +134,19 @@ def main(): # prepare arguments arguments["--project"] = get_path(os.getcwd(), arguments["--project"]) - arguments["--execute-created-models"] = ( - False if arguments["--execute-created-models"] == "False" else True - ) + arguments["--execute-created-models"] = False if arguments["--execute-created-models"] == "False" else True arguments["--execution-append-measurements"] = ( False if arguments["--execution-append-measurements"] == "False" else True ) - arguments["--clean-created-code"] = ( - False if arguments["--clean-created-code"] == "False" else True - ) - arguments["--code-export-path"] = get_path( - arguments["--project"], arguments["--code-export-path"] - ) - arguments["--exhaustive-search"] = ( - False if arguments["--exhaustive-search"] == "False" else True - ) + arguments["--clean-created-code"] = False if arguments["--clean-created-code"] == "False" else True + arguments["--code-export-path"] = get_path(arguments["--project"], arguments["--code-export-path"]) + arguments["--exhaustive-search"] = False if arguments["--exhaustive-search"] == "False" else True arguments["--headless-mode"] = False if arguments["--headless-mode"] == "False" else True arguments["--dp-output-path"] = get_path(arguments["--project"], arguments["--dp-output-path"]) - arguments["--file-mapping"] = get_path( - arguments["--dp-output-path"], arguments["--file-mapping"] - ) - arguments["--detection-result-dump"] = get_path( - arguments["--dp-output-path"], arguments["--detection-result-dump"] - ) + arguments["--file-mapping"] = get_path(arguments["--dp-output-path"], arguments["--file-mapping"]) + arguments["--detection-result-dump"] = get_path(arguments["--dp-output-path"], arguments["--detection-result-dump"]) arguments["--dp-optimizer-path"] = os.path.join(arguments["--project"], ".discopop_optimizer") - arguments["--make-target"] = ( - None if arguments["--make-target"] == "None" else arguments["--make-target"] - ) + arguments["--make-target"] = None if arguments["--make-target"] == "None" else arguments["--make-target"] arguments["--execute-single-model"] = ( None if len(arguments["--execute-single-model"]) == 0 @@ -221,9 +206,7 @@ def start_optimizer(arguments, parent_frame: Optional[tk.Frame] = None): ) if load_result: # load results from previous session - experiment = restore_session( - os.path.join(arguments["--dp-optimizer-path"], "last_experiment.pickle") - ) + experiment = restore_session(os.path.join(arguments["--dp-optimizer-path"], "last_experiment.pickle")) show_function_models(experiment, parent_frame, destroy_window_after_execution) # save experiment to disk export_to_json(experiment) @@ -263,9 +246,7 @@ def start_optimizer(arguments, parent_frame: Optional[tk.Frame] = None): # construct and set overhead model for reduction suggestions system.set_reduction_overhead_model( system.get_device(0), - ExtrapInterpolatedMicrobench(microbench_file).getFunctionSympy( - benchType=MicrobenchType.FOR - ), + ExtrapInterpolatedMicrobench(microbench_file).getFunctionSympy(benchType=MicrobenchType.FOR), ) # define Environment diff --git a/discopop_library/discopop_optimizer/bindings/CodeGenerator.py b/discopop_library/discopop_optimizer/bindings/CodeGenerator.py index 98d3d0e13..77e8c6772 100644 --- a/discopop_library/discopop_optimizer/bindings/CodeGenerator.py +++ b/discopop_library/discopop_optimizer/bindings/CodeGenerator.py @@ -19,7 +19,6 @@ from discopop_explorer.PETGraphX import NodeID, PETGraphX from discopop_explorer.pattern_detectors.PatternInfo import PatternInfo from discopop_explorer.pattern_detectors.device_updates import DeviceUpdateInfo -from discopop_explorer.variable import Variable from discopop_library.CodeGenerator.CodeGenerator import ( from_pattern_info as code_gen_from_pattern_info, ) @@ -56,9 +55,7 @@ def export_code( suggestions: List[Tuple[Device, PatternInfo, str, Optional[int]]] = [] for decision in cost_model.path_decisions: graph_node = data_at(graph, decision) - device: Device = experiment.get_system().get_device( - 0 if graph_node.device_id is None else graph_node.device_id - ) + device: Device = experiment.get_system().get_device(0 if graph_node.device_id is None else graph_node.device_id) if graph_node.suggestion is None: continue suggestion, suggestion_type = device.get_device_specific_pattern_info( @@ -79,18 +76,12 @@ def export_code( if is_child_of_any( graph, cur_node, - [ - s_node_id - for _, sugg, s_type, s_node_id in suggestions - if s_type.startswith("gpu_") - ], + [s_node_id for _, sugg, s_type, s_node_id in suggestions if s_type.startswith("gpu_")], ): tmp_parents = get_parents(graph, cur_node) if len(tmp_parents) != 0: unchecked_target_nodes += [ - p - for p in tmp_parents - if p not in checked_target_nodes and p not in unchecked_target_nodes + p for p in tmp_parents if p not in checked_target_nodes and p not in unchecked_target_nodes ] else: @@ -98,9 +89,7 @@ def export_code( checked_target_nodes.append(cur_node) for checked_node_id in checked_target_nodes: - start_line = pet.node_at( - cast(NodeID, data_at(graph, checked_node_id).original_cu_id) - ).start_position() + start_line = pet.node_at(cast(NodeID, data_at(graph, checked_node_id).original_cu_id)).start_position() end_line = start_line # register update @@ -108,14 +97,10 @@ def export_code( target_cu_id = cast(NodeID, data_at(graph, update.target_node_id).original_cu_id) if source_cu_id is None or target_cu_id is None: - warnings.warn( - "Could not register update: " + str(update) + " @ Line: " + start_line - ) + warnings.warn("Could not register update: " + str(update) + " @ Line: " + start_line) else: # get updated variable - var_obj = pet.get_variable( - target_cu_id, cast(str, update.write_data_access.var_name) - ) + var_obj = pet.get_variable(target_cu_id, cast(str, update.write_data_access.var_name)) if var_obj is None: raise ValueError( "Could not find variable object for: " @@ -126,11 +111,7 @@ def export_code( # get amount of elements targeted by the update update_elements = int( - int( - experiment.get_memory_region_size( - update.write_data_access.memory_region - )[0].evalf() - ) + int(experiment.get_memory_region_size(update.write_data_access.memory_region)[0].evalf()) / var_obj.sizeInByte ) @@ -146,11 +127,7 @@ def export_code( ) # add range to updated var name if necessary - if ( - update_elements > 1 - and update.write_data_access.var_name is not None - and "**" in var_obj.type - ): + if update_elements > 1 and update.write_data_access.var_name is not None and "**" in var_obj.type: updated_var_name: Optional[str] = ( str(update.write_data_access.var_name) + "[:" @@ -171,18 +148,12 @@ def export_code( update.write_data_access.memory_region, updated_var_name, 0 if update.source_device_id is None else update.source_device_id, - 0 - if update.source_device_id is None - else cast(int, update.target_device_id), + 0 if update.source_device_id is None else cast(int, update.target_device_id), start_line, end_line, update.is_first_data_occurrence, - cast( - GPU, experiment.get_system().get_device(update.source_device_id) - ).openmp_device_id, - cast( - GPU, experiment.get_system().get_device(update.target_device_id) - ).openmp_device_id, + cast(GPU, experiment.get_system().get_device(update.source_device_id)).openmp_device_id, + cast(GPU, experiment.get_system().get_device(update.target_device_id)).openmp_device_id, ), "device_update", None, @@ -315,9 +286,7 @@ def export_code( print("\n") -def __convert_modified_code_to_patch( - experiment: Experiment, modified_code: Dict[int, str] -) -> Dict[int, str]: +def __convert_modified_code_to_patch(experiment: Experiment, modified_code: Dict[int, str]) -> Dict[int, str]: patches: Dict[int, str] = dict() hash_name = "".join(random.choices(string.ascii_uppercase + string.digits, k=8)) tmp_file_name = os.path.join(os.getcwd(), hash_name + ".tmp") diff --git a/discopop_library/discopop_optimizer/bindings/CodeStorageObject.py b/discopop_library/discopop_optimizer/bindings/CodeStorageObject.py index 99acc2184..adf0c34d6 100644 --- a/discopop_library/discopop_optimizer/bindings/CodeStorageObject.py +++ b/discopop_library/discopop_optimizer/bindings/CodeStorageObject.py @@ -8,7 +8,6 @@ from typing import Dict from discopop_library.discopop_optimizer.CostModels.CostModel import CostModel -from discopop_library.discopop_optimizer.classes.context.ContextObject import ContextObject from discopop_library.discopop_optimizer.classes.nodes.FunctionRoot import FunctionRoot diff --git a/discopop_library/discopop_optimizer/bindings/utilities.py b/discopop_library/discopop_optimizer/bindings/utilities.py index 00e9732c7..cf8dd71ad 100644 --- a/discopop_library/discopop_optimizer/bindings/utilities.py +++ b/discopop_library/discopop_optimizer/bindings/utilities.py @@ -12,9 +12,7 @@ from discopop_library.discopop_optimizer.utilities.MOGUtilities import get_parents -def is_child_of_any( - graph: nx.DiGraph, start_node: int, potential_parents: List[Optional[int]] -) -> bool: +def is_child_of_any(graph: nx.DiGraph, start_node: int, potential_parents: List[Optional[int]]) -> bool: """returns True, if start_node is a child of any of the potential_parents""" # get all parents of start_node all_parents: Set[int] = set() diff --git a/discopop_library/discopop_optimizer/classes/context/ContextObject.py b/discopop_library/discopop_optimizer/classes/context/ContextObject.py index 5e7466e39..c40714286 100644 --- a/discopop_library/discopop_optimizer/classes/context/ContextObject.py +++ b/discopop_library/discopop_optimizer/classes/context/ContextObject.py @@ -5,7 +5,7 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from typing import Dict, Set, cast, List, Optional +from typing import Dict, Set, List, Optional from sympy import Expr, Integer, Symbol # type: ignore @@ -24,9 +24,7 @@ class ContextObject(object): seen_writes_by_device: Dict[DeviceID, Dict[MemoryRegion, Set[WriteDataAccess]]] necessary_updates: Set[Update] - def __init__( - self, initializing_node_id: int, last_seen_device_ids: Optional[List[DeviceID]] = None - ): + def __init__(self, initializing_node_id: int, last_seen_device_ids: Optional[List[DeviceID]] = None): self.seen_writes_by_device = dict() self.necessary_updates = set() self.last_visited_node_id = initializing_node_id @@ -56,9 +54,7 @@ def calculate_and_perform_necessary_updates( if read.memory_region not in self.seen_writes_by_device[device_id]: # read memory region is currently "unknown" to the device, thus is can be skipped continue - other_devices_known_writes = self.seen_writes_by_device[device_id][ - read.memory_region - ] + other_devices_known_writes = self.seen_writes_by_device[device_id][read.memory_region] is_first_data_occurrence = False if read.memory_region not in self.seen_writes_by_device[reading_device_id]: @@ -90,10 +86,7 @@ def calculate_and_perform_necessary_updates( # check if data is known to the host if data_write.memory_region not in self.seen_writes_by_device[0]: self.seen_writes_by_device[0][data_write.memory_region] = set() - if ( - data_write - not in self.seen_writes_by_device[0][data_write.memory_region] - ): + if data_write not in self.seen_writes_by_device[0][data_write.memory_region]: # register source device -> host update required_updates.add( Update( @@ -154,9 +147,9 @@ def calculate_and_perform_necessary_updates( self.seen_writes_by_device[update.target_device_id][ update.write_data_access.memory_region ] = set() - self.seen_writes_by_device[update.target_device_id][ - update.write_data_access.memory_region - ].add(update.write_data_access) + self.seen_writes_by_device[update.target_device_id][update.write_data_access.memory_region].add( + update.write_data_access + ) self.necessary_updates.update(required_updates) diff --git a/discopop_library/discopop_optimizer/classes/context/ContextObjectUtils.py b/discopop_library/discopop_optimizer/classes/context/ContextObjectUtils.py index 394c450f7..099a052a8 100644 --- a/discopop_library/discopop_optimizer/classes/context/ContextObjectUtils.py +++ b/discopop_library/discopop_optimizer/classes/context/ContextObjectUtils.py @@ -27,9 +27,7 @@ def get_transfer_costs(context: ContextObject, environment: Experiment) -> CostM system = environment.get_system() source_device = system.get_device(cast(int, update.source_device_id)) target_device = system.get_device(cast(int, update.target_device_id)) - initialization_costs = system.get_network().get_transfer_initialization_costs( - source_device, target_device - ) + initialization_costs = system.get_network().get_transfer_initialization_costs(source_device, target_device) total_transfer_costs += initialization_costs @@ -49,8 +47,6 @@ def get_transfer_costs(context: ContextObject, environment: Experiment) -> CostM total_transfer_costs += transfer_costs if symbolic_memory_region_sizes: - return CostModel( - Integer(0), total_transfer_costs, symbol_value_suggestions=symbol_value_suggestions - ) + return CostModel(Integer(0), total_transfer_costs, symbol_value_suggestions=symbol_value_suggestions) else: return CostModel(Integer(0), total_transfer_costs) diff --git a/discopop_library/discopop_optimizer/classes/edges/SuccessorEdge.py b/discopop_library/discopop_optimizer/classes/edges/SuccessorEdge.py index 7c98a18e6..fdfb09b9d 100644 --- a/discopop_library/discopop_optimizer/classes/edges/SuccessorEdge.py +++ b/discopop_library/discopop_optimizer/classes/edges/SuccessorEdge.py @@ -7,7 +7,6 @@ # directory for details. from sympy import Symbol # type: ignore -from discopop_library.discopop_optimizer.CostModels.CostModel import CostModel from discopop_library.discopop_optimizer.classes.edges.GenericEdge import GenericEdge diff --git a/discopop_library/discopop_optimizer/classes/nodes/ContextMerge.py b/discopop_library/discopop_optimizer/classes/nodes/ContextMerge.py index ed7b4d3ef..97fb53d72 100644 --- a/discopop_library/discopop_optimizer/classes/nodes/ContextMerge.py +++ b/discopop_library/discopop_optimizer/classes/nodes/ContextMerge.py @@ -27,22 +27,16 @@ def get_modified_context( if len(context.snapshot_stack) < 1 or len(context.save_stack) < 1: raise ValueError("Context can not be merged before creating a snapshot!") - context.last_seen_device_ids = cast( - ContextObject, context.snapshot_stack[-1] - ).last_seen_device_ids + context.last_seen_device_ids = cast(ContextObject, context.snapshot_stack[-1]).last_seen_device_ids for saved_contex in context.save_stack[-1]: context.necessary_updates.update(cast(ContextObject, saved_contex).necessary_updates) for device_id in cast(ContextObject, saved_contex).seen_writes_by_device: if device_id not in context.seen_writes_by_device: context.seen_writes_by_device[device_id] = dict() - for mem_reg_id in cast(ContextObject, saved_contex).seen_writes_by_device[ - device_id - ]: + for mem_reg_id in cast(ContextObject, saved_contex).seen_writes_by_device[device_id]: if mem_reg_id not in context.seen_writes_by_device[device_id]: context.seen_writes_by_device[device_id][mem_reg_id] = set() context.seen_writes_by_device[device_id][mem_reg_id].update( - cast(ContextObject, saved_contex).seen_writes_by_device[device_id][ - mem_reg_id - ] + cast(ContextObject, saved_contex).seen_writes_by_device[device_id][mem_reg_id] ) return context diff --git a/discopop_library/discopop_optimizer/classes/nodes/ContextNode.py b/discopop_library/discopop_optimizer/classes/nodes/ContextNode.py index f7615fdf9..4f8679e4f 100644 --- a/discopop_library/discopop_optimizer/classes/nodes/ContextNode.py +++ b/discopop_library/discopop_optimizer/classes/nodes/ContextNode.py @@ -6,11 +6,11 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. import networkx as nx # type: ignore +from sympy import Integer from discopop_library.discopop_optimizer.CostModels.CostModel import CostModel from discopop_library.discopop_optimizer.classes.context.ContextObject import ContextObject from discopop_library.discopop_optimizer.classes.nodes.Workload import Workload -from sympy import Integer class ContextNode(Workload): diff --git a/discopop_library/discopop_optimizer/classes/nodes/GenericNode.py b/discopop_library/discopop_optimizer/classes/nodes/GenericNode.py index 92f490d6d..114219f8b 100644 --- a/discopop_library/discopop_optimizer/classes/nodes/GenericNode.py +++ b/discopop_library/discopop_optimizer/classes/nodes/GenericNode.py @@ -75,20 +75,14 @@ def get_hover_text(self) -> str: return "" def get_cost_model(self, experiment, all_function_nodes, current_device) -> CostModel: - raise NotImplementedError( - "Implementation needs to be provided by derived class: !", type(self) - ) + raise NotImplementedError("Implementation needs to be provided by derived class: !", type(self)) def register_child(self, other, experiment, all_function_nodes, current_device): """Registers a child node for the given model. Does not modify the stored model in self or other.""" - raise NotImplementedError( - "Implementation needs to be provided by derived class: !", type(self) - ) + raise NotImplementedError("Implementation needs to be provided by derived class: !", type(self)) def register_successor(self, other, root_node): """Registers a successor node for the given model. Does not modify the stored model in self or other.""" - raise NotImplementedError( - "Implementation needs to be provided by derived class: !", type(self) - ) + raise NotImplementedError("Implementation needs to be provided by derived class: !", type(self)) diff --git a/discopop_library/discopop_optimizer/classes/nodes/Loop.py b/discopop_library/discopop_optimizer/classes/nodes/Loop.py index 09126f960..593e1ecf7 100644 --- a/discopop_library/discopop_optimizer/classes/nodes/Loop.py +++ b/discopop_library/discopop_optimizer/classes/nodes/Loop.py @@ -5,8 +5,7 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -import sys -from typing import Optional, cast, Set +from typing import Optional from sympy import Symbol, Integer, Expr, Float # type: ignore @@ -33,31 +32,19 @@ def __init__( iterations_symbol: Optional[Symbol] = None, ): self.position = position - self.iterations = max( - iterations, 1 - ) # to prevent dividing by 0 in case the loop has not been executed + self.iterations = max(iterations, 1) # to prevent dividing by 0 in case the loop has not been executed if iterations_symbol is None: - self.iterations_symbol = Symbol( - "loop_" + str(node_id) + "_pos_" + str(self.position) + "_iterations" - ) + self.iterations_symbol = Symbol("loop_" + str(node_id) + "_pos_" + str(self.position) + "_iterations") else: self.iterations_symbol = iterations_symbol # create parallelizable_workload_symbol self.per_iteration_parallelizable_workload = Symbol( - "loop_" - + str(node_id) - + "_pos_" - + str(self.position) - + "_per_iteration_parallelizable_workload" + "loop_" + str(node_id) + "_pos_" + str(self.position) + "_per_iteration_parallelizable_workload" ) self.per_iteration_sequential_workload = Symbol( - "loop_" - + str(node_id) - + "_pos_" - + str(self.position) - + "_per_iteration_sequential_workload" + "loop_" + str(node_id) + "_pos_" + str(self.position) + "_per_iteration_sequential_workload" ) # calculate workload per iteration @@ -75,9 +62,7 @@ def __init__( ) # register iteration symbol in environment - experiment.register_free_symbol( - self.iterations_symbol, value_suggestion=Integer(self.iterations) - ) + experiment.register_free_symbol(self.iterations_symbol, value_suggestion=Integer(self.iterations)) # register per iteration parallelizable workload symbol in environment experiment.register_free_symbol( self.per_iteration_parallelizable_workload, @@ -126,9 +111,7 @@ def register_child(self, other, experiment, all_function_nodes, current_device): # at every time, only a single child is possible for each loop node self.registered_child = other - experiment.substitutions[ - self.per_iteration_parallelizable_workload - ] = other.parallelizable_costs + experiment.substitutions[self.per_iteration_parallelizable_workload] = other.parallelizable_costs experiment.substitutions[self.per_iteration_sequential_workload] = other.sequential_costs cm = self.get_cost_model(experiment, all_function_nodes, current_device) diff --git a/discopop_library/discopop_optimizer/classes/nodes/Workload.py b/discopop_library/discopop_optimizer/classes/nodes/Workload.py index 91cd003e5..6635000c7 100644 --- a/discopop_library/discopop_optimizer/classes/nodes/Workload.py +++ b/discopop_library/discopop_optimizer/classes/nodes/Workload.py @@ -5,7 +5,7 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from typing import Optional, Set, List, cast, Union +from typing import Optional, Set, List, cast from sympy import Integer, Expr # type: ignore @@ -80,27 +80,19 @@ def get_cost_model(self, experiment, all_function_nodes, current_device) -> Cost .parallelizable_multiply_combine(self.cost_multiplier) .parallelizable_plus_combine(self.overhead) .parallelizable_plus_combine( - self.__get_costs_of_function_call( - experiment, all_function_nodes, current_device - ) + self.__get_costs_of_function_call(experiment, all_function_nodes, current_device) ) ) else: cm = ( CostModel( - current_device.get_estimated_execution_time_in_micro_seconds( - self.parallelizable_workload, True - ), - current_device.get_estimated_execution_time_in_micro_seconds( - self.sequential_workload, True - ), + current_device.get_estimated_execution_time_in_micro_seconds(self.parallelizable_workload, True), + current_device.get_estimated_execution_time_in_micro_seconds(self.sequential_workload, True), ) .parallelizable_multiply_combine(self.cost_multiplier) .parallelizable_plus_combine(self.overhead) .parallelizable_plus_combine( - self.__get_costs_of_function_call( - experiment, all_function_nodes, current_device - ) + self.__get_costs_of_function_call(experiment, all_function_nodes, current_device) ) ) @@ -110,9 +102,7 @@ def get_cost_model(self, experiment, all_function_nodes, current_device) -> Cost return cm - def __get_costs_of_function_call( - self, experiment, all_function_nodes, current_device - ) -> CostModel: + def __get_costs_of_function_call(self, experiment, all_function_nodes, current_device) -> CostModel: """Check if the node performs a function call and returns the total costs for these.""" total_costs = CostModel(Integer(0), Integer(0)) # get CUIDs of called functions @@ -124,17 +114,13 @@ def __get_costs_of_function_call( ) ] # filter for called FunctionRoots - called_function_nodes = [ - fr for fr in all_function_nodes if str(fr.original_cu_id) in called_cu_ids - ] + called_function_nodes = [fr for fr in all_function_nodes if str(fr.original_cu_id) in called_cu_ids] # remove duplicates called_function_nodes = list(set(called_function_nodes)) # add costs of called function nodes to total costs for called_function_root in called_function_nodes: total_costs = total_costs.parallelizable_plus_combine( - called_function_root.get_cost_model( - experiment, all_function_nodes, current_device - ) + called_function_root.get_cost_model(experiment, all_function_nodes, current_device) ) return total_costs diff --git a/discopop_library/discopop_optimizer/classes/system/Network.py b/discopop_library/discopop_optimizer/classes/system/Network.py index 829aac6fc..1a18be0a4 100644 --- a/discopop_library/discopop_optimizer/classes/system/Network.py +++ b/discopop_library/discopop_optimizer/classes/system/Network.py @@ -20,9 +20,7 @@ def __init__(self): self.__transfer_speeds = dict() self.__transfer_initialization_costs = dict() - def add_connection( - self, source: Device, target: Device, transfer_speed: Expr, initialization_delay: Expr - ): + def add_connection(self, source: Device, target: Device, transfer_speed: Expr, initialization_delay: Expr): self.__transfer_speeds[(source, target)] = transfer_speed self.__transfer_initialization_costs[(source, target)] = initialization_delay diff --git a/discopop_library/discopop_optimizer/classes/system/System.py b/discopop_library/discopop_optimizer/classes/system/System.py index 25d015dda..e49cd3804 100644 --- a/discopop_library/discopop_optimizer/classes/system/System.py +++ b/discopop_library/discopop_optimizer/classes/system/System.py @@ -8,7 +8,7 @@ import warnings from typing import Dict, List, Tuple, Optional -from sympy import Symbol, Expr, Integer, simplify +from sympy import Symbol, Expr, Integer from discopop_library.discopop_optimizer.classes.system.Network import Network from discopop_library.discopop_optimizer.classes.system.devices.CPU import CPU diff --git a/discopop_library/discopop_optimizer/classes/system/devices/Device.py b/discopop_library/discopop_optimizer/classes/system/devices/Device.py index aacf98173..a7bc23607 100644 --- a/discopop_library/discopop_optimizer/classes/system/devices/Device.py +++ b/discopop_library/discopop_optimizer/classes/system/devices/Device.py @@ -8,9 +8,9 @@ from typing import Tuple, List, Optional, cast from sympy import Expr, Symbol +from sympy import Integer from discopop_explorer.pattern_detectors.PatternInfo import PatternInfo -from sympy import Integer class Device(object): @@ -73,9 +73,7 @@ def get_estimated_execution_time_in_micro_seconds(self, workload: Expr, is_seque instructions_per_second = instructions_per_core_per_second * ( Integer(1) if is_sequential else self.__thread_count ) - workload_in_instructions = ( - workload * 2.120152292 - ) # todo (get from benchmarking / extra-p model) + workload_in_instructions = workload * 2.120152292 # todo (get from benchmarking / extra-p model) # current factor determined by manual "measurement" based on a single example! execution_time_in_seconds = workload_in_instructions / instructions_per_second diff --git a/discopop_library/discopop_optimizer/classes/system/devices/GPU.py b/discopop_library/discopop_optimizer/classes/system/devices/GPU.py index 5a8918d10..8d66604cf 100644 --- a/discopop_library/discopop_optimizer/classes/system/devices/GPU.py +++ b/discopop_library/discopop_optimizer/classes/system/devices/GPU.py @@ -7,8 +7,8 @@ # directory for details. from typing import Tuple -from discopop_library.discopop_optimizer.classes.system.devices.Device import Device from discopop_explorer.pattern_detectors.PatternInfo import PatternInfo +from discopop_library.discopop_optimizer.classes.system.devices.Device import Device class GPU(Device): diff --git a/discopop_library/discopop_optimizer/classes/types/DataAccessType.py b/discopop_library/discopop_optimizer/classes/types/DataAccessType.py index 0b3ea4ce6..7e7c367a8 100644 --- a/discopop_library/discopop_optimizer/classes/types/DataAccessType.py +++ b/discopop_library/discopop_optimizer/classes/types/DataAccessType.py @@ -5,7 +5,7 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -from typing import Tuple, Optional +from typing import Optional from discopop_explorer.PETGraphX import MemoryRegion @@ -33,9 +33,7 @@ def __init__(self, memory_region: MemoryRegion, unique_id: int, var_name: Option self.var_name = var_name def __str__(self): - return ( - "W(" + self.memory_region + "-" + str(self.unique_id) + ", --> " + self.var_name + ")" - ) + return "W(" + self.memory_region + "-" + str(self.unique_id) + ", --> " + self.var_name + ")" def __hash__(self): return self.unique_id diff --git a/discopop_library/discopop_optimizer/execution/stored_models.py b/discopop_library/discopop_optimizer/execution/stored_models.py index fb62ab5d1..f638aac6e 100644 --- a/discopop_library/discopop_optimizer/execution/stored_models.py +++ b/discopop_library/discopop_optimizer/execution/stored_models.py @@ -14,7 +14,7 @@ import time import warnings from pathlib import Path -from typing import Dict, cast, List, TextIO, Tuple +from typing import Dict, cast, List, TextIO, Tuple, Any import jsonpickle # type: ignore @@ -22,7 +22,7 @@ from discopop_library.discopop_optimizer.bindings.CodeStorageObject import CodeStorageObject -def execute_stored_models(arguments: Dict): +def execute_stored_models(arguments: Dict[str, Any]): """Collects and executes all models stored in the current project path""" print("Cleaning environment...") __initialize_measurement_directory(arguments) @@ -33,9 +33,7 @@ def execute_stored_models(arguments: Dict): for file_name in os.listdir(str(arguments["--code-export-path"])): print("\t", file_name) __create_project_copy(arguments["--project"], working_copy_dir) - code_modifications = __load_code_storage_object( - os.path.join(str(arguments["--code-export-path"]), file_name) - ) + code_modifications = __load_code_storage_object(os.path.join(str(arguments["--code-export-path"]), file_name)) __apply_modifications( arguments["--project"], working_copy_dir, @@ -47,7 +45,7 @@ def execute_stored_models(arguments: Dict): # __cleanup(working_copy_dir) -def execute_single_model(arguments: Dict): +def execute_single_model(arguments: Dict[str, Any]): """Executes the single models specified by the given arguments""" print("Cleaning environment...") __initialize_measurement_directory(arguments) @@ -59,9 +57,7 @@ def execute_single_model(arguments: Dict): file_name = arguments["--execute-single-model"] print("\t", file_name) __create_project_copy(arguments["--project"], working_copy_dir) - code_modifications = __load_code_storage_object( - os.path.join(str(arguments["--code-export-path"]), file_name) - ) + code_modifications = __load_code_storage_object(os.path.join(str(arguments["--code-export-path"]), file_name)) __apply_modifications( arguments["--project"], working_copy_dir, @@ -73,7 +69,7 @@ def execute_single_model(arguments: Dict): # __cleanup(working_copy_dir) -def __initialize_measurement_directory(arguments: Dict): +def __initialize_measurement_directory(arguments: Dict[str, Any]): measurement_dir = os.path.join(arguments["--project"], ".discopop_optimizer_measurements") if not arguments["--execution-append-measurements"]: # delete measurement directory @@ -92,9 +88,7 @@ def __initialize_measurement_file(measurement_file: str): f.write(header_line) -def __measure_and_execute( - arguments: Dict, working_copy_dir: str, code_mod_object: CodeStorageObject -): +def __measure_and_execute(arguments: Dict[str, Any], working_copy_dir: str, code_mod_object: CodeStorageObject): """Setup measurements, execute the compiled program and output the measurement results to a file""" measurement_dir = os.path.join(arguments["--project"], ".discopop_optimizer_measurements") # create output file for specific model measurement @@ -135,9 +129,7 @@ def __measure_and_execute( print("\t\t\tVariance: ", statistics.variance(execution_times)) -def __execute( - arguments: Dict, working_copy_dir: str, measurements_file: TextIO -) -> Tuple[int, float, float]: +def __execute(arguments: Dict[str, Any], working_copy_dir: str, measurements_file: TextIO) -> Tuple[int, float, float]: """Executes the current model and returns the exit code as well as the start and end time of the execution""" print("\t\texecuting...") command = ["./" + arguments["--executable-name"], arguments["--executable-arguments"]] @@ -165,7 +157,7 @@ def __execute( return 1, start_time, end_time -def __compile(arguments: Dict, working_copy_dir, compile_command): +def __compile(arguments: Dict[str, Any], working_copy_dir, compile_command): print("\t\tbuilding...") # command = compile_command # command = shlex.split(compile_command) diff --git a/discopop_library/discopop_optimizer/gui/plotting/CostModels.py b/discopop_library/discopop_optimizer/gui/plotting/CostModels.py index abe756759..084fdaf6a 100644 --- a/discopop_library/discopop_optimizer/gui/plotting/CostModels.py +++ b/discopop_library/discopop_optimizer/gui/plotting/CostModels.py @@ -8,17 +8,16 @@ import copy from typing import List, Dict, Tuple, Optional, cast +import matplotlib # type: ignore import numpy as np +import sympy from matplotlib import pyplot as plt # type: ignore -import matplotlib from spb import plot3d, MB, plot # type: ignore +from sympy import Integer from sympy import Symbol, Expr -import sympy from discopop_library.discopop_optimizer.CostModels.CostModel import CostModel from discopop_library.discopop_optimizer.Variables.Experiment import Experiment -from discopop_library.discopop_optimizer.utilities.MOGUtilities import data_at, show -from sympy import Integer def plot_CostModels( @@ -45,12 +44,12 @@ def plot_CostModels( local_substitutions = copy.deepcopy(experiment.substitutions) for function in experiment.selected_paths_per_function: # register substitution - local_substitutions[ - cast(Symbol, function.sequential_costs) - ] = experiment.selected_paths_per_function[function][0].sequential_costs - local_substitutions[ - cast(Symbol, function.parallelizable_costs) - ] = experiment.selected_paths_per_function[function][0].parallelizable_costs + local_substitutions[cast(Symbol, function.sequential_costs)] = experiment.selected_paths_per_function[function][ + 0 + ].sequential_costs + local_substitutions[cast(Symbol, function.parallelizable_costs)] = experiment.selected_paths_per_function[ + function + ][0].parallelizable_costs local_models = copy.deepcopy(models) @@ -119,12 +118,12 @@ def plot_CostModels_using_function_path_selections( local_substitutions = copy.deepcopy(experiment.substitutions) for function in experiment.selected_paths_per_function: # register substitution - local_substitutions[ - cast(Symbol, function.sequential_costs) - ] = experiment.selected_paths_per_function[function][0].sequential_costs - local_substitutions[ - cast(Symbol, function.parallelizable_costs) - ] = experiment.selected_paths_per_function[function][0].parallelizable_costs + local_substitutions[cast(Symbol, function.sequential_costs)] = experiment.selected_paths_per_function[function][ + 0 + ].sequential_costs + local_substitutions[cast(Symbol, function.parallelizable_costs)] = experiment.selected_paths_per_function[ + function + ][0].parallelizable_costs local_models = copy.deepcopy(models) @@ -227,12 +226,12 @@ def print_simplified_function( local_substitutions = copy.deepcopy(experiment.substitutions) for function in experiment.selected_paths_per_function: # register substitution - local_substitutions[ - cast(Symbol, function.sequential_costs) - ] = experiment.selected_paths_per_function[function][0].sequential_costs - local_substitutions[ - cast(Symbol, function.parallelizable_costs) - ] = experiment.selected_paths_per_function[function][0].parallelizable_costs + local_substitutions[cast(Symbol, function.sequential_costs)] = experiment.selected_paths_per_function[function][ + 0 + ].sequential_costs + local_substitutions[cast(Symbol, function.parallelizable_costs)] = experiment.selected_paths_per_function[ + function + ][0].parallelizable_costs local_models = copy.deepcopy(models) diff --git a/discopop_library/discopop_optimizer/gui/presentation/ChoiceDetails.py b/discopop_library/discopop_optimizer/gui/presentation/ChoiceDetails.py index f7f0dd54c..03781dc8f 100644 --- a/discopop_library/discopop_optimizer/gui/presentation/ChoiceDetails.py +++ b/discopop_library/discopop_optimizer/gui/presentation/ChoiceDetails.py @@ -5,22 +5,18 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. +from tkinter import * from typing import Optional import networkx as nx # type: ignore from discopop_library.discopop_optimizer.CostModels.CostModel import CostModel - -from tkinter import * - from discopop_library.discopop_optimizer.gui.widgets.ScrollableFrame import ScrollableFrameWidget from discopop_library.discopop_optimizer.utilities.MOGUtilities import data_at from discopop_wizard.screens.widgets.ScrollableText import ScrollableTextWidget -def display_choices_for_model( - graph: nx.DiGraph, model: CostModel, window_title: Optional[str] = None -): +def display_choices_for_model(graph: nx.DiGraph, model: CostModel, window_title: Optional[str] = None): root = Tk() if window_title is not None: root.configure() diff --git a/discopop_library/discopop_optimizer/gui/presentation/OptionTable.py b/discopop_library/discopop_optimizer/gui/presentation/OptionTable.py index a3838c1c8..c0c94584b 100644 --- a/discopop_library/discopop_optimizer/gui/presentation/OptionTable.py +++ b/discopop_library/discopop_optimizer/gui/presentation/OptionTable.py @@ -7,10 +7,10 @@ # directory for details. import tkinter from tkinter import * -from typing import List, Tuple, Dict, Union, Optional, cast +from typing import List, Tuple, Dict, Optional, cast import networkx as nx # type: ignore -from sympy import Symbol, Expr +from sympy import Symbol from discopop_explorer.PETGraphX import PETGraphX from discopop_library.discopop_optimizer.CostModels.CostModel import CostModel @@ -158,9 +158,7 @@ def show_options( window_title, ), ).grid() - Button( - root, text="Save Models", command=lambda: __save_models(experiment, function_root, options) - ).grid() + Button(root, text="Save Models", command=lambda: __save_models(experiment, function_root, options)).grid() Button( root, text="Export all codes", @@ -235,9 +233,7 @@ def __update_selection(cm, ctx): update_selection_button = Button( options_field, text="Update selection", - command=lambda opt=option, opt_name=option_name, ctx=context: __update_selection( # type: ignore - opt, ctx - ), + command=lambda opt=option, opt_name=option_name, ctx=context: __update_selection(opt, ctx), # type: ignore ) update_selection_button.grid(row=0, column=3) diff --git a/discopop_library/discopop_optimizer/gui/queries/ValueTableQuery.py b/discopop_library/discopop_optimizer/gui/queries/ValueTableQuery.py index dd777e877..34f7af283 100644 --- a/discopop_library/discopop_optimizer/gui/queries/ValueTableQuery.py +++ b/discopop_library/discopop_optimizer/gui/queries/ValueTableQuery.py @@ -5,13 +5,11 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -import sys -from typing import List, Tuple, Optional, Dict, cast +import tkinter as tk +from tkinter import * +from typing import List, Tuple, Optional, Dict, cast, Union from sympy import Symbol, Expr -from tkinter import * -from tkinter import ttk -import tkinter as tk from discopop_library.discopop_optimizer.classes.enums.Distributions import FreeSymbolDistribution from discopop_library.discopop_optimizer.gui.widgets.ScrollableFrame import ScrollableFrameWidget @@ -20,13 +18,9 @@ def query_user_for_symbol_values( symbols: List[Symbol], suggested_values: Dict[Symbol, Expr], - arguments: Dict, + arguments: Dict[str, Union[str, bool]], parent_frame: Optional[tk.Frame], -) -> List[ - Tuple[ - Symbol, Optional[float], Optional[float], Optional[float], Optional[FreeSymbolDistribution] - ] -]: +) -> List[Tuple[Symbol, Optional[float], Optional[float], Optional[float], Optional[FreeSymbolDistribution]]]: """Opens a GUI-Table to query values for each given Symbol from the user. The queried values are: Specific value, Range start, Range end. In every case, either a specific value, or a range must be given. diff --git a/discopop_library/discopop_optimizer/gui/widgets/ScrollableFrame.py b/discopop_library/discopop_optimizer/gui/widgets/ScrollableFrame.py index 9cec33f6b..56cbec48b 100644 --- a/discopop_library/discopop_optimizer/gui/widgets/ScrollableFrame.py +++ b/discopop_library/discopop_optimizer/gui/widgets/ScrollableFrame.py @@ -29,24 +29,18 @@ def __init__(self, parent_frame): self.scrollable_frame.rowconfigure(0, weight=1) self.scrollable_frame.columnconfigure(0, weight=1) - self.scrollable_frame.bind( - "", lambda e: self.canvas.configure(scrollregion=self.canvas.bbox("all")) - ) + self.scrollable_frame.bind("", lambda e: self.canvas.configure(scrollregion=self.canvas.bbox("all"))) self.canvas.create_window((0, 0), window=self.scrollable_frame, anchor="nw") self.canvas.configure(yscrollcommand=self.scrollbar.set) - def finalize( - self, row_count: int, row: int = 0, col: int = 0, rowspan: int = 1, columnspan: int = 1 - ): + def finalize(self, row_count: int, row: int = 0, col: int = 0, rowspan: int = 1, columnspan: int = 1): if rowspan < 1: rowspan = 1 if columnspan < 1: columnspan = 1 - self.container.grid( - row=row, column=col, columnspan=columnspan, rowspan=rowspan, sticky=tk.NSEW - ) + self.container.grid(row=row, column=col, columnspan=columnspan, rowspan=rowspan, sticky=tk.NSEW) self.canvas.grid(row=0, column=0, sticky=tk.NSEW) self.scrollbar.grid(row=0, rowspan=max(row_count, 1), column=1, sticky=tk.NS) diff --git a/discopop_library/discopop_optimizer/scheduling/workload_delta.py b/discopop_library/discopop_optimizer/scheduling/workload_delta.py index 4e6c3a517..ad1cbd298 100644 --- a/discopop_library/discopop_optimizer/scheduling/workload_delta.py +++ b/discopop_library/discopop_optimizer/scheduling/workload_delta.py @@ -5,7 +5,6 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -import sys from typing import List, Tuple, cast, Any, Union, Optional import networkx as nx # type: ignore @@ -23,7 +22,6 @@ get_children, get_successors, ) -from discopop_library.discopop_optimizer.utilities.visualization.plotting import show # define aliases for type checking purposes @@ -36,9 +34,7 @@ class BranchCosts(RegularCosts): class WorkloadStack(object): - stack: List[ - Union[List, RegularCosts] - ] # List of WorkloadStacks or integer tuples (min_wl, max_wl) + stack: List[Union[List[Any], RegularCosts]] # List of WorkloadStacks or integer tuples (min_wl, max_wl) def __init__(self): self.stack = [] @@ -141,7 +137,7 @@ def get_max_workload(self) -> int: max_wl_sum += entry[1] return max_wl_sum - def __get_innermost_workload_stack(self) -> Tuple[List, List[int]]: + def __get_innermost_workload_stack(self) -> Tuple[List[Any], List[int]]: """identifies and returns a reference to the currently active (innermost) workload stack stored in self.stack, as well as the list of indices to access this element""" indices: List[int] = [] @@ -160,9 +156,9 @@ def __get_innermost_workload_stack(self) -> Tuple[List, List[int]]: break return innermost_stack, indices - def __get_parent_of_innermost_stack(self) -> Optional[List]: + def __get_parent_of_innermost_stack(self) -> Optional[List[Any]]: """identifies and returns a reference to the parent List of the innermost workload stack""" - parents: List[Optional[List]] = [None] + parents: List[Optional[List[Any]]] = [None] innermost_stack = self.stack while True: if len(innermost_stack) == 0: @@ -237,9 +233,7 @@ def __parse_node( else: iteration_factor = 1 for child_id in get_children(experiment.optimization_graph, node_id): - workload_stack = __parse_node( - experiment, child_id, workload_stack, iteration_factor=iteration_factor - ) + workload_stack = __parse_node(experiment, child_id, workload_stack, iteration_factor=iteration_factor) # set next node successors = get_successors(experiment.optimization_graph, node_id) diff --git a/discopop_library/discopop_optimizer/suggestions/importers/do_all.py b/discopop_library/discopop_optimizer/suggestions/importers/do_all.py index 638be66d9..4c6272294 100644 --- a/discopop_library/discopop_optimizer/suggestions/importers/do_all.py +++ b/discopop_library/discopop_optimizer/suggestions/importers/do_all.py @@ -13,7 +13,6 @@ from discopop_library.discopop_optimizer.CostModels.CostModel import CostModel from discopop_library.discopop_optimizer.Microbench.utils import ( - convert_microbench_to_discopop_workload, convert_discopop_to_microbench_workload, ) from discopop_library.discopop_optimizer.Variables.Experiment import Experiment @@ -50,9 +49,7 @@ def import_suggestion( node_data_copy.execute_in_parallel = True # copy loop iteration variable - cast(Loop, node_data_copy).iterations_symbol = cast( - Loop, node_data_copy - ).iterations_symbol + cast(Loop, node_data_copy).iterations_symbol = cast(Loop, node_data_copy).iterations_symbol # add suggestion to node data node_data_copy.suggestion = suggestion node_data_copy.suggestion_type = "do_all" @@ -104,9 +101,7 @@ def import_suggestion( return graph -def get_cost_multiplier( - node_id: int, environment: Experiment, device_id: int -) -> Tuple[CostModel, List[Symbol]]: +def get_cost_multiplier(node_id: int, environment: Experiment, device_id: int) -> Tuple[CostModel, List[Symbol]]: """Creates and returns the multiplier to represent the effects of the given suggestion on the cost model. A CostModel object is used to store the information on the path selection. Returns the multiplier and the list of introduces symbols @@ -123,9 +118,7 @@ def get_cost_multiplier( return cm, [] -def get_overhead_term( - node_data: Loop, environment: Experiment, device_id: int -) -> Tuple[CostModel, List[Symbol]]: +def get_overhead_term(node_data: Loop, environment: Experiment, device_id: int) -> Tuple[CostModel, List[Symbol]]: """Creates and returns the Expression which represents the Overhead incurred by the given suggestion. For testing purposes, the following function is used to represent the overhead incurred by a do-all loop. The function has been created using Extra-P. @@ -140,9 +133,7 @@ def get_overhead_term( # since node_data is of type Loop, parallelizable_workload must be set per_iteration_workload = cast(Expr, node_data.parallelizable_workload) # convert DiscoPoP workload to Microbench workload - converted_per_iteration_workload = convert_discopop_to_microbench_workload( - per_iteration_workload, iterations - ) + converted_per_iteration_workload = convert_discopop_to_microbench_workload(per_iteration_workload, iterations) substitutions: Dict[Symbol, Expr] = {} @@ -159,9 +150,7 @@ def get_overhead_term( substituted_overhead_model = overhead_model.xreplace(substitutions) # register symbol for overhead - doall_overhead_symbol = Symbol( - "doall_" + str(node_data.node_id) + "_pos_" + str(node_data.position) + "_overhead" - ) + doall_overhead_symbol = Symbol("doall_" + str(node_data.node_id) + "_pos_" + str(node_data.position) + "_overhead") environment.substitutions[doall_overhead_symbol] = substituted_overhead_model diff --git a/discopop_library/discopop_optimizer/suggestions/importers/reduction.py b/discopop_library/discopop_optimizer/suggestions/importers/reduction.py index 714ab3c49..96e00174e 100644 --- a/discopop_library/discopop_optimizer/suggestions/importers/reduction.py +++ b/discopop_library/discopop_optimizer/suggestions/importers/reduction.py @@ -13,7 +13,6 @@ from discopop_library.discopop_optimizer.CostModels.CostModel import CostModel from discopop_library.discopop_optimizer.Microbench.utils import ( - convert_microbench_to_discopop_workload, convert_discopop_to_microbench_workload, ) from discopop_library.discopop_optimizer.Variables.Experiment import Experiment @@ -43,9 +42,7 @@ def import_suggestion( # mark node for parallel execution node_data_copy.execute_in_parallel = True # copy loop iteration variable - cast(Loop, node_data_copy).iterations_symbol = cast( - Loop, node_data_copy - ).iterations_symbol + cast(Loop, node_data_copy).iterations_symbol = cast(Loop, node_data_copy).iterations_symbol # add suggestion to node data node_data_copy.suggestion = suggestion node_data_copy.suggestion_type = "reduction" @@ -82,9 +79,7 @@ def import_suggestion( return graph -def get_cost_multiplier( - node_id: int, environment: Experiment, device_id: int -) -> Tuple[CostModel, List[Symbol]]: +def get_cost_multiplier(node_id: int, environment: Experiment, device_id: int) -> Tuple[CostModel, List[Symbol]]: """Creates and returns the multiplier to represent the effects of the given suggestion on the cost model. A CostModel object is used to store the information on the path selection. Returns the multiplier and the list of introduces symbols @@ -100,9 +95,7 @@ def get_cost_multiplier( return cm, [] -def get_overhead_term( - node_data: Loop, environment: Experiment, device_id: int -) -> Tuple[CostModel, List[Symbol]]: +def get_overhead_term(node_data: Loop, environment: Experiment, device_id: int) -> Tuple[CostModel, List[Symbol]]: """Creates and returns the Expression which represents the Overhead incurred by the given suggestion. For testing purposes, the following function is used to represent the overhead incurred by a do-all loop. The function has been created using Extra-P. @@ -118,9 +111,7 @@ def get_overhead_term( # since node_data is of type Loop, parallelizable_workload has to exist per_iteration_workload = cast(Expr, node_data.parallelizable_workload) # convert DiscoPoP workload to Microbench workload - converted_per_iteration_workload = convert_discopop_to_microbench_workload( - per_iteration_workload, iterations - ) + converted_per_iteration_workload = convert_discopop_to_microbench_workload(per_iteration_workload, iterations) substitutions: Dict[Symbol, Expr] = {} diff --git a/discopop_library/discopop_optimizer/utilities/MOGUtilities.py b/discopop_library/discopop_optimizer/utilities/MOGUtilities.py index f5884d7b5..48588cf20 100644 --- a/discopop_library/discopop_optimizer/utilities/MOGUtilities.py +++ b/discopop_library/discopop_optimizer/utilities/MOGUtilities.py @@ -5,13 +5,11 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -import random -from typing import List, cast, Set, Optional, Tuple +from typing import List, cast, Set, Tuple +import matplotlib # type: ignore import matplotlib.pyplot as plt # type:ignore -import matplotlib import networkx as nx # type: ignore -import sympy from discopop_explorer.PETGraphX import MemoryRegion, NodeID from discopop_library.discopop_optimizer.classes.edges.ChildEdge import ChildEdge @@ -46,73 +44,42 @@ def get_edge_data(graph: nx.DiGraph, source: int, target: int) -> GenericEdge: def get_successors(graph: nx.DiGraph, node_id: int) -> List[int]: """Returns a list of node ids for the successors of the given node""" - return [ - edge[1] - for edge in graph.out_edges(node_id, data="data") - if isinstance(edge[2], SuccessorEdge) - ] + return [edge[1] for edge in graph.out_edges(node_id, data="data") if isinstance(edge[2], SuccessorEdge)] def get_temporary_successors(graph: nx.DiGraph, node_id: int) -> List[int]: """Returns a list of node ids for the temporary successors of the given node""" - return [ - edge[1] - for edge in graph.out_edges(node_id, data="data") - if isinstance(edge[2], TemporaryEdge) - ] + return [edge[1] for edge in graph.out_edges(node_id, data="data") if isinstance(edge[2], TemporaryEdge)] def get_predecessors(graph: nx.DiGraph, node_id: int) -> List[int]: """Returns a list of node ids for the predecessors of the given node""" - return [ - edge[0] - for edge in graph.in_edges(node_id, data="data") - if isinstance(edge[2], SuccessorEdge) - ] + return [edge[0] for edge in graph.in_edges(node_id, data="data") if isinstance(edge[2], SuccessorEdge)] def get_children(graph: nx.DiGraph, node_id: int) -> List[int]: """Returns a list of node ids for the children of the given node""" - return [ - edge[1] for edge in graph.out_edges(node_id, data="data") if isinstance(edge[2], ChildEdge) - ] + return [edge[1] for edge in graph.out_edges(node_id, data="data") if isinstance(edge[2], ChildEdge)] def get_out_options(graph: nx.DiGraph, node_id: int) -> List[int]: """Returns a list of node ids for the parallelization options of the given node""" - return [ - edge[1] for edge in graph.out_edges(node_id, data="data") if isinstance(edge[2], OptionEdge) - ] + return [edge[1] for edge in graph.out_edges(node_id, data="data") if isinstance(edge[2], OptionEdge)] def get_in_options(graph: nx.DiGraph, node_id: int) -> List[int]: """Returns a list of node ids for the parallelization options of the given node""" - return [ - edge[1] for edge in graph.in_edges(node_id, data="data") if isinstance(edge[2], OptionEdge) - ] + return [edge[1] for edge in graph.in_edges(node_id, data="data") if isinstance(edge[2], OptionEdge)] def get_requirements(graph: nx.DiGraph, node_id: int) -> List[int]: """Returns a list of node ids for the requirements of the parallelization option in the given node""" - return [ - edge[1] - for edge in graph.out_edges(node_id, data="data") - if isinstance(edge[2], RequirementEdge) - ] + return [edge[1] for edge in graph.out_edges(node_id, data="data") if isinstance(edge[2], RequirementEdge)] def has_temporary_successor(graph: nx.DiGraph, node_id: int) -> bool: """Checks whether the given node has outgoing temporary successor edges""" - return ( - len( - [ - edge - for edge in graph.out_edges(node_id, data="data") - if isinstance(edge[2], TemporaryEdge) - ] - ) - > 0 - ) + return len([edge for edge in graph.out_edges(node_id, data="data") if isinstance(edge[2], TemporaryEdge)]) > 0 def show(graph): @@ -183,18 +150,10 @@ def show(graph): node_size=200, node_color="#2B85FD", node_shape="o", - nodelist=[ - n - for n in graph.nodes - if isinstance(data_at(graph, n), Workload) and n not in drawn_nodes - ], - ) - node_ids[Workload] = [ - n for n in graph.nodes if isinstance(data_at(graph, n), Workload) and n not in drawn_nodes - ] - drawn_nodes.update( - [n for n in graph.nodes if isinstance(data_at(graph, n), Workload) and n not in drawn_nodes] + nodelist=[n for n in graph.nodes if isinstance(data_at(graph, n), Workload) and n not in drawn_nodes], ) + node_ids[Workload] = [n for n in graph.nodes if isinstance(data_at(graph, n), Workload) and n not in drawn_nodes] + drawn_nodes.update([n for n in graph.nodes if isinstance(data_at(graph, n), Workload) and n not in drawn_nodes]) # id as label labels = {} @@ -378,12 +337,8 @@ def get_read_and_written_data_from_subgraph( written_memory_regions.update(writes) # add reads and writes of the node itself node_data = data_at(graph, node_id) - read_memory_regions.update( - [read_access.memory_region for read_access in node_data.read_memory_regions] - ) - written_memory_regions.update( - [write_access.memory_region for write_access in node_data.written_memory_regions] - ) + read_memory_regions.update([read_access.memory_region for read_access in node_data.read_memory_regions]) + written_memory_regions.update([write_access.memory_region for write_access in node_data.written_memory_regions]) return read_memory_regions, written_memory_regions @@ -398,9 +353,7 @@ def get_parents(graph: nx.DiGraph, node_id: int) -> List[int]: preds = get_predecessors(graph, current_node) parents = [ - cast(int, edge[0]) - for edge in graph.in_edges(current_node, data="data") - if isinstance(edge[2], ChildEdge) + cast(int, edge[0]) for edge in graph.in_edges(current_node, data="data") if isinstance(edge[2], ChildEdge) ] return parents @@ -416,9 +369,7 @@ def get_all_parents(graph: nx.DiGraph, node_id: int) -> List[int]: current = queue.pop() all_parents.add(current) new_parents = [ - parent - for parent in get_parents(graph, current) - if parent not in queue and parent not in all_parents + parent for parent in get_parents(graph, current) if parent not in queue and parent not in all_parents ] queue.update(new_parents) return list(all_parents) diff --git a/discopop_library/discopop_optimizer/utilities/optimization/GlobalOptimization/RandomSamples.py b/discopop_library/discopop_optimizer/utilities/optimization/GlobalOptimization/RandomSamples.py index 3cb2a16c3..c202c9901 100644 --- a/discopop_library/discopop_optimizer/utilities/optimization/GlobalOptimization/RandomSamples.py +++ b/discopop_library/discopop_optimizer/utilities/optimization/GlobalOptimization/RandomSamples.py @@ -6,13 +6,11 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. import copy -import random -from random import shuffle from typing import List, Dict, Tuple import networkx as nx # type: ignore from spb import plot3d, MB, plot # type: ignore -from sympy import Symbol, Expr +from sympy import Symbol from discopop_library.discopop_optimizer.CostModels.CostModel import CostModel from discopop_library.discopop_optimizer.CostModels.utilities import get_random_path @@ -21,8 +19,6 @@ from discopop_library.discopop_optimizer.classes.context.ContextObject import ContextObject from discopop_library.discopop_optimizer.classes.enums.Distributions import FreeSymbolDistribution from discopop_library.discopop_optimizer.classes.nodes.FunctionRoot import FunctionRoot -from discopop_library.discopop_optimizer.gui.plotting.CostModels import plot_CostModels -from discopop_library.discopop_optimizer.utilities.MOGUtilities import show def find_quasi_optimal_using_random_samples( @@ -49,9 +45,7 @@ def find_quasi_optimal_using_random_samples( experiment.substitutions = copy.deepcopy(substitutions_buffer) tmp_dict = dict() - tmp_dict[function_root] = [ - get_random_path(experiment, graph, function_root.node_id, must_contain=None) - ] + tmp_dict[function_root] = [get_random_path(experiment, graph, function_root.node_id, must_contain=None)] try: random_paths.append(calculate_data_transfers(graph, tmp_dict)[function_root][0]) i += 1 diff --git a/discopop_library/discopop_optimizer/utilities/optimization/LocalOptimization/TopDown.py b/discopop_library/discopop_optimizer/utilities/optimization/LocalOptimization/TopDown.py index 1e08492e7..3aa095f2d 100644 --- a/discopop_library/discopop_optimizer/utilities/optimization/LocalOptimization/TopDown.py +++ b/discopop_library/discopop_optimizer/utilities/optimization/LocalOptimization/TopDown.py @@ -26,7 +26,6 @@ get_successors, get_children, data_at, - show, ) @@ -40,9 +39,7 @@ def get_locally_optimized_models( ) -> Dict[FunctionRoot, List[Tuple[CostModel, ContextObject]]]: result_dict: Dict[FunctionRoot, List[Tuple[CostModel, ContextObject]]] = dict() all_function_ids = get_all_function_nodes(graph) - all_function_nodes: List[FunctionRoot] = [ - cast(FunctionRoot, data_at(graph, fn_id)) for fn_id in all_function_ids - ] + all_function_nodes: List[FunctionRoot] = [cast(FunctionRoot, data_at(graph, fn_id)) for fn_id in all_function_ids] for function_node in all_function_ids: # get a list of all decisions that have to be made decisions_to_be_made = __find_decisions(graph, function_node) @@ -86,15 +83,11 @@ def get_locally_optimized_models( continue # iteratively apply variable substitutions - decision_models_with_substitutions: List[ - Tuple[int, Tuple[CostModel, ContextObject, CostModel]] - ] = [] + decision_models_with_substitutions: List[Tuple[int, Tuple[CostModel, ContextObject, CostModel]]] = [] # initialize substitution for decision, pair in decision_models: model, context = pair - decision_models_with_substitutions.append( - (decision, (model, context, copy.deepcopy(model))) - ) + decision_models_with_substitutions.append((decision, (model, context, copy.deepcopy(model)))) modification_found = True while modification_found: @@ -106,17 +99,13 @@ def get_locally_optimized_models( tmp_model = substituted_model.parallelizable_costs.subs(substitutions) if tmp_model != substituted_model.parallelizable_costs: modification_found = True - substituted_model.parallelizable_costs = ( - substituted_model.parallelizable_costs.subs(substitutions) - ) + substituted_model.parallelizable_costs = substituted_model.parallelizable_costs.subs(substitutions) # apply substitutions to sequential costs tmp_model = substituted_model.sequential_costs.subs(substitutions) if tmp_model != substituted_model.sequential_costs: modification_found = True - substituted_model.sequential_costs = substituted_model.sequential_costs.subs( - substitutions - ) + substituted_model.sequential_costs = substituted_model.sequential_costs.subs(substitutions) # decision_models_with_substitutions.append( ## (decision, (model, context, substituted_model)) @@ -161,9 +150,7 @@ def get_locally_optimized_models( ) # calculate and append costs of data transfers to the performance models - complete_performance_models = add_data_transfer_costs( - graph, performance_models_with_transfers, environment - ) + complete_performance_models = add_data_transfer_costs(graph, performance_models_with_transfers, environment) # merge dictionaries result_dict = {**result_dict, **complete_performance_models} diff --git a/discopop_library/discopop_optimizer/utilities/visualization/plotting.py b/discopop_library/discopop_optimizer/utilities/visualization/plotting.py index 5ec68d9ef..01d5294bf 100644 --- a/discopop_library/discopop_optimizer/utilities/visualization/plotting.py +++ b/discopop_library/discopop_optimizer/utilities/visualization/plotting.py @@ -5,6 +5,9 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. +import matplotlib.pyplot as plt # type: ignore +import networkx as nx # type: ignore + from discopop_library.discopop_optimizer.classes.edges.ChildEdge import ChildEdge from discopop_library.discopop_optimizer.classes.edges.OptionEdge import OptionEdge from discopop_library.discopop_optimizer.classes.edges.RequirementEdge import RequirementEdge @@ -16,9 +19,6 @@ from discopop_library.discopop_optimizer.classes.nodes.Workload import Workload from discopop_library.discopop_optimizer.utilities.MOGUtilities import data_at -import matplotlib.pyplot as plt # type: ignore -import networkx as nx # type: ignore - def show(graph): """Plots the graph @@ -87,18 +87,10 @@ def show(graph): node_size=200, node_color="#2B85FD", node_shape="o", - nodelist=[ - n - for n in graph.nodes - if isinstance(data_at(graph, n), Workload) and n not in drawn_nodes - ], - ) - node_ids[Workload] = [ - n for n in graph.nodes if isinstance(data_at(graph, n), Workload) and n not in drawn_nodes - ] - drawn_nodes.update( - [n for n in graph.nodes if isinstance(data_at(graph, n), Workload) and n not in drawn_nodes] + nodelist=[n for n in graph.nodes if isinstance(data_at(graph, n), Workload) and n not in drawn_nodes], ) + node_ids[Workload] = [n for n in graph.nodes if isinstance(data_at(graph, n), Workload) and n not in drawn_nodes] + drawn_nodes.update([n for n in graph.nodes if isinstance(data_at(graph, n), Workload) and n not in drawn_nodes]) # id as label labels = {} diff --git a/discopop_library/global_data/version/VERSION b/discopop_library/global_data/version/VERSION index ccbccc3dc..276cbf9e2 100644 --- a/discopop_library/global_data/version/VERSION +++ b/discopop_library/global_data/version/VERSION @@ -1 +1 @@ -2.2.0 +2.3.0 diff --git a/discopop_library/result_classes/DetectionResult.py b/discopop_library/result_classes/DetectionResult.py index 3db720003..8feed82de 100644 --- a/discopop_library/result_classes/DetectionResult.py +++ b/discopop_library/result_classes/DetectionResult.py @@ -8,6 +8,7 @@ from typing import List import jsonpickle # type: ignore + from discopop_explorer.PETGraphX import PETGraphX from discopop_explorer.pattern_detectors.PatternInfo import PatternInfo from discopop_explorer.pattern_detectors.do_all_detector import DoAllInfo diff --git a/discopop_profiler/__init__.py b/discopop_profiler/__init__.py index 93008121e..9a57e9dde 100644 --- a/discopop_profiler/__init__.py +++ b/discopop_profiler/__init__.py @@ -35,10 +35,7 @@ def update_filemapping(self): # Do not regenerate if FileMapping.txt is still up-to-date. filemapping_mtime = os.stat("FileMapping.txt").st_mtime if os.stat(cwd).st_mtime < filemapping_mtime and all( - [ - not entry.is_dir() or entry.stat().st_mtime < filemapping_mtime - for entry in recursive_scandir(cwd) - ] + [not entry.is_dir() or entry.stat().st_mtime < filemapping_mtime for entry in recursive_scandir(cwd)] ): return logging.info("Generating FileMapping.txt.") @@ -82,7 +79,7 @@ def wrap_clang_args(self, clang_args: List[str]) -> List[str]: ] return args - def invoke(self, clang_args: List[str]) -> subprocess.CompletedProcess: + def invoke(self, clang_args: List[str]) -> subprocess.CompletedProcess: # type: ignore args = self.wrap_clang_args(clang_args) logging.info(" ".join(args)) return subprocess.run(args) diff --git a/discopop_profiler/__main__.py b/discopop_profiler/__main__.py index c50636bf6..c00ad4690 100644 --- a/discopop_profiler/__main__.py +++ b/discopop_profiler/__main__.py @@ -12,8 +12,8 @@ import logging import shutil -from . import DiscopopCpp from discopop_library.global_data.version.utils import get_version +from . import DiscopopCpp PROG = "discopop_profiler" diff --git a/discopop_profiler/utils.py b/discopop_profiler/utils.py index 9d4fe684e..5ee7fb686 100644 --- a/discopop_profiler/utils.py +++ b/discopop_profiler/utils.py @@ -37,16 +37,14 @@ def get_library(name: str) -> str: def is_compile(clang_args: List[str]) -> bool: - return "-c" in clang_args or any( - [re.match(r"^[^-].+\.(?:c|cc|cpp)$", arg) for arg in clang_args] - ) + return "-c" in clang_args or any([re.match(r"^[^-].+\.(?:c|cc|cpp)$", arg) for arg in clang_args]) def is_link(clang_args: List[str]) -> bool: return "-c" not in clang_args -def recursive_scandir(path: str) -> Iterator[os.DirEntry]: +def recursive_scandir(path: str) -> Iterator[os.DirEntry]: # type: ignore with os.scandir(path) as dir_iter: for entry in dir_iter: yield entry diff --git a/discopop_wizard/classes/CodePreview.py b/discopop_wizard/classes/CodePreview.py index a66a4197a..f2bbbd610 100644 --- a/discopop_wizard/classes/CodePreview.py +++ b/discopop_wizard/classes/CodePreview.py @@ -6,15 +6,12 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -import copy -import sys -from pathlib import Path -from typing import Tuple, Optional, List, Dict, Sequence, cast import tkinter as tk +from pathlib import Path +from typing import Optional, List, cast from discopop_library.CodeGenerator.classes.ContentBuffer import ContentBuffer from discopop_library.CodeGenerator.classes.Line import Line -from discopop_wizard.classes.Pragma import Pragma, PragmaPosition class CodePreviewLine(Line): @@ -98,9 +95,7 @@ def show_in(self, parent_element: tk.Text): for line_idx, line in enumerate(self.lines): # offset line_id to account for start with 1 offset_line_id = line_idx + 1 - cast(CodePreviewLine, line).display( - self.wizard, parent_element, offset_line_id, self.max_line_num - ) + cast(CodePreviewLine, line).display(self.wizard, parent_element, offset_line_id, self.max_line_num) def jump_to_first_modification(self, parent_element: tk.Text): """Jumps to the location of the first modified source code location.""" diff --git a/discopop_wizard/classes/Console.py b/discopop_wizard/classes/Console.py index 45fb05e94..f6fd37a1e 100644 --- a/discopop_wizard/classes/Console.py +++ b/discopop_wizard/classes/Console.py @@ -56,9 +56,7 @@ def __show_console(self): y_scrollbar = ttk.Scrollbar(self.parent_frame, command=self.log_screen.yview) y_scrollbar.grid(row=0, column=1, sticky="nsew") self.log_screen["yscrollcommand"] = y_scrollbar.set - x_scrollbar = ttk.Scrollbar( - self.parent_frame, orient="horizontal", command=self.log_screen.xview - ) + x_scrollbar = ttk.Scrollbar(self.parent_frame, orient="horizontal", command=self.log_screen.xview) x_scrollbar.grid(row=1, column=0, columnspan=2, sticky="nsew") self.log_screen["xscrollcommand"] = x_scrollbar.set diff --git a/discopop_wizard/classes/ExecutionConfiguration.py b/discopop_wizard/classes/ExecutionConfiguration.py index ad977662e..77ac3dab7 100644 --- a/discopop_wizard/classes/ExecutionConfiguration.py +++ b/discopop_wizard/classes/ExecutionConfiguration.py @@ -13,7 +13,7 @@ import tkinter as tk from json import JSONDecodeError from tkinter import filedialog -from typing import TextIO, List, Dict, Tuple +from typing import TextIO, List, Dict, Tuple, Any import jsons # type:ignore @@ -28,7 +28,7 @@ class ExecutionConfiguration(object): button: tk.Button - value_dict: Dict + value_dict: Dict[str, Any] def __init__(self, wizard): self.value_dict = { @@ -93,14 +93,10 @@ def __extract_values_from_help_string(self): self.value_dict[target_name + "_" + value_name] = "" # self.type_dict[target_name + "_" + value_name] = value_type - def get_as_button( - self, wizard, main_screen_obj, parent_frame: tk.Frame, all_buttons: List[tk.Button] - ) -> tk.Button: + def get_as_button(self, wizard, main_screen_obj, parent_frame: tk.Frame, all_buttons: List[tk.Button]) -> tk.Button: button = tk.Button(parent_frame, text=self.value_dict["label"]) button.config( - command=lambda: self.highlight_and_update_notebook_screens( - wizard, main_screen_obj, button, all_buttons - ) + command=lambda: self.highlight_and_update_notebook_screens(wizard, main_screen_obj, button, all_buttons) ) self.button = button return button @@ -129,12 +125,8 @@ def highlight_and_update_notebook_screens( self.show_details_screen(wizard, main_screen_obj) # update results screen of pressed configuration button and set results tab state based on result existence - main_screen_obj.notebook.tab( - main_screen_obj.results_frame, state=self.__button_state_from_result_existence() - ) - main_screen_obj.notebook.tab( - main_screen_obj.optimizer_frame, state=self.__button_state_from_result_existence() - ) + main_screen_obj.notebook.tab(main_screen_obj.results_frame, state=self.__button_state_from_result_existence()) + main_screen_obj.notebook.tab(main_screen_obj.optimizer_frame, state=self.__button_state_from_result_existence()) if self.__button_state_from_result_existence() == "normal": show_suggestions_overview_screen(wizard, main_screen_obj.results_frame, self) @@ -153,12 +145,10 @@ def show_details_screen(self, wizard, main_screen_obj): canvas.grid(row=1) # show labels - tk.Label( - canvas, text="Label:", justify=tk.RIGHT, anchor="e", font=wizard.style_font_bold_small - ).grid(row=1, column=1, sticky="ew") - tk.Label(canvas, text="Description", justify=tk.RIGHT, anchor="e").grid( - row=2, column=1, sticky="ew" + tk.Label(canvas, text="Label:", justify=tk.RIGHT, anchor="e", font=wizard.style_font_bold_small).grid( + row=1, column=1, sticky="ew" ) + tk.Label(canvas, text="Description", justify=tk.RIGHT, anchor="e").grid(row=2, column=1, sticky="ew") tk.Label( canvas, text="Executable name:", @@ -166,12 +156,8 @@ def show_details_screen(self, wizard, main_screen_obj): anchor="e", font=wizard.style_font_bold_small, ).grid(row=3, column=1, sticky="ew") - tk.Label(canvas, text="Executable arguments:", justify=tk.RIGHT, anchor="e").grid( - row=4, column=1, sticky="ew" - ) - tk.Label(canvas, text="Make flags:", justify=tk.RIGHT, anchor="e").grid( - row=5, column=1, sticky="ew" - ) + tk.Label(canvas, text="Executable arguments:", justify=tk.RIGHT, anchor="e").grid(row=4, column=1, sticky="ew") + tk.Label(canvas, text="Make flags:", justify=tk.RIGHT, anchor="e").grid(row=5, column=1, sticky="ew") tk.Label( canvas, text="Project path:", @@ -179,12 +165,8 @@ def show_details_screen(self, wizard, main_screen_obj): anchor="e", font=wizard.style_font_bold_small, ).grid(row=6, column=1, sticky="ew") - tk.Label(canvas, text="Project linker flags:", justify=tk.RIGHT, anchor="e").grid( - row=7, column=1, sticky="ew" - ) - tk.Label(canvas, text="Make target:", justify=tk.RIGHT, anchor="e").grid( - row=8, column=1, sticky="ew" - ) + tk.Label(canvas, text="Project linker flags:", justify=tk.RIGHT, anchor="e").grid(row=7, column=1, sticky="ew") + tk.Label(canvas, text="Make target:", justify=tk.RIGHT, anchor="e").grid(row=8, column=1, sticky="ew") tk.Label( canvas, text="Memory Profiling:", @@ -192,9 +174,7 @@ def show_details_screen(self, wizard, main_screen_obj): anchor="e", font=wizard.style_font_bold_small, ).grid(row=9, column=1, sticky="ew") - tk.Label(canvas, text="Skip function params:", justify=tk.RIGHT, anchor="e").grid( - row=10, column=1, sticky="ew" - ) + tk.Label(canvas, text="Skip function params:", justify=tk.RIGHT, anchor="e").grid(row=10, column=1, sticky="ew") tk.Label( canvas, text="Additional:", @@ -202,20 +182,14 @@ def show_details_screen(self, wizard, main_screen_obj): anchor="e", font=wizard.style_font_bold_small, ).grid(row=11, column=1, sticky="ew") - tk.Label(canvas, text="Tags:", justify=tk.RIGHT, anchor="e").grid( - row=12, column=1, sticky="ew" - ) - tk.Label(canvas, text="Notes:", justify=tk.RIGHT, anchor="e").grid( - row=13, column=1, sticky="ew" - ) + tk.Label(canvas, text="Tags:", justify=tk.RIGHT, anchor="e").grid(row=12, column=1, sticky="ew") + tk.Label(canvas, text="Notes:", justify=tk.RIGHT, anchor="e").grid(row=13, column=1, sticky="ew") # show input fields label = tk.Entry(canvas) label.grid(row=1, column=2, sticky="ew") label.insert(tk.END, self.value_dict["label"]) - create_tool_tip( - label, "Name of the configuration. Used to distinguish configurations in the main menu." - ) + create_tool_tip(label, "Name of the configuration. Used to distinguish configurations in the main menu.") description = tk.Entry(canvas) description.grid(row=2, column=2, sticky="ew") @@ -235,8 +209,7 @@ def show_details_screen(self, wizard, main_screen_obj): executable_args.insert(tk.END, self.value_dict["executable_arguments"]) create_tool_tip( executable_args, - "Specify arguments which shall be forwarded to the call of the created executable for the " - "profiling.", + "Specify arguments which shall be forwarded to the call of the created executable for the " "profiling.", ) make_flags = tk.Entry(canvas) @@ -250,9 +223,7 @@ def show_details_screen(self, wizard, main_screen_obj): project_path = tk.Entry(canvas) project_path.grid(row=6, column=2, sticky="ew") project_path.insert(tk.END, self.value_dict["project_path"]) - create_tool_tip( - project_path, "Path to the project which shall be analyzed for potential parallelism." - ) + create_tool_tip(project_path, "Path to the project which shall be analyzed for potential parallelism.") def overwrite_with_selection(target: tk.Entry): prompt_result = tk.filedialog.askdirectory() @@ -260,9 +231,7 @@ def overwrite_with_selection(target: tk.Entry): target.delete(0, tk.END) target.insert(0, prompt_result) - project_path_selector = tk.Button( - canvas, text="Select", command=lambda: overwrite_with_selection(project_path) - ) + project_path_selector = tk.Button(canvas, text="Select", command=lambda: overwrite_with_selection(project_path)) project_path_selector.grid(row=6, column=3) project_linker_flags = tk.Entry(canvas) @@ -270,8 +239,7 @@ def overwrite_with_selection(target: tk.Entry): project_linker_flags.insert(tk.END, self.value_dict["linker_flags"]) create_tool_tip( project_linker_flags, - "Linker flags which need to be passed to the build system in order to create a valid " - "executable.", + "Linker flags which need to be passed to the build system in order to create a valid " "executable.", ) make_target = tk.Entry(canvas) @@ -281,9 +249,7 @@ def overwrite_with_selection(target: tk.Entry): mpsfp_var = tk.IntVar() mpsfp_var.set(self.value_dict["memory_profiling_skip_function_parameters"]) - memory_profiling_skip_function_parameters = tk.Checkbutton( - canvas, onvalue=1, offvalue=0, variable=mpsfp_var - ) + memory_profiling_skip_function_parameters = tk.Checkbutton(canvas, onvalue=1, offvalue=0, variable=mpsfp_var) memory_profiling_skip_function_parameters.grid(row=10, column=2, sticky="w") create_tool_tip( memory_profiling_skip_function_parameters, @@ -415,9 +381,7 @@ def save_changes( self.value_dict["working_copy_path"] = self.value_dict["project_path"] + "/.discopop" self.value_dict["linker_flags"] = project_linker_flags.get() self.value_dict["make_target"] = make_target.get() - self.value_dict[ - "memory_profiling_skip_function_parameters" - ] = memory_profiling_skip_function_parameters.get() + self.value_dict["memory_profiling_skip_function_parameters"] = memory_profiling_skip_function_parameters.get() self.value_dict["tags"] = tags.get() self.value_dict["notes"] = additional_notes.get("1.0", tk.END) @@ -491,12 +455,8 @@ def execute_configuration( # create execution view and update results frame ExecutionView(self, wizard, main_screen_obj.results_frame) # set results tab state based on result existence - main_screen_obj.notebook.tab( - main_screen_obj.results_frame, state=self.__button_state_from_result_existence() - ) - main_screen_obj.notebook.tab( - main_screen_obj.optimizer_frame, state=self.__button_state_from_result_existence() - ) + main_screen_obj.notebook.tab(main_screen_obj.results_frame, state=self.__button_state_from_result_existence()) + main_screen_obj.notebook.tab(main_screen_obj.optimizer_frame, state=self.__button_state_from_result_existence()) # show results tab main_screen_obj.notebook.select(main_screen_obj.results_frame) diff --git a/discopop_wizard/classes/ProfilingContainer.py b/discopop_wizard/classes/ProfilingContainer.py index c46fa089a..bb9b1ef19 100644 --- a/discopop_wizard/classes/ProfilingContainer.py +++ b/discopop_wizard/classes/ProfilingContainer.py @@ -25,13 +25,9 @@ def start(self): ) self.__execute_command("docker kill discopop_container") self.__execute_command("docker rm discopop_container") - exit_code = self.__execute_command( - "docker build -t discopop_container " + docker_context_path - ) + exit_code = self.__execute_command("docker build -t discopop_container " + docker_context_path) assert exit_code == 0 - exit_code = self.__execute_command( - "docker run --name discopop_container -d -t discopop_container" - ) + exit_code = self.__execute_command("docker run --name discopop_container -d -t discopop_container") assert exit_code == 0 def stop(self): @@ -75,9 +71,7 @@ def copy_results_from_container(self, target_path: str, execution_view): def analyze_project(self, execution_view): # copy project folder to container. Note: mounting would be nicer but requires restarting the container. # might be a nicer solution in the long run, especially for larger projects - self.copy_project_folder_to_container( - execution_view.execution_configuration.value_dict["project_path"] - ) + self.copy_project_folder_to_container(execution_view.execution_configuration.value_dict["project_path"]) # settings command = "/discopop/build/scripts/runDiscoPoP " @@ -91,49 +85,25 @@ def analyze_project(self, execution_view): command += "--gllvm /software/go/bin " # execution configuration command += "--project /project " - command += ( - '--linker-flags "' - + execution_view.execution_configuration.value_dict["linker_flags"] - + '" ' - ) - command += ( - '--executable-name "' - + execution_view.execution_configuration.value_dict["executable_name"] - + '" ' - ) + command += '--linker-flags "' + execution_view.execution_configuration.value_dict["linker_flags"] + '" ' + command += '--executable-name "' + execution_view.execution_configuration.value_dict["executable_name"] + '" ' command += ( '--executable-arguments "' + execution_view.execution_configuration.value_dict["executable_arguments"] + '" ' ) - command += ( - '--make-flags "' - + execution_view.execution_configuration.value_dict["make_flags"] - + '" ' - ) - command += ( - '--make-target "' - + execution_view.execution_configuration.value_dict["make_target"] - + '" ' - ) - command += ( - '--explorer-flags "' - + execution_view.execution_configuration.value_dict["explorer_flags"] - + '" ' - ) + command += '--make-flags "' + execution_view.execution_configuration.value_dict["make_flags"] + '" ' + command += '--make-target "' + execution_view.execution_configuration.value_dict["make_target"] + '" ' + command += '--explorer-flags "' + execution_view.execution_configuration.value_dict["explorer_flags"] + '" ' self.__execute_command("docker exec -it discopop_container " + command) # copy results from container into working copy path - if not os.path.exists( - execution_view.execution_configuration.value_dict["working_copy_path"] - ): + if not os.path.exists(execution_view.execution_configuration.value_dict["working_copy_path"]): os.mkdir(execution_view.execution_configuration.value_dict["working_copy_path"]) # remove previous results - self.remove_previous_results( - execution_view.execution_configuration.value_dict["working_copy_path"] - ) + self.remove_previous_results(execution_view.execution_configuration.value_dict["working_copy_path"]) # copy results from container self.copy_results_from_container( @@ -157,9 +127,7 @@ def __correct_file_mapping_paths(self, execution_view): file.write(contents) def __execute_command(self, command: str) -> int: - with subprocess.Popen( - command, stdout=subprocess.PIPE, bufsize=1, universal_newlines=True, shell=True - ) as p: + with subprocess.Popen(command, stdout=subprocess.PIPE, bufsize=1, universal_newlines=True, shell=True) as p: if p.stdout is None: print("executing command was not successfull") else: diff --git a/discopop_wizard/classes/Settings.py b/discopop_wizard/classes/Settings.py index 0c0c3718a..a45e17192 100644 --- a/discopop_wizard/classes/Settings.py +++ b/discopop_wizard/classes/Settings.py @@ -10,7 +10,6 @@ import shutil import jsons # type:ignore -import tkinter as tk class Settings(object): @@ -33,9 +32,7 @@ class Settings(object): code_preview_show_line_numbers: int = 1 # 1 = True, 0 = False code_preview_disable_compile_check: int = 0 # 1 = True, 0 = False - def __init__( - self, discopop_build_dir="", go_bin_dir="", use_docker_container: bool = True - ) -> None: + def __init__(self, discopop_build_dir="", go_bin_dir="", use_docker_container: bool = True) -> None: self.discopop_build_dir = discopop_build_dir self.go_bin = go_bin_dir self.use_docker_container_for_profiling = use_docker_container @@ -45,11 +42,7 @@ def __init__( # get llvm_bin_dir from stored build configuration llvm_bin_dir = "" if settings_valid: - command = ( - "cat " - + self.discopop_build_dir - + '/build_config.txt | grep -oP "(?<=LLVM_BIN_DIR=).*"' - ) + command = "cat " + self.discopop_build_dir + '/build_config.txt | grep -oP "(?<=LLVM_BIN_DIR=).*"' llvm_bin_dir = os.popen(command).read().replace("\n", "") if not os.path.exists(llvm_bin_dir): llvm_bin_dir = "" @@ -143,9 +136,7 @@ def load_from_config_file(config_dir: str) -> Settings: settings.code_preview_show_metadata_regions = __load_or_get_default( value_dict, "code_preview_show_metadata_regions" ) - settings.code_preview_show_line_numbers = __load_or_get_default( - value_dict, "code_preview_show_line_numbers" - ) + settings.code_preview_show_line_numbers = __load_or_get_default(value_dict, "code_preview_show_line_numbers") settings.code_preview_show_metadata_live_device_variables = __load_or_get_default( value_dict, "code_preview_show_metadata_live_device_variables" ) diff --git a/discopop_wizard/classes/Suggestion.py b/discopop_wizard/classes/Suggestion.py index 4b61075cd..3998cdb01 100644 --- a/discopop_wizard/classes/Suggestion.py +++ b/discopop_wizard/classes/Suggestion.py @@ -7,25 +7,14 @@ # directory for details. import os -import sys - import tkinter as tk from enum import IntEnum from pathlib import Path from tkinter import ttk from typing import Any, Dict, List, Tuple -from discopop_explorer.pattern_detectors.combined_gpu_patterns.classes.Enums import ( - ExitPointPositioning, - EntryPointPositioning, - ExitPointType, - EntryPointType, - UpdateType, -) -from discopop_explorer.pattern_detectors.simple_gpu_patterns.GPULoop import OmpConstructPositioning from discopop_library.CodeGenerator.classes.UnpackedSuggestion import UnpackedSuggestion from discopop_wizard.classes.CodePreview import CodePreviewContentBuffer -from discopop_wizard.classes.Pragma import Pragma, PragmaPosition class PragmaType(IntEnum): @@ -34,7 +23,7 @@ class PragmaType(IntEnum): class Suggestion(UnpackedSuggestion): - def __init__(self, wizard, type_str: str, values: dict): + def __init__(self, wizard, type_str: str, values: Dict[str, Any]): super().__init__(type_str, values) self.wizard = wizard @@ -63,9 +52,7 @@ def show_code_section(self, parent_frame: tk.Frame, execution_configuration): # load file mapping from project path file_mapping: Dict[int, Path] = dict() with open( - os.path.join( - execution_configuration.value_dict["working_copy_path"], "FileMapping.txt" - ), + os.path.join(execution_configuration.value_dict["working_copy_path"], "FileMapping.txt"), "r", ) as f: for line in f.readlines(): @@ -76,9 +63,7 @@ def show_code_section(self, parent_frame: tk.Frame, execution_configuration): file_mapping[id] = Path(path) # create CodePreview object - code_preview = CodePreviewContentBuffer( - self.wizard, self.file_id, file_mapping[self.file_id] - ) + code_preview = CodePreviewContentBuffer(self.wizard, self.file_id, file_mapping[self.file_id]) # get and insert pragmas pragmas = self.get_pragmas() @@ -95,9 +80,7 @@ def show_code_section(self, parent_frame: tk.Frame, execution_configuration): file_mapping, pragma, [], - skip_compilation_check=True - if self.wizard.settings.code_preview_disable_compile_check == 1 - else False, + skip_compilation_check=True if self.wizard.settings.code_preview_disable_compile_check == 1 else False, compile_check_command=compile_check_command, ) # if the addition resulted in a non-compilable file, add the pragma as a comment @@ -122,9 +105,7 @@ def show_code_section(self, parent_frame: tk.Frame, execution_configuration): # disable source code text widget to disallow editing source_code.config(state=tk.DISABLED) - def get_as_button( - self, frame: tk.Frame, code_preview_frame: tk.Frame, execution_configuration - ) -> tk.Button: + def get_as_button(self, frame: tk.Frame, code_preview_frame: tk.Frame, execution_configuration) -> tk.Button: return tk.Button( frame, text=self.type + " @ " + self.values["start_line"], @@ -158,9 +139,7 @@ def __highlight_code(self, source_code: tk.Text, start_line: int, end_line: int, start_pos = str(start_line) + ".0" end_pos = str(end_line) + "." + str(end_line_length) source_code.tag_add("start" + str(index), start_pos, end_pos) - source_code.tag_config( - "start" + str(index), background=background_color, foreground="black" - ) + source_code.tag_config("start" + str(index), background=background_color, foreground="black") return start_pos def __get_pragmas(self) -> List[Tuple[int, int, str]]: @@ -185,9 +164,7 @@ def __get_pragmas(self) -> List[Tuple[int, int, str]]: reductions_dict[red_type] = [] reductions_dict[red_type].append(var) for red_type in reductions_dict: - pragma += ( - "reduction(" + red_type + ":" + ",".join(reductions_dict[red_type]) + ") " - ) + pragma += "reduction(" + red_type + ":" + ",".join(reductions_dict[red_type]) + ") " pragma_tuple = (self.start_line, self.end_line, pragma) pragmas.append(pragma_tuple) return pragmas @@ -209,13 +186,7 @@ def __get_pragmas(self) -> List[Tuple[int, int, str]]: reductions_dict[red_type] = [] reductions_dict[red_type].append(var) for red_type in reductions_dict: - pragma += ( - "reduction(" - + red_type - + ":" - + ",".join(reductions_dict[red_type]) - + ") " - ) + pragma += "reduction(" + red_type + ":" + ",".join(reductions_dict[red_type]) + ") " if len(stage["in_deps"]) > 0: pragma += "depends(in:" + ",".join(stage["in_deps"]) + ") " if len(stage["out_deps"]) > 0: diff --git a/discopop_wizard/classes/TKVarStorage.py b/discopop_wizard/classes/TKVarStorage.py index fe869e417..e046d2dda 100644 --- a/discopop_wizard/classes/TKVarStorage.py +++ b/discopop_wizard/classes/TKVarStorage.py @@ -39,9 +39,7 @@ def toggle_code_preview_setting_action(self): self.wizard.settings.code_preview_show_metadata_live_device_variables = ( self.toggle_var_code_preview_show_metadata_live_device_variables.get() ) - self.wizard.settings.code_preview_show_line_numbers = ( - self.toggle_var_code_preview_show_line_numbers.get() - ) + self.wizard.settings.code_preview_show_line_numbers = self.toggle_var_code_preview_show_line_numbers.get() self.wizard.settings.code_preview_disable_compile_check = ( self.toggle_var_code_preview_disable_compile_check.get() ) diff --git a/discopop_wizard/headless/headless_execution.py b/discopop_wizard/headless/headless_execution.py index 9a2d58fb8..e0e76b6fe 100644 --- a/discopop_wizard/headless/headless_execution.py +++ b/discopop_wizard/headless/headless_execution.py @@ -32,9 +32,7 @@ def execute_tag_filtered_configurations(args: Arguments, source_dir: str): # get tags from config # get tags from arguments # if an overlap exists, the configurations shall be executed - overlapping_tags = [ - tag for tag in config.get_tags() if tag in args.execute_configurations_with_tag - ] + overlapping_tags = [tag for tag in config.get_tags() if tag in args.execute_configurations_with_tag] if len(overlapping_tags) > 0: filtered_execution_configs.append(config) @@ -59,9 +57,7 @@ def __load_data( for filename in os.listdir(os.path.join(config_dir, "execution_configurations")): if not filename.endswith(".json"): continue - with open( - os.path.join(os.path.join(config_dir, "execution_configurations"), filename), "r" - ) as json_file: + with open(os.path.join(os.path.join(config_dir, "execution_configurations"), filename), "r") as json_file: config = ExecutionConfiguration(wizard) config.init_from_json(json_file) execution_configs.append(config) diff --git a/discopop_wizard/screens/execution.py b/discopop_wizard/screens/execution.py index def0372e1..43615bfd4 100644 --- a/discopop_wizard/screens/execution.py +++ b/discopop_wizard/screens/execution.py @@ -63,37 +63,22 @@ def __assemble_command_string(self) -> str: command += '--gllvm "' + self.wizard.settings.go_bin + '" ' # execution configuration command += '--project "' + self.execution_configuration.value_dict["project_path"] + '" ' - command += ( - '--linker-flags "' + self.execution_configuration.value_dict["linker_flags"] + '" ' - ) - command += ( - '--executable-name "' - + self.execution_configuration.value_dict["executable_name"] - + '" ' - ) - command += ( - '--executable-arguments "' - + self.execution_configuration.value_dict["executable_arguments"] - + '" ' - ) + command += '--linker-flags "' + self.execution_configuration.value_dict["linker_flags"] + '" ' + command += '--executable-name "' + self.execution_configuration.value_dict["executable_name"] + '" ' + command += '--executable-arguments "' + self.execution_configuration.value_dict["executable_arguments"] + '" ' command += '--make-flags "' + self.execution_configuration.value_dict["make_flags"] + '" ' command += '--make-target "' + self.execution_configuration.value_dict["make_target"] + '" ' command += ( "--memory-profiling-skip-function-arguments " - if self.execution_configuration.value_dict["memory_profiling_skip_function_parameters"] - == 1 + if self.execution_configuration.value_dict["memory_profiling_skip_function_parameters"] == 1 else "" ) - command += ( - '--explorer-flags "' + self.execution_configuration.value_dict["explorer_flags"] + '" ' - ) + command += '--explorer-flags "' + self.execution_configuration.value_dict["explorer_flags"] + '" ' return command def __execute_command(self, command: str) -> int: - with subprocess.Popen( - command, stdout=subprocess.PIPE, bufsize=1, universal_newlines=True, shell=True - ) as p: + with subprocess.Popen(command, stdout=subprocess.PIPE, bufsize=1, universal_newlines=True, shell=True) as p: if p.stdout is None: print("command execution was not successfull") else: @@ -117,6 +102,4 @@ def __print_to_console(self, msg: str): print(msg) def __show_suggestions(self): - show_suggestions_overview_screen( - self.wizard, cast(tk.Frame, self.details_frame), self.execution_configuration - ) + show_suggestions_overview_screen(self.wizard, cast(tk.Frame, self.details_frame), self.execution_configuration) diff --git a/discopop_wizard/screens/main.py b/discopop_wizard/screens/main.py index aaab6ae55..3449f1a74 100644 --- a/discopop_wizard/screens/main.py +++ b/discopop_wizard/screens/main.py @@ -12,7 +12,6 @@ from discopop_wizard.classes.ExecutionConfiguration import ExecutionConfiguration from discopop_wizard.screens.settings import show_settings_screen - from discopop_wizard.utils import support_scrolling @@ -102,16 +101,12 @@ def __create_new_execution_configuration(self, wizard): def __display_execution_configurations(self, wizard): # based on https://blog.teclado.com/tkinter-scrollable-frames/ # load configuration options - configs: List[ExecutionConfiguration] = self.load_execution_configurations( - wizard.config_dir - ) + configs: List[ExecutionConfiguration] = self.load_execution_configurations(wizard.config_dir) frame = tk.Frame(self.configuration_frame) frame.pack(fill=tk.BOTH, expand=True) tk.Label(frame, text="Configurations", font=wizard.style_font_bold, pady=10).pack() # add New.. Button - tk.Button( - frame, text="New..", command=lambda: self.__create_new_execution_configuration(wizard) - ).pack() + tk.Button(frame, text="New..", command=lambda: self.__create_new_execution_configuration(wizard)).pack() tmp_frame = tk.Frame(frame) tmp_frame.pack(fill=tk.BOTH, expand=True) @@ -120,15 +115,11 @@ def __display_execution_configurations(self, wizard): canvas = tk.Canvas(tmp_frame) scrollbar = tk.Scrollbar(tmp_frame, orient="vertical", command=canvas.yview) scrollable_frame = tk.Frame(canvas) - scrollable_frame.bind( - "", lambda e: canvas.configure(scrollregion=canvas.bbox("all")) - ) + scrollable_frame.bind("", lambda e: canvas.configure(scrollregion=canvas.bbox("all"))) canvas.create_window((0, 0), window=scrollable_frame, anchor="nw") canvas.configure(yscrollcommand=scrollbar.set) - all_buttons: List[ - tk.Button - ] = [] # used to manage highlights when a different configuration is selected + all_buttons: List[tk.Button] = [] # used to manage highlights when a different configuration is selected for row, config in enumerate(configs): button = config.get_as_button(wizard, self, scrollable_frame, all_buttons) button.pack(fill=tk.BOTH, expand=True) @@ -145,9 +136,7 @@ def load_execution_configurations(self, config_dir: str) -> List[ExecutionConfig for filename in os.listdir(os.path.join(config_dir, "execution_configurations")): if not filename.endswith(".json"): continue - with open( - os.path.join(os.path.join(config_dir, "execution_configurations"), filename), "r" - ) as json_file: + with open(os.path.join(os.path.join(config_dir, "execution_configurations"), filename), "r") as json_file: config = ExecutionConfiguration(self.wizard) config.init_from_json(json_file) execution_configs.append(config) diff --git a/discopop_wizard/screens/optimizer/binding.py b/discopop_wizard/screens/optimizer/binding.py index d4afbf9d6..b17dd4e13 100644 --- a/discopop_wizard/screens/optimizer/binding.py +++ b/discopop_wizard/screens/optimizer/binding.py @@ -120,9 +120,7 @@ def __start_optimizer( execution_configuration.value_dict["project_path"], ".discopop_optimizer/code_exports" ), "--dp-output-path": execution_configuration.value_dict["working_copy_path"], - "--file-mapping": get_path( - execution_configuration.value_dict["working_copy_path"], "FileMapping.txt" - ), + "--file-mapping": get_path(execution_configuration.value_dict["working_copy_path"], "FileMapping.txt"), "--executable-arguments": execution_configuration.value_dict["executable_arguments"], "--executable-name": execution_configuration.value_dict["executable_name"], "--linker-flags": execution_configuration.value_dict["linker_flags"], @@ -136,9 +134,7 @@ def __start_optimizer( "--headless-mode": False, "--doall-microbench-file": doall_microbench_file.get(), "--reduction-microbench-file": reduction_microbench_file.get(), - "--dp-optimizer-path": get_path( - execution_configuration.value_dict["project_path"], ".discopop_optimizer" - ), + "--dp-optimizer-path": get_path(execution_configuration.value_dict["project_path"], ".discopop_optimizer"), } # close elements on optimizer_frame diff --git a/discopop_wizard/screens/settings.py b/discopop_wizard/screens/settings.py index dd8865dcf..9aa382cf2 100644 --- a/discopop_wizard/screens/settings.py +++ b/discopop_wizard/screens/settings.py @@ -6,7 +6,6 @@ # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -import os import tkinter as tk from tkinter import filedialog from tkinter import ttk @@ -27,41 +26,21 @@ def show_settings_screen(wizard): tk.Label(frame, text="Directories:", justify=tk.RIGHT, font=wizard.style_font_bold).grid( row=1, column=1, sticky="ew" ) - tk.Label(frame, text="DiscoPoP build:", justify=tk.RIGHT, anchor="e").grid( - row=3, column=1, sticky="ew" - ) - tk.Label(frame, text="go/bin directory:", justify=tk.RIGHT, anchor="e").grid( - row=4, column=1, sticky="ew" - ) + tk.Label(frame, text="DiscoPoP build:", justify=tk.RIGHT, anchor="e").grid(row=3, column=1, sticky="ew") + tk.Label(frame, text="go/bin directory:", justify=tk.RIGHT, anchor="e").grid(row=4, column=1, sticky="ew") ttk.Separator(frame, orient="horizontal").grid(row=5, column=1, sticky="ew", pady=10) - tk.Label(frame, text="Executables:", justify=tk.RIGHT, font=wizard.style_font_bold).grid( - row=6, column=1 - ) + tk.Label(frame, text="Executables:", justify=tk.RIGHT, font=wizard.style_font_bold).grid(row=6, column=1) tk.Label(frame, text="clang:", justify=tk.RIGHT, anchor="e").grid(row=7, column=1, sticky="ew") - tk.Label(frame, text="clang++:", justify=tk.RIGHT, anchor="e").grid( - row=8, column=1, sticky="ew" - ) - tk.Label(frame, text="llvm-ar:", justify=tk.RIGHT, anchor="e").grid( - row=9, column=1, sticky="ew" - ) - tk.Label(frame, text="llvm-link:", justify=tk.RIGHT, anchor="e").grid( - row=10, column=1, sticky="ew" - ) - tk.Label(frame, text="llvm-dis:", justify=tk.RIGHT, anchor="e").grid( - row=11, column=1, sticky="ew" - ) - tk.Label(frame, text="llvm-opt:", justify=tk.RIGHT, anchor="e").grid( - row=12, column=1, sticky="ew" - ) - tk.Label(frame, text="llvm-llc:", justify=tk.RIGHT, anchor="e").grid( - row=13, column=1, sticky="ew" - ) + tk.Label(frame, text="clang++:", justify=tk.RIGHT, anchor="e").grid(row=8, column=1, sticky="ew") + tk.Label(frame, text="llvm-ar:", justify=tk.RIGHT, anchor="e").grid(row=9, column=1, sticky="ew") + tk.Label(frame, text="llvm-link:", justify=tk.RIGHT, anchor="e").grid(row=10, column=1, sticky="ew") + tk.Label(frame, text="llvm-dis:", justify=tk.RIGHT, anchor="e").grid(row=11, column=1, sticky="ew") + tk.Label(frame, text="llvm-opt:", justify=tk.RIGHT, anchor="e").grid(row=12, column=1, sticky="ew") + tk.Label(frame, text="llvm-llc:", justify=tk.RIGHT, anchor="e").grid(row=13, column=1, sticky="ew") ttk.Separator(frame, orient="horizontal").grid(row=14, column=1, sticky="ew", pady=10) - tk.Label(frame, text="Options:", justify=tk.RIGHT, font=wizard.style_font_bold).grid( - row=15, column=1 - ) + tk.Label(frame, text="Options:", justify=tk.RIGHT, font=wizard.style_font_bold).grid(row=15, column=1) tk.Label(frame, text="Use Docker Container for profiling:", justify=tk.RIGHT, anchor="e").grid( row=16, column=1, sticky="ew" ) @@ -124,9 +103,7 @@ def __get_field_state(): llvm_llc.config(state=__get_field_state()) create_tool_tip(llvm_llc, "Path to the llvm_llc executable.") - use_docker_container_var = tk.IntVar( - value=1 if wizard.settings.use_docker_container_for_profiling else 0 - ) + use_docker_container_var = tk.IntVar(value=1 if wizard.settings.use_docker_container_for_profiling else 0) use_docker_container = tk.Checkbutton(frame, variable=use_docker_container_var) create_tool_tip( use_docker_container, @@ -136,33 +113,15 @@ def __get_field_state(): use_docker_container.grid(row=16, column=2) # show path selector buttons - tk.Button( - frame, text="Select", command=lambda: __overwrite_with_selection(discopop_build) - ).grid(row=3, column=3) - tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(go_bin_path)).grid( - row=4, column=3 - ) - tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(clang)).grid( - row=7, column=3 - ) - tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(clangpp)).grid( - row=8, column=3 - ) - tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(llvm_ar)).grid( - row=9, column=3 - ) - tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(llvm_link)).grid( - row=10, column=3 - ) - tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(llvm_dis)).grid( - row=11, column=3 - ) - tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(llvm_opt)).grid( - row=12, column=3 - ) - tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(llvm_llc)).grid( - row=13, column=3 - ) + tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(discopop_build)).grid(row=3, column=3) + tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(go_bin_path)).grid(row=4, column=3) + tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(clang)).grid(row=7, column=3) + tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(clangpp)).grid(row=8, column=3) + tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(llvm_ar)).grid(row=9, column=3) + tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(llvm_link)).grid(row=10, column=3) + tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(llvm_dis)).grid(row=11, column=3) + tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(llvm_opt)).grid(row=12, column=3) + tk.Button(frame, text="Select", command=lambda: __overwrite_with_selection(llvm_llc)).grid(row=13, column=3) # show save button tk.Button( @@ -213,9 +172,7 @@ def save_settings( wizard.settings.llvm_dis = llvm_dis.get() wizard.settings.llvm_opt = llvm_opt.get() wizard.settings.llvm_llc = llvm_llc.get() - wizard.settings.use_docker_container_for_profiling = ( - True if use_docker_container_var.get() == 1 else False - ) + wizard.settings.use_docker_container_for_profiling = True if use_docker_container_var.get() == 1 else False wizard.settings.save_to_file(config_dir=wizard.config_dir) diff --git a/discopop_wizard/screens/suggestions/overview.py b/discopop_wizard/screens/suggestions/overview.py index cff3cb6f2..eaa29e5c9 100644 --- a/discopop_wizard/screens/suggestions/overview.py +++ b/discopop_wizard/screens/suggestions/overview.py @@ -5,7 +5,6 @@ # This software may be modified and distributed under the terms of # the 3-Clause BSD License. See the LICENSE file in the package base # directory for details. -import functools import json import os import tkinter as tk @@ -15,7 +14,6 @@ from discopop_wizard.classes.Suggestion import Suggestion from discopop_wizard.screens.utils import create_tool_tip from discopop_wizard.screens.widgets.ScrollableText import ScrollableTextWidget - from discopop_wizard.utils import support_scrolling @@ -62,20 +60,15 @@ def show_suggestions_overview_screen(wizard, details_frame: tk.Frame, execution_ result_notebook.add(dynamic_dep_display_widget.frame, text="Dynamic DEP's") # add static dependency preview static_dep_display_widget = ScrollableTextWidget(result_notebook) - if os.path.exists( - execution_configuration_obj.value_dict["working_copy_path"] + "/static_dependencies.txt" - ): + if os.path.exists(execution_configuration_obj.value_dict["working_copy_path"] + "/static_dependencies.txt"): with open( - execution_configuration_obj.value_dict["working_copy_path"] - + "/static_dependencies.txt", + execution_configuration_obj.value_dict["working_copy_path"] + "/static_dependencies.txt", "r", ) as f: static_dep_display_widget.set_text(f.read()) else: static_dep_display_widget.set_text( - execution_configuration_obj.value_dict["working_copy_path"] - + "/static_dependencies.txt" - + " NOT FOUND." + execution_configuration_obj.value_dict["working_copy_path"] + "/static_dependencies.txt" + " NOT FOUND." ) result_notebook.add(static_dep_display_widget.frame, text="Static DEP's") # add instrumented LLVM IR preview @@ -105,9 +98,7 @@ def show_suggestions_overview_screen(wizard, details_frame: tk.Frame, execution_ result_notebook.add(instrumented_llvm_ir_display_widget.frame, text="LLVM IR") # add patterns.json preview patterns_json_display_widget = ScrollableTextWidget(result_notebook) - with open( - execution_configuration_obj.value_dict["working_copy_path"] + "/" + "patterns.json", "r" - ) as f: + with open(execution_configuration_obj.value_dict["working_copy_path"] + "/" + "patterns.json", "r") as f: patterns_json_display_widget.set_text(f.read()) result_notebook.add(patterns_json_display_widget.frame, text="patterns.json") @@ -115,16 +106,12 @@ def show_suggestions_overview_screen(wizard, details_frame: tk.Frame, execution_ canvas = tk.Canvas(tmp_frame) scrollbar = tk.Scrollbar(tmp_frame, orient="vertical", command=canvas.yview) scrollable_frame = tk.Frame(canvas) - scrollable_frame.bind( - "", lambda e: canvas.configure(scrollregion=canvas.bbox("all")) - ) + scrollable_frame.bind("", lambda e: canvas.configure(scrollregion=canvas.bbox("all"))) canvas.create_window((0, 0), window=scrollable_frame, anchor="nw") canvas.configure(yscrollcommand=scrollbar.set) for row, suggestion in enumerate(suggestions): # create button to load code preview - button = suggestion.get_as_button( - scrollable_frame, code_preview_frame, execution_configuration_obj - ) + button = suggestion.get_as_button(scrollable_frame, code_preview_frame, execution_configuration_obj) button.pack(fill=tk.BOTH, expand=True) # register hover message (suggestion details) @@ -141,9 +128,7 @@ def show_suggestions_overview_screen(wizard, details_frame: tk.Frame, execution_ def get_suggestion_objects(wizard, execution_configuration_obj) -> List[Suggestion]: - suggestions_path = os.path.join( - execution_configuration_obj.value_dict["working_copy_path"], "patterns.json" - ) + suggestions_path = os.path.join(execution_configuration_obj.value_dict["working_copy_path"], "patterns.json") suggestions_list: List[Suggestion] = [] with open(suggestions_path, "r") as f: diff --git a/discopop_wizard/screens/widgets/ScrollableText.py b/discopop_wizard/screens/widgets/ScrollableText.py index c7aff5228..6d0b998b4 100644 --- a/discopop_wizard/screens/widgets/ScrollableText.py +++ b/discopop_wizard/screens/widgets/ScrollableText.py @@ -32,9 +32,7 @@ def __init__(self, parent_frame): y_scrollbar = ttk.Scrollbar(self.frame, command=self.text_container.yview) y_scrollbar.grid(row=0, column=1, sticky="nsew") self.text_container["yscrollcommand"] = y_scrollbar.set - x_scrollbar = ttk.Scrollbar( - self.frame, orient="horizontal", command=self.text_container.xview - ) + x_scrollbar = ttk.Scrollbar(self.frame, orient="horizontal", command=self.text_container.xview) x_scrollbar.grid(row=1, column=0, columnspan=2, sticky="nsew") self.text_container["xscrollcommand"] = x_scrollbar.set self.text_container.config(state=tk.DISABLED) diff --git a/discopop_wizard/utils.py b/discopop_wizard/utils.py index 0ee1eb7b3..3087a882d 100644 --- a/discopop_wizard/utils.py +++ b/discopop_wizard/utils.py @@ -8,8 +8,8 @@ import functools -from sys import platform from enum import Enum +from sys import platform class Platform(Enum): @@ -43,13 +43,9 @@ def _bind_to_mousewheel(event): canvas.bind_all("", functools.partial(_on_mousewheel, scroll=-1)) canvas.bind_all("", functools.partial(_on_mousewheel, scroll=1)) elif pf == Platform.WINDOWS: - canvas.bind_all( - "", functools.partial(_on_mousewheel, scroll=(-1 * (event.delta / 120))) - ) + canvas.bind_all("", functools.partial(_on_mousewheel, scroll=(-1 * (event.delta / 120)))) elif pf == Platform.OSX: - canvas.bind_all( - "", functools.partial(_on_mousewheel, scroll=-1 * event.delta) - ) + canvas.bind_all("", functools.partial(_on_mousewheel, scroll=-1 * event.delta)) else: canvas.bind_all("", functools.partial(_on_mousewheel, scroll=-1)) canvas.bind_all("", functools.partial(_on_mousewheel, scroll=1)) diff --git a/discopop_wizard/wizard.py b/discopop_wizard/wizard.py index e09d7deea..3c129dde8 100644 --- a/discopop_wizard/wizard.py +++ b/discopop_wizard/wizard.py @@ -11,10 +11,10 @@ import signal import tkinter as tk import warnings -from tkinter import messagebox, filedialog -from tkinter import ttk from enum import IntEnum from os.path import dirname +from tkinter import messagebox, filedialog +from tkinter import ttk from typing import Optional from discopop_wizard.classes.Arguments import Arguments @@ -26,8 +26,7 @@ # todo add command line option to list available run configurations # todo add command line option to execute run configuration (by name) -from discopop_wizard.screens.settings import show_settings_screen, save_settings - +from discopop_wizard.screens.settings import show_settings_screen from discopop_wizard.utils import get_platform, Platform diff --git a/docs/How_to_contribute.md b/docs/How_to_contribute.md index fdca00cf5..573ba0279 100644 --- a/docs/How_to_contribute.md +++ b/docs/How_to_contribute.md @@ -30,13 +30,25 @@ In general it is sufficient to follow the general installation instructions. How - The `-e` switch ensures that changes in the python source code are immediately active. - `[dev]` also installs some development requirements (e.g. mypy, black, pre-commit). - Install some git hooks by running `pre-commit install` from the main directory of this project. These hooks help to ensure a good quality of the commited code by automatically running the black **formatter** and checking for **type safety** with mypy on every commit. + - Activate the git `commit-msg` hook to validate commit message formatting by creating a file named `.git/hooks/commit-msg` with the following contents: + ``` + #!/bin/sh + python scripts/dev/check-commit-msg.py $1 + ``` + +## Commit messages +Commit messages should follow the following format: +``` +(scope)[optional info]: commit message +``` +where `` can be any of `feat,fix,test,chore,wip`. ## Creating a new release Execute the following steps in order to create a new DiscoPoP release: - Switch to the release branch (e.g. `release/1.2.3`) which shall be released - Update the version file in the repository (`discopop_library/global_data/version/VERSION`) - Create a pull request to the `master` branch and validate the changes -- Merge the pull request and create a tag on the `master` branch with the name `v1.2.3` +- Merge the pull request by rebasing and create a tag on the `master` branch with the name `v1.2.3` - Creating the tag triggers the automatic publication of the project to PyPi - Creating the tag triggers the automatic creation of a release draft - Update the newly created release draft diff --git a/docs/img/DPWorkflow.svg b/docs/img/DPWorkflow.svg index 1a01f3cd1..758f9166a 100644 --- a/docs/img/DPWorkflow.svg +++ b/docs/img/DPWorkflow.svg @@ -1,6 +1,7 @@ - + diff --git a/docs/img/init1.svg b/docs/img/init1.svg index 2c9471c0c..5a4b20a1b 100644 --- a/docs/img/init1.svg +++ b/docs/img/init1.svg @@ -1,16 +1,15 @@ image/svg+xml image/svg+xml(scope)[optional info]: commit message") + print("where `` can be any of `feat,fix,test,chore,wip`.") + sys.exit(1)