Gephi-File dump

.github/workflows/ci.yml

@@ -111,7 +111,7 @@ jobs:
       - name: C++ unit tests
         run: |
           build/test/unit_tests/DiscoPoP_UT
-  
+
   execute_end_to_end_tests:
     runs-on: ubuntu-20.04
     name: Execute end-to-end tests
@@ -158,6 +158,7 @@ jobs:
 
       - name: Python end-to-end-tests
         run: |
+          . venv/bin/activate
           python -m unittest -v -k *.end_to_end.*
   update_wiki-build:
     name: "Update Wiki - Build"

.gitignore

@@ -136,6 +136,9 @@ venv.bak/
 *.a
 *.lib
 
+# Packages
+packages/
+
 # Executables
 *.exe
 *.out

CMakeLists.txt

@@ -102,32 +102,67 @@ file(APPEND "${DiscoPoP_SOURCE_DIR}/discopop_library/ConfigProvider/assets/build
 file(APPEND "${DiscoPoP_SOURCE_DIR}/discopop_library/ConfigProvider/assets/build_config.py" "DP_BRANCH_TRACKING=\"${DP_BRANCH_TRACKING}\"\n")
 
 
-# print DiscoPoP configuration
-file(READ ${DiscoPoP_SOURCE_DIR}/discopop_library/ConfigProvider/assets/build_config.py DiscoPoP_CONFIGURATION)
-string(REGEX REPLACE "\n$" "" DiscoPoP_CONFIGURATION "${DiscoPoP_CONFIGURATION}")
-string(REGEX REPLACE "\n" "\n    " DiscoPoP_CONFIGURATION "${DiscoPoP_CONFIGURATION}")
-message(STATUS "DiscoPoP configuration:\n    ${DiscoPoP_CONFIGURATION}")
-
-# install DiscoPoP python modules
-find_package(Python3 REQUIRED COMPONENTS Interpreter)
-
-# check if python tkinter module is available
-execute_process(
-    COMMAND ${Python3_EXECUTABLE} -c "import tkinter"
-    RESULT_VARIABLE TKINTER_AVAILABLE_EXIT_CODE
-)
-if(${TKINTER_AVAILABLE_EXIT_CODE})
-    message(FATAL_ERROR "Python module 'tkinter' not found. Please install the 'python3-tk' package via a package manager.")
-endif()
+if(NOT ${IS_DEB_INSTALL} STREQUAL "")
+    message(STATUS "performing installation from .deb package")
+else()
+    # print DiscoPoP configuration
+    file(READ ${DiscoPoP_SOURCE_DIR}/discopop_library/ConfigProvider/assets/build_config.py DiscoPoP_CONFIGURATION)
+    string(REGEX REPLACE "\n$" "" DiscoPoP_CONFIGURATION "${DiscoPoP_CONFIGURATION}")
+    string(REGEX REPLACE "\n" "\n    " DiscoPoP_CONFIGURATION "${DiscoPoP_CONFIGURATION}")
+    message(STATUS "DiscoPoP configuration:\n    ${DiscoPoP_CONFIGURATION}")
+
+    # install DiscoPoP python modules
+    find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
+    # setup DiscoPoP venv
+    message(STATUS "Setting up DiscoPoP python venv: ${DiscoPoP_SOURCE_DIR}/venv")
+    execute_process(
+        COMMAND ${Python3_EXECUTABLE} -m venv ${DiscoPoP_SOURCE_DIR}/venv
+    )
+    set(Python3_VENV_EXECUTABLE ${DiscoPoP_SOURCE_DIR}/venv/bin/python3)
+
+
+    # check if python tkinter module is available
+    execute_process(
+        COMMAND ${Python3_VENV_EXECUTABLE} -c "import tkinter"
+        RESULT_VARIABLE TKINTER_AVAILABLE_EXIT_CODE
+    )
+    if(${TKINTER_AVAILABLE_EXIT_CODE})
+        message(FATAL_ERROR "Python module 'tkinter' not found. Please install the 'python3-tk' package via a package manager.")
+    endif()
+
+    # install DiscoPoP python modules
+    message(STATUS "Installing DiscoPoP python modules")
+    execute_process(
+            COMMAND ${Python3_VENV_EXECUTABLE} -m pip install ${DiscoPoP_SOURCE_DIR}
+            RESULT_VARIABLE DP_INSTALLATION_EXIT_CODE
+            OUTPUT_VARIABLE DP_INSTALLATION_OUTPUT
+    )
+    # check if installation of DiscoPoP Modules was successful
+    if(${DP_INSTALLATION_EXIT_CODE})
+        message(FATAL_ERROR "${DP_INSTALLATION_OUTPUT}")
+    endif()
+
+    # create symlinks for python executables
+    message(STATUS "Creating symlinks for DiscoPoP python executables")
+    set(DP_LOCAL_BIN_DIR "$ENV{HOME}/.local/bin")
+    execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_auto_tuner)
+    execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_config_provider)
+    execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_explorer)
+    execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_optimizer)
+    execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_patch_applicator)
+    execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_patch_generator)
+    message(STATUS "--> discopop_auto_tuner")
+    execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_auto_tuner ${DP_LOCAL_BIN_DIR}/discopop_auto_tuner)
+    message(STATUS "--> discopop_config_provider")
+    execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_config_provider ${DP_LOCAL_BIN_DIR}/discopop_config_provider)
+    message(STATUS "--> discopop_explorer")
+    execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_explorer ${DP_LOCAL_BIN_DIR}/discopop_explorer)
+    message(STATUS "--> discopop_optimizer")
+    execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_optimizer ${DP_LOCAL_BIN_DIR}/discopop_optimizer)
+    message(STATUS "--> discopop_patch_applicator")
+    execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_patch_applicator ${DP_LOCAL_BIN_DIR}/discopop_patch_applicator)
+    message(STATUS "--> discopop_patch_generator")
+    execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_patch_generator ${DP_LOCAL_BIN_DIR}/discopop_patch_generator)
 
-# install DiscoPoP python modules
-message(STATUS "Installing DiscoPoP python modules")
-execute_process(
-        COMMAND ${Python3_EXECUTABLE} -m pip install ${DiscoPoP_SOURCE_DIR}
-        RESULT_VARIABLE DP_INSTALLATION_EXIT_CODE
-        OUTPUT_VARIABLE DP_INSTALLATION_OUTPUT
-)
-# check if installation of DiscoPoP Modules was successful
-if(${DP_INSTALLATION_EXIT_CODE})
-    message(FATAL_ERROR "${DP_INSTALLATION_OUTPUT}")
 endif()

DEBIAN/control.raw

@@ -0,0 +1,6 @@
+Package: discopop
+Maintainer: Lukas Rothenberger <[email protected]>
+Depends: libc6,python3-tk,python3
+Architecture: all
+Homepage: https://github.com/discopop-project/discopop
+Description: DiscoPoP is an open-source tool that helps software developers parallelize their programs with threads. It is a joint project of the Laboratory for Parallel Programming @ TU Darmstadt and the Software Analytics and Pervasive Parallelism Lab at Iowa State University.

DEBIAN/postinst

@@ -0,0 +1,55 @@
+DP_DIR=/opt/DiscoPoP
+chmod 777 ${DP_DIR}
+DP_BUILD_DIR=/opt/DiscoPoP/build
+mkdir -p ${DP_BUILD_DIR}
+chmod 777 ${DP_BUILD_DIR}
+cd ${DP_BUILD_DIR}
+echo "Created DiscoPoP build dir: ${DP_BUILD_DIR}"
+
+echo "Downloading llvm 11.1.0"
+mkdir third_party
+cd third_party
+wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.1.0/clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-20.10.tar.xz
+tar -xvf clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-20.10.tar.xz
+mv clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-20.10 llvm-11.1.0
+rm clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-20.10.tar.xz
+LLVM_DIR=${DP_BUILD_DIR}/third_party/llvm-11.1.0
+
+echo "Building DiscoPoP"
+cd ${DP_BUILD_DIR}
+cmake .. -DLLVM_DIST_PATH=${LLVM_DIR} -DIS_DEB_INSTALL="TRUE" -DDP_PTHREAD_COMPATIBILITY_MODE=0 -DDP_NUM_WORKERS=8 -DDP_RTLIB_VERBOSE=0 -DDP_MEMORY_REGION_DEALIASING=0 -DDP_BRANCH_TRACKING=0 -DDP_CALLSTACK_PROFILING=0 -DDP_STACK_ACCESS_DETECTION=0 -DDP_CALLSTACK_PROFILING_ENABLE_CUTOFF=1 -DDP_INTERNAL_TIMER=0 -DDP_HYBRID_PROFILING=1 -DDP_HYBRID_PROFILING_CUTOFF=0 -DDP_HYBRID_PROFILING_CUTOFF_IGNORE_PROBABILITY=1 -DDP_PROFILING_SAMPLING_PROBABILITY=0 -DDP_CALLTREE_PROFILING=1 -DDP_CALLTREE_PROFILING_METADATA_CUTOFF=5 -DDP_CALLTREE_PROFILING_METADATA_CUTOFF_IGNORE_PROBABILITY=1
+make -j 4
+
+chmod -R 777 ${DP_DIR}
+
+echo "Setting up DiscoPoP python venv"
+cd ${DP_DIR}
+su ${SUDO_USER} -c "python3 -m venv venv"
+VENV_PYTHON=${DP_DIR}/venv/bin/python3
+
+echo "Installing DiscoPoP python modules"
+su ${SUDO_USER} -c "${VENV_PYTHON} -m pip install . -v"
+
+#
+#echo "Creating symlinks"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_cc"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_cxx"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_cmake"
+
+su ${SUDO_USER} -c "ln -sf ${DP_BUILD_DIR}/scripts/CC_wrapper.sh ~/.local/bin/discopop_cc"
+su ${SUDO_USER} -c "ln -sf ${DP_BUILD_DIR}/scripts/CXX_wrapper.sh ~/.local/bin/discopop_cxx"
+su ${SUDO_USER} -c "ln -sf ${DP_BUILD_DIR}/scripts/CMAKE_wrapper.sh ~/.local/bin/discopop_cmake"
+
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_auto_tuner"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_config_provider"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_explorer"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_optimizer"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_patch_applicator"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_patch_generator"
+
+su ${SUDO_USER} -c "ln -sf ${DP_DIR}/venv/bin/discopop_auto_tuner ~/.local/bin/discopop_auto_tuner"
+su ${SUDO_USER} -c "ln -sf ${DP_DIR}/venv/bin/discopop_config_provider ~/.local/bin/discopop_config_provider"
+su ${SUDO_USER} -c "ln -sf ${DP_DIR}/venv/bin/discopop_explorer ~/.local/bin/discopop_explorer"
+su ${SUDO_USER} -c "ln -sf ${DP_DIR}/venv/bin/discopop_optimizer ~/.local/bin/discopop_optimizer"
+su ${SUDO_USER} -c "ln -sf ${DP_DIR}/venv/bin/discopop_patch_applicator ~/.local/bin/discopop_patch_applicator"
+su ${SUDO_USER} -c "ln -sf ${DP_DIR}/venv/bin/discopop_patch_generator ~/.local/bin/discopop_patch_generator"

DEBIAN/postrm

@@ -0,0 +1,12 @@
+rm -rf /opt/DiscoPoP
+
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_cc"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_cxx"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_cmake"
+
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_auto_tuner"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_config_provider"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_explorer"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_optimizer"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_patch_applicator"
+su ${SUDO_USER} -c "rm -f ~/.local/bin/discopop_patch_generator"

README.md

@@ -34,25 +34,25 @@ For a brief introduction to the command line tools, please refer to the [tools o
 For detailed information on the gathered and stored data as well as the tools themselves, please refer to [data](https://discopop-project.github.io/discopop/Data) and the pages of the individual tools in the [tools overview](https://discopop-project.github.io/discopop/Tools).
 
 ## TL;DR
-This example installs DiscoPoP, instruments and builds the provided example, analyzes the results, and prints the identified parallelization suggestions to the console.
+If you are interested in installing DiscoPoP as a `developer`, please refer to the [DiscoPoP setup wiki page](https://discopop-project.github.io/discopop/setup/discopop/).
+
+The following example installs DiscoPoP for `users`, instruments and builds the provided example, analyzes the results, and prints the identified parallelization suggestions to the console.
 In case any issues arise during the process, please refer to the detailed [setup instructions](https://discopop-project.github.io/discopop/Setup), contact us via GitHub messages, or get in contact by mail to [[email protected]](mailto:[email protected]).
+
+### Prerequisites
+- Download `.deb` package from [latest Release](https://github.com/discopop-project/discopop/releases/latest)
+### Example
 ```
-# setup DiscoPoP
-git clone [email protected]:discopop-project/discopop.git
-cd discopop
-mkdir build && cd build
-DP_BUILD=$(pwd)
-cmake .. && make
+# install package
+sudo apt install ./<packagename>.deb
 # instrument and build the example code
-cd ../example
-mkdir build && cd build && cmake -DCMAKE_CXX_COMPILER=${DP_BUILD}/scripts/CXX_wrapper.sh .. && make
+cd /opt/DiscoPoP/example
+mkdir build && cd build && cmake -DCMAKE_CXX_COMPILER=discopop_cxx .. && make
 # execute instrumented code
 ./cmake_example
 # identify parallel patterns
 cd .discopop
 discopop_explorer
-# create applicable patches from patterns
-discopop_patch_generator
 # print patches to the console
 for f in $(find patch_generator -maxdepth 1 -type d); do
     echo "SUGGESTION: $f"

discopop_explorer/__main__.py

@@ -108,6 +108,10 @@ def parse_args() -> ExplorerArguments:
     experimental_parser.add_argument(
         "--enable-statistics", action="store_true", help="Disable the calculation and storing of statistics for code and generated suggestions."
     )
+    experimental_parser.add_argument(
+        "--plot-pet", type=str, nargs="?", default=None, const="explorer/pet_plot.gexf",
+        help="Plots PET as a GEXF file. If a path is given (file extension has to be .gexf), the PET Graph is written to the given file, otherwise to pet_plot.gexf"
+    )
     # fmt: on
 
     arguments = parser.parse_args()
@@ -135,6 +139,7 @@ def parse_args() -> ExplorerArguments:
     arguments.dump_pet = get_path_or_none(arguments.path, arguments.dump_pet)
     arguments.dump_detection_result = get_path_or_none(arguments.path, arguments.dump_detection_result)
     arguments.microbench_file = get_path_or_none(arguments.path, arguments.microbench_file)
+    arguments.plot_pet = get_path_or_none(arguments.path, arguments.plot_pet)
 
     return ExplorerArguments(
         discopop_build_path=arguments.dp_build_path,
@@ -162,6 +167,7 @@ def parse_args() -> ExplorerArguments:
         load_existing_doall_and_reduction_patterns=arguments.load_existing_doall_and_reduction_patterns,
         collect_statistics=arguments.enable_statistics,
         jobs=arguments.jobs,
+        enable_pet_plot_file=arguments.plot_pet,
     )
 
 

discopop_explorer/discopop_explorer.py

@@ -21,6 +21,7 @@
 import pstats2  # type:ignore
 from pluginbase import PluginBase  # type: ignore
 from discopop_explorer.functions.PEGraph.output.json import dump_to_pickled_json
+from discopop_explorer.functions.PEGraph.output.gephi import dump_to_gephi_file
 from discopop_explorer.utilities.statistics.collect_statistics import collect_statistics
 from discopop_library.ArgumentClasses.GeneralArguments import GeneralArguments  # type: ignore
 from discopop_library.FolderStructure.setup import setup_explorer
@@ -74,6 +75,8 @@ class ExplorerArguments(GeneralArguments):
     microbench_file: Optional[str]
     load_existing_doall_and_reduction_patterns: bool
     collect_statistics: bool
+    #enable_gephi_file: Optional[str]
+    enable_pet_plot_file: Optional[str] # None means no dump, otherwise the path 
 
     def __post_init__(self) -> None:
         self.__validate()
@@ -258,6 +261,11 @@ def run(arguments: ExplorerArguments) -> None:
                 f.flush()
                 f.close()
 
+        # experimental
+        # dumps gexf plot of pet
+        if arguments.enable_pet_plot_file is not None:
+            dump_to_gephi_file(res.pet, arguments.enable_pet_plot_file)
+
         if arguments.enable_detection_result_dump_file is not None:
             with open(arguments.enable_detection_result_dump_file, "w+") as f:
                 f.write(res.dump_to_pickled_json())

discopop_explorer/functions/PEGraph/output/gephi.py

@@ -12,7 +12,7 @@
 import networkx as nx  # type: ignore
 
 
-def dump_to_gephi_file(pet: PEGraphX, name: str = "pet.gexf") -> None:
+def dump_to_gephi_file(pet: PEGraphX, name: str) -> None:
     """Note: Destroys the PETGraph!"""
     # replace node data with label
     for node_id in pet.g.nodes:
@@ -29,4 +29,4 @@ def dump_to_gephi_file(pet: PEGraphX, name: str = "pet.gexf") -> None:
             if dep.dtype is None:
                 raise ValueError("dep.dtype has no type name!")
             pet.g.edges[edge]["dep_type"] = str(dep.dtype.name)
-    nx.write_gexf(pet.g, name)
+    nx.write_gexf(pet.g, name)

discopop_library/global_data/version/VERSION

@@ -1 +1 @@
-4.0.0
+4.0.1

docs/Gemfile.lock

@@ -56,13 +56,11 @@ GEM
     rb-fsevent (0.11.2)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
-    rexml (3.3.6)
-      strscan
+    rexml (3.3.9)
     rouge (4.0.0)
     safe_yaml (1.0.5)
     sassc (2.4.0)
       ffi (~> 1.9)
-    strscan (3.1.0)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
     unicode-display_width (2.3.0)

docs/How_to_contribute.md

@@ -43,6 +43,7 @@ Commit messages should follow the conventional commits format: [conventional com
 Execute the following steps in order to create a new DiscoPoP release:
 - Switch to the release branch (e.g. `release/1.2.3`) which shall be released
 - Update the version file in the repository (`discopop_library/global_data/version/VERSION`)
+- Create a `.deb` package by executing `scripts/dev/create_package.sh`. The package will be created in `packages`.
 - Create a pull request to the `master` branch and validate the changes
 - Merge the pull request by rebasing and create a tag on the `master` branch with the name `v1.2.3`
     - Creating the tag triggers the automatic publication of the project to PyPi
@@ -51,6 +52,7 @@ Execute the following steps in order to create a new DiscoPoP release:
   - Release tag: `v1.2.3`
   - Release title: `DiscoPoP Version 1.2.3`
   - Description should contain a summary of the most relevant changes
+  - Add the created `.deb` package to the list of files
 - If everything is fine, publish the new release
 
 ### Determining the Version Number

docs/setup/discopop.md

@@ -5,7 +5,16 @@ parent: Setup
 nav_order: 1
 ---
 
-# DiscoPoP Setup
+# DiscoPoP Setup - Package
+- Proposed method of installation for `users` of DiscoPoP
+- Download the `.deb` package of choice from [releases](https://github.com/discopop-project/discopop/releases).
+    - Packages for different targets and configurations might become available in the future
+- Install via a package manager of choice (example: `sudo apt install ./<packagename>.deb`)
+- Uninstall via a package manager of choice (example: `sudo apt remove discopop`)
+
+
+# DiscoPoP Setup - Manual
+- Proposed method of installation for `developers` of DiscoPoP
 ## Prerequisites
 - LLVM/clang version 11
 - Python version 3.6 or greater
@@ -47,5 +56,6 @@ where `<CMAKE_FLAGS>` can consist of any combination of the following flags and
 ## Testing the installation
 To test the installation, it is possible to execute the provided set of unit tests.
 ```
-python -m unittest -v
+cd <dp_source_dir>
+venv/bin/python -m unittest -v
 ```