Discopop project folder structure

.github/workflows/tests/profiler.sh

@@ -23,7 +23,7 @@ function test_discopopPass {
   cp ${DISCOPOP_SRC}/scripts/dp-fmap .
   ./dp-fmap
   clang++ -g -c -O0 -S -emit-llvm -fno-discard-value-names "$1" -o out.ll || return 1
-  opt-11 -S -load=${DISCOPOP_INSTALL}/libi/LLVMDiscoPoP.so --DiscoPoP out.ll -o out_dp.ll --fm-path FileMapping.txt || return 1
+  opt-11 -S -load=${DISCOPOP_INSTALL}/libi/LLVMDiscoPoP.so --DiscoPoP out.ll -o out_dp.ll || return 1
   clang++ out_dp.ll -o out_prof -L${DISCOPOP_INSTALL}/rtlib -lDiscoPoP_RT -lpthread || return 1
   ./out_prof || return 1
 }

DiscoPoP/DiscoPoP.cpp

@@ -112,6 +112,18 @@ bool DiscoPoP::doInitialization(Module &M) {
         errs() << "DiscoPoP | 190: init pass DiscoPoP \n";
     }
 
+    // prepare .discopop directory if not present
+      struct stat st1 = {0};
+      if (stat(".discopop", &st1) == -1){
+          mkdir(".discopop", 0777);
+      }
+      // prepare profiler directory if not present
+      struct stat st2 = {0};
+      if (stat(".discopop/profiler", &st2) == -1){
+          mkdir(".discopop/profiler", 0777);
+      }
+
+
 // CUGeneration
     {
         CUIDCounter = 0;
@@ -871,7 +883,7 @@ void DiscoPoP::fillStartEndLineNumbers(Node *root, LoopInfo &LI) {
 }
 
 void DiscoPoP::initializeCUIDCounter() {
-    std::string CUCounterFile = "DP_CUIDCounter.txt";
+    std::string CUCounterFile = ".discopop/profiler/DP_CUIDCounter.txt";
     if (dputil::fexists(CUCounterFile)) {
         std::fstream inCUIDCounter(CUCounterFile, std::ios_base::in);;
         inCUIDCounter >> CUIDCounter;
@@ -925,16 +937,17 @@ bool DiscoPoP::inlinedFunction(Function *F) {
 void DiscoPoP::instrument_function(llvm::Function *function, map <string, string> *trueVarNamesFromMetadataMap) {
 
     // get the corresponding file id
-    unsigned file_id = dp_reduction_get_file_id(function);
-    if (file_id == 0) {
+    int32_t tmp_file_id;
+    determineFileID(*function, tmp_file_id);
+    if (tmp_file_id == 0) {
         return;
     }
 
     llvm::LoopInfo &loop_info = getAnalysis<llvm::LoopInfoWrapperPass>(*function).getLoopInfo();
 
     for (auto loop_it = loop_info.begin(); loop_it != loop_info.end();
          ++loop_it) {
-        instrument_loop(*function, file_id, *loop_it, loop_info, trueVarNamesFromMetadataMap);
+        instrument_loop(*function, tmp_file_id, *loop_it, loop_info, trueVarNamesFromMetadataMap);
     }
 }
 
@@ -1811,7 +1824,7 @@ void DiscoPoP::dp_reduction_insert_functions() {
 
     // insert function calls to monitor loop iterations
     std::ofstream loop_metadata_file;
-    loop_metadata_file.open("loop_meta.txt");
+    loop_metadata_file.open(".discopop/profiler/loop_meta.txt");
     int loop_id = 1;
     llvm::Type* loop_incr_fn_arg_type = llvm::Type::getInt32Ty(*ctx_);
     llvm::ArrayRef<llvm::Type*> loop_incr_fn_args(loop_incr_fn_arg_type);
@@ -1999,16 +2012,18 @@ bool DiscoPoP::runOnModule(Module &M) {
     ctx_ = &module_->getContext();
 
     reduction_file = new std::ofstream();
-    reduction_file->open("reduction.txt", std::ios_base::app);
+    reduction_file->open(".discopop/profiler/reduction.txt", std::ios_base::app);
 
     loop_counter_file = new std::ofstream();
-    loop_counter_file->open("loop_counter_output.txt", std::ios_base::app);
+    loop_counter_file->open(".discopop/profiler/loop_counter_output.txt", std::ios_base::app);
 
+    /*
     bool success = dp_reduction_init_util(FileMappingPath);
     if (!success) {
         llvm::errs() << "could not find the FileMapping file: " << FileMappingPath << "\n";
         return false;
     }
+    */
     instrument_module(&M, &trueVarNamesFromMetadataMap);
 
     dp_reduction_insert_functions();
@@ -2435,7 +2450,7 @@ bool DiscoPoP::runOnFunction(Function &F) {
         // Report statically identified dependencies
 
         staticDependencyFile = new std::ofstream();
-        staticDependencyFile->open("static_dependencies.txt", std::ios_base::app);
+        staticDependencyFile->open(".discopop/profiler/static_dependencies.txt", std::ios_base::app);
 
         for (auto pair: conditionalBBDepMap) {
                 for (auto s: pair.second) {
@@ -2845,13 +2860,13 @@ string DiscoPoP::xmlEscape(string data) {
 
 void DiscoPoP::secureStream() {
     outOriginalVariables = new std::ofstream();
-    outOriginalVariables->open("OriginalVariables.txt", std::ios_base::app);
+    outOriginalVariables->open(".discopop/profiler/OriginalVariables.txt", std::ios_base::app);
 
     outCUs = new std::ofstream();
-    outCUs->open("Data.xml", std::ios_base::app);
+    outCUs->open(".discopop/profiler/Data.xml", std::ios_base::app);
 
     outCUIDCounter = new std::ofstream();
-    outCUIDCounter->open("DP_CUIDCounter.txt", std::ios_base::out);
+    outCUIDCounter->open(".discopop/profiler/DP_CUIDCounter.txt", std::ios_base::out);
 }
 
 string DiscoPoP::getLineNumbersString(set<int> LineNumbers) {

DiscoPoP/DiscoPoP.hpp

@@ -57,6 +57,9 @@
 #include <string.h>
 #include <utility>
 
+#include <sys/stat.h>
+#include <sys/types.h>
+
 #define DP_DEBUG false
 
 using namespace llvm;

discopop_explorer/__main__.py

@@ -25,20 +25,20 @@ def parse_args() -> ExplorerArguments:
     # fmt: off
     parser.add_argument(
         "--path", type=str, default="./",
-        help="Directory with input data")
+        help="Path to the .discopop directory to be analyzed")
     parser.add_argument(
-        "--cu-xml", type=str, default="Data.xml",
+        "--cu-xml", type=str, default="profiler/Data.xml",
         help="CU node xml file")
     parser.add_argument(
-        "--dep-file", type=str, default="dp_run_dep.txt",
+        "--dep-file", type=str, default="profiler/dynamic_dependencies.txt",
         help="Dependencies text file"
     )
     parser.add_argument(
-        "--loop-counter", type=str, default="loop_counter_output.txt",
+        "--loop-counter", type=str, default="profiler/loop_counter_output.txt",
         help="Loop counter data"
     )
     parser.add_argument(
-        "--reduction", type=str, default="reduction.txt",
+        "--reduction", type=str, default="profiler/reduction.txt",
         help="Reduction variables file"
     )
     parser.add_argument(
@@ -53,18 +53,18 @@ def parse_args() -> ExplorerArguments:
     )
     # flags related to output and formatting:
     parser.add_argument(
-        "--json", type=str, nargs="?", default=None, const="patterns.json",
+        "--json", type=str, nargs="?", default="explorer/patterns.json",
         help="Json output")
     parser.add_argument(
-        "--profiling", type=str, nargs="?", default=None, const="profiling_stats.txt",
+        "--profiling", type=str, nargs="?", default=None, const="explorer/profiling_stats.txt",
         help="Enable profiling. If a path is given, the profiling stats are written to the given file, otherwise to profiling_stats.txt",
     )
     parser.add_argument(
-        "--dump-pet", type=str, nargs="?", default=None, const="pet_dump.json",
+        "--dump-pet", type=str, nargs="?", default=None, const="explorer/pet_dump.json",
         help="Dump PET Graph to JSON file. If a path is given, the PET Graph is written to the given file, otherwise to pet_dump.json",
     )
     parser.add_argument(
-        "--dump-detection-result", type=str, nargs="?", default=None, const="detection_result_dump.json",
+        "--dump-detection-result", type=str, nargs="?", default="explorer/detection_result_dump.json",
         help="Dump DetectionResult object to JSON file. If a path is given, the DetectionResult object is written to the given file, otherwise to detection_result_dump.json. Contents are equivalent to the json output. NOTE: This dump contains a dump of the PET Graph!",
     )
 

discopop_explorer/discopop_explorer.py

@@ -141,6 +141,10 @@ def __run(
 
 def run(arguments: ExplorerArguments):
     """Run the discopop_explorer with the given arguments"""
+    # create explorer directory if not already present
+    if not os.path.exists(os.path.join(arguments.project_path, "explorer")):
+        os.mkdir(os.path.join(arguments.project_path, "explorer"))
+
     if arguments.enable_profiling_dump_file is not None:
         profile = cProfile.Profile()
         profile.enable()

discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/EntryPoint.py

@@ -133,7 +133,7 @@ def get_as_metadata(self, pet: PETGraphX, project_folder_path: str):
 
         # get size of memory region
         memory_region_sizes = get_sizes_of_memory_regions(
-            self.memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+            self.memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
         )
         if len(memory_region_sizes) > 0:
             max_mem_reg_size = max(memory_region_sizes.values())

discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/ExitPoint.py

@@ -131,7 +131,7 @@ def get_as_metadata(self, pet: PETGraphX, project_folder_path: str):
 
         # get size of memory region
         memory_region_sizes = get_sizes_of_memory_regions(
-            self.memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+            self.memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
         )
         if len(memory_region_sizes) > 0:
             max_mem_reg_size = max(memory_region_sizes.values())

discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/Update.py

@@ -180,7 +180,7 @@ def get_as_metadata_using_variable_names(self, pet: PETGraphX, project_folder_pa
 
         # get size of memory region
         memory_region_sizes = get_sizes_of_memory_regions(
-            self.memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+            self.memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
         )
         if len(memory_region_sizes) > 0:
             max_mem_reg_size = max(memory_region_sizes.values())
@@ -238,7 +238,7 @@ def get_as_metadata_using_variable_names_and_memory_regions(self, pet: PETGraphX
 
         # get size of memory region
         memory_region_sizes = get_sizes_of_memory_regions(
-            self.memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+            self.memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
         )
         if len(memory_region_sizes) > 0:
             max_mem_reg_size = max(memory_region_sizes.values())

discopop_explorer/pattern_detectors/simple_gpu_patterns/GPULoop.py

@@ -291,7 +291,7 @@ def __get_constructs(
 
                 # get size of memory region
                 memory_region_sizes = get_sizes_of_memory_regions(
-                    memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+                    memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
                 )
                 if len(memory_region_sizes) > 0:
                     max_mem_reg_size = max(memory_region_sizes.values())
@@ -318,7 +318,7 @@ def __get_constructs(
 
                 # get size of memory region
                 memory_region_sizes = get_sizes_of_memory_regions(
-                    memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+                    memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
                 )
                 if len(memory_region_sizes) > 0:
                     max_mem_reg_size = max(memory_region_sizes.values())
@@ -343,7 +343,7 @@ def __get_constructs(
 
                 # get size of memory region
                 memory_region_sizes = get_sizes_of_memory_regions(
-                    memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+                    memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
                 )
                 if len(memory_region_sizes) > 0:
                     max_mem_reg_size = max(memory_region_sizes.values())
@@ -368,7 +368,7 @@ def __get_constructs(
 
                 # get size of memory region
                 memory_region_sizes = get_sizes_of_memory_regions(
-                    memory_regions, os.path.join(project_folder_path, "memory_regions.txt")
+                    memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt")
                 )
                 if len(memory_region_sizes) > 0:
                     max_mem_reg_size = max(memory_region_sizes.values())

discopop_library/discopop_optimizer/Variables/Experiment.py

@@ -85,7 +85,7 @@ def __init__(
 
         self.__memory_region_sizes = get_sizes_of_memory_regions(
             set(),
-            os.path.join(discopop_output_path, "memory_regions.txt"),
+            os.path.join(discopop_output_path, "profiler/memory_regions.txt"),
             return_all_memory_regions=True,
         )
 

discopop_library/discopop_optimizer/__main__.py

@@ -18,7 +18,7 @@
 OPTIONAL ARGUMENTS:
     --project=<path>            Path to the directory that contains your makefile [default: .]
     --file-mapping=<path>       Path to the FileMapping.txt. [default: FileMapping.txt]
-    --detection-result-dump=<path>  Path to the dumped detection result JSON. [default: detection_result_dump.json]
+    --detection-result-dump=<path>  Path to the dumped detection result JSON. [default: explorer/detection_result_dump.json]
     --execute-created-models    Compiles, executes and measures models already stored in the project folder.
                                 Does not start the optimization pipeline.
                                 Required: --executable-name
@@ -29,8 +29,8 @@
     --execution-append-measurements     If set, measurement files from previous executions will be kept and new results
                                         will be appended.
     --clean-created-code        Removes all stored code modifications.
-    --code-export-path=<path>   Directory where generated CodeStorageObjects are located. [default: .discopop_optimizer/code_exports]
-    --dp-output-path=<path>     Directory where output files of DiscoPoP are located. [default: .discopop]
+    --code-export-path=<path>   Directory where generated CodeStorageObjects are located. [default: optimizer/code_exports]
+    --dp-output-path=<path>     Directory where output files of DiscoPoP are located. [default: .]
     --executable-name=<string>  Name of the executable generate by your makefile.
                                 Must be specified if --execute-created-models is used! [default: ]
     --executable-arguments=<string>

docs/How_to_contribute.md

@@ -64,3 +64,24 @@ A new version number shall be determined as follows:
 * Release adds a major new feature or modifies any interface (for example by modifying the format of input or output data) in such a way that it is <b>not fully compatible</b> with the previous version anymore.
     * ==> Increase first digit by `1`.
     * ==> Set remaining digits to `0`.
+
+# Developer hints
+## Output folder structure
+All tools developed as part of the DiscoPop Project make use of the following folder structure:
+```
+- project root/
+ - .discopop/
+  - FileMapping.txt
+  - <tool_1>/
+    - tool_1 output files
+    - private/
+     - tool_1 intermediate files
+  - <tool_2>/
+    - ...
+  - ...
+```
+, where no data should be stored / created outside the `.discopop` folder in order to keep the users build directory as clean as possible.
+Files which are read by different tools, e.g. the `FileMapping.txt`, shall be stored in the `.discopop` folder.
+Each tool may create a folder. In case data from these files is required by another tool, think about how to encode the information in a easy-to-use and structured format, preferrably JSON. If possible, please do not rely on exporting to simple `.txt` files as parsing adds a potential point of failure.
+Output data for use by other tools should be stored in the folder of the creating tool. Intermediate files may be stored in a folder named `private` and shall not be used by other tools.
+Analysis tools, like pattern detection scripts etc. shall be structured in such a way, that a execution from within the `.discopop` folder is intended.

example/execute_cmake_example.sh

@@ -16,18 +16,14 @@ DISCOPOP_BUILD_DIR=$(pwd)/../build
 rm -rf build
 mkdir build
 
-# generate FileMapping.txt
-${DISCOPOP_BUILD_DIR}/scripts/dp-fmap
-
 # Use DiscoPoP's compiler wrappers to build the cmake example
 # CMAKE_wrapper.sh acts as a substitute for the `cmake` command
 cd build
 ${DISCOPOP_BUILD_DIR}/scripts/CMAKE_wrapper.sh ..
-# Set the DP_FM_PATH environment variable
-DP_FM_PATH=${EXAMPLE_DIR}/../FileMapping.txt make
+make
 
 # execute the example
 ./cmake_example
-# Now, profiling results (e.g. cmake_example_dep.txt (dynamically identified data dependencies)) are available
+# Now, profiling results (e.g. dynamically identified data dependencies) are available
 # The created output can be used for pattern detection using the discopop_explorer.
 # Please refer to the Wiki for detailed information and instructions.

rtlib/iFunctions.cpp

@@ -308,7 +308,7 @@ namespace __dp {
 
     void outputAllocations() {
         auto allocationsFileStream = new ofstream();
-        allocationsFileStream->open("memory_regions.txt", ios::out);
+        allocationsFileStream->open(".discopop/profiler/memory_regions.txt", ios::out);
         for(auto memoryRegion : *allocatedMemoryRegions){
             string position = decodeLID(get<0>(memoryRegion));
             string id = get<1>(memoryRegion);
@@ -1088,7 +1088,8 @@ namespace __dp {
                       selfPath = nullptr;
                       out->open("Output.txt", ios::out);
                  }
-                 out->open(string(selfPath) + "_dep.txt", ios::out);
+                 //out->open(string(selfPath) + "_dep.txt", ios::out);  # results in the old <prog>_dep.txt
+                 out->open(".discopop/profiler/dynamic_dependencies.txt", ios::out);
             }
 #else
             out->open("Output.txt", ios::out);

rtlib/loop_counter.cpp

@@ -57,7 +57,7 @@ void loop_counter_output() {
     // get meta information about the loops
     std::vector <loop_info_t> loop_infos;
     loop_infos.push_back(loop_info_t()); // dummy
-    ifile.open("loop_meta.txt");
+    ifile.open(".discopop/profiler/loop_meta.txt");
     while (std::getline(ifile, line)) {
         loop_info_t loop_info;
         int cnt = sscanf(line.c_str(), "%d %d %d", &loop_info.file_id_,
@@ -69,7 +69,7 @@ void loop_counter_output() {
     ifile.close();
 
     // output information about the loops
-    ofile.open("loop_counter_output.txt");
+    ofile.open(".discopop/profiler/loop_counter_output.txt");
     for (auto i = 1; i < lc.loop_counters_.size(); ++i) {
         loop_info_t &loop_info = loop_infos[i];
         ofile << loop_info.file_id_ << " ";

scripts/CC_wrapper.sh

@@ -31,16 +31,16 @@ echo "WRAPPED CC COMPILE..."
 echo "ARGS: ${@}"
 echo "DP_FM_PATH: ${DP_FM_PATH}"
 
-# check if environment is prepared
-if [ -z ${DP_FM_PATH} ]; then
-  echo "ERROR: DP_FM_PATH unspecified!"
-  echo "  Generate a FileMapping.txt file and create an environment Variable which points to this file."
-  echo "  Please refer to https://discopop-project.github.io/discopop/Profiling/File_Mapping/ for further information."
-  exit 1
-fi
-
-echo "${LLVM_CLANG} -g -c -O0 -S -emit-llvm -fno-discard-value-names ${@} -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} "
+## check if environment is prepared
+#if [ -z ${DP_FM_PATH} ]; then
+#  echo "ERROR: DP_FM_PATH unspecified!"
+#  echo "  Generate a FileMapping.txt file and create an environment Variable which points to this file."
+#  echo "  Please refer to https://discopop-project.github.io/discopop/Profiling/File_Mapping/ for further information."
+#  exit 1
+#fi
+
+echo "${LLVM_CLANG} -g -c -O0 -S -emit-llvm -fno-discard-value-names ${@} -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP"
 #clang-11 -g -c -O0 -S -emit-llvm -fno-discard-value-names "$@" -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH}
-${LLVM_CLANG} "$@" -g -c -O0 -fno-discard-value-names -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} -fPIC
+${LLVM_CLANG} "$@" -g -O0 -fno-discard-value-names -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -fPIC -Xlinker -L${DP_BUILD}/rtlib -Xlinker -lDiscoPoP_RT -Xlinker -lpthread -Xlinker -v -Xlinker -lstdc++
 
 # WARNING: OUTPUT IS A .ll FILE, ENDING IS .o