From 8a0021b6aa744bb1ed529810f26391ec4a8bbb3a Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Tue, 17 Oct 2023 15:14:17 +0200 Subject: [PATCH 01/14] feat(llvm-args): removed fmap argument --- scripts/CC_wrapper.sh | 16 ++++++++-------- scripts/CMAKE_wrapper.sh | 10 ++++++---- scripts/CXX_wrapper.sh | 17 ++++++++--------- scripts/MPI_CC_wrapper.sh | 2 +- scripts/MPI_CXX_wrapper.sh | 2 +- share/include/DPUtils.hpp | 3 ++- share/lib/DPUtils.cpp | 6 ++++-- 7 files changed, 30 insertions(+), 26 deletions(-) diff --git a/scripts/CC_wrapper.sh b/scripts/CC_wrapper.sh index 01cfea9cc..511840b46 100755 --- a/scripts/CC_wrapper.sh +++ b/scripts/CC_wrapper.sh @@ -31,16 +31,16 @@ echo "WRAPPED CC COMPILE..." echo "ARGS: ${@}" echo "DP_FM_PATH: ${DP_FM_PATH}" -# check if environment is prepared -if [ -z ${DP_FM_PATH} ]; then - echo "ERROR: DP_FM_PATH unspecified!" - echo " Generate a FileMapping.txt file and create an environment Variable which points to this file." - echo " Please refer to https://discopop-project.github.io/discopop/Profiling/File_Mapping/ for further information." - exit 1 -fi +## check if environment is prepared +#if [ -z ${DP_FM_PATH} ]; then +# echo "ERROR: DP_FM_PATH unspecified!" +# echo " Generate a FileMapping.txt file and create an environment Variable which points to this file." +# echo " Please refer to https://discopop-project.github.io/discopop/Profiling/File_Mapping/ for further information." +# exit 1 +#fi echo "${LLVM_CLANG} -g -c -O0 -S -emit-llvm -fno-discard-value-names ${@} -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} " #clang-11 -g -c -O0 -S -emit-llvm -fno-discard-value-names "$@" -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} -${LLVM_CLANG} "$@" -g -c -O0 -fno-discard-value-names -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} -fPIC +${LLVM_CLANG} "$@" -g -O0 -fno-discard-value-names -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -fPIC -Xlinker -L${DP_BUILD}/rtlib -Xlinker -lDiscoPoP_RT -Xlinker -lpthread -Xlinker -v -Xlinker -lstdc++ # WARNING: OUTPUT IS A .ll FILE, ENDING IS .o diff --git a/scripts/CMAKE_wrapper.sh b/scripts/CMAKE_wrapper.sh index 7db6a7626..c905f9040 100755 --- a/scripts/CMAKE_wrapper.sh +++ b/scripts/CMAKE_wrapper.sh @@ -30,10 +30,12 @@ echo "ARGS: ${@}" # execute cmake using CC, CXX and LINKER wrappers # re-define structure of the link-executable to include the updated linker cmake \ - -DCMAKE_C_COMPILER_WORKS=1 \ - -DCMAKE_CXX_COMPILER_WORKS=1 \ -DCMAKE_CXX_COMPILER=${DP_SCRIPTS}/CXX_wrapper.sh \ -DCMAKE_C_COMPILER=${DP_SCRIPTS}/CC_wrapper.sh \ - -DCMAKE_LINKER=${DP_SCRIPTS}/LINKER_wrapper.sh \ - -DCMAKE_CXX_LINK_EXECUTABLE=" -o " \ "$@" +# -DCMAKE_C_COMPILER_WORKS=1 \ +# -DCMAKE_CXX_COMPILER_WORKS=1 \ + +# -DCMAKE_LINKER=${DP_SCRIPTS}/LINKER_wrapper.sh \ +# -DCMAKE_CXX_LINK_EXECUTABLE=" -o " \ +# -DCMAKE_C_LINK_EXECUTABLE=" -o " \ diff --git a/scripts/CXX_wrapper.sh b/scripts/CXX_wrapper.sh index 9efb97051..70ac6eac2 100755 --- a/scripts/CXX_wrapper.sh +++ b/scripts/CXX_wrapper.sh @@ -31,16 +31,15 @@ echo "WRAPPED CXX COMPILE..." echo "ARGS: ${@}" echo "DP_FM_PATH: ${DP_FM_PATH}" -# check if environment is prepared -if [ -z ${DP_FM_PATH} ]; then - echo "ERROR: DP_FM_PATH unspecified!" - echo " Generate a FileMapping.txt file and create an environment Variable which points to this file." - echo " Please refer to https://discopop-project.github.io/discopop/Profiling/File_Mapping/ for further information." - exit 1 -fi +## check if environment is prepared +#if [ -z ${DP_FM_PATH} ]; then +# echo "ERROR: DP_FM_PATH unspecified!" +# echo " Generate a FileMapping.txt file and create an environment Variable which points to this file." +# echo " Please refer to https://discopop-project.github.io/discopop/Profiling/File_Mapping/ for further information." +# exit 1 +#fi echo "${LLVM_CLANGPP} -g -c -O0 -S -emit-llvm -fno-discard-value-names ${@} -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} " #clang++-11 -g -c -O0 -S -emit-llvm -fno-discard-value-names "$@" -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} -${LLVM_CLANGPP} "$@" -g -c -O0 -fno-discard-value-names -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} -fPIC - +${LLVM_CLANGPP} "$@" -g -O0 -fno-discard-value-names -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -fPIC -Xlinker -L${DP_BUILD}/rtlib -Xlinker -lDiscoPoP_RT -Xlinker -lpthread -Xlinker -v # WARNING: OUTPUT IS A .ll FILE, ENDING IS .o diff --git a/scripts/MPI_CC_wrapper.sh b/scripts/MPI_CC_wrapper.sh index 56b06f19a..f72a14296 100755 --- a/scripts/MPI_CC_wrapper.sh +++ b/scripts/MPI_CC_wrapper.sh @@ -49,6 +49,6 @@ fi echo "${LLVM_CLANG} ${MPI_INCLUDES} -g -c -S -emit-llvm -fno-discard-value-names ${@} -O0 -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH}" #clang-11 -g -c -O0 -S -emit-llvm -fno-discard-value-names "$@" -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} -${LLVM_CLANG} "${@}" -g -c -O0 -fno-discard-value-names -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} -fPIC ${MPI_INCLUDES} +${LLVM_CLANG} "${@}" -g -c -O0 -fno-discard-value-names -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -fPIC ${MPI_INCLUDES} -Xlinker -lstdc++ # WARNING: OUTPUT IS A .ll FILE, ENDING IS .o diff --git a/scripts/MPI_CXX_wrapper.sh b/scripts/MPI_CXX_wrapper.sh index 1bb5d63e5..b053454bc 100755 --- a/scripts/MPI_CXX_wrapper.sh +++ b/scripts/MPI_CXX_wrapper.sh @@ -49,6 +49,6 @@ fi echo "${LLVM_CLANGPP} ${MPI_INCLUDES} -g -c -O0 -S -emit-llvm -fno-discard-value-names ${@} -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH}" #clang++-11 -g -c -O0 -S -emit-llvm -fno-discard-value-names "$@" -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} -${LLVM_CLANGPP} "$@" -g -c -O0 -fno-discard-value-names -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} -fPIC ${MPI_INCLUDES} +${LLVM_CLANGPP} "$@" -g -c -O0 -fno-discard-value-names -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -fPIC ${MPI_INCLUDES} # WARNING: OUTPUT IS A .ll FILE, ENDING IS .o diff --git a/share/include/DPUtils.hpp b/share/include/DPUtils.hpp index 748f6002f..94cc658d4 100644 --- a/share/include/DPUtils.hpp +++ b/share/include/DPUtils.hpp @@ -72,7 +72,8 @@ typedef int64_t ADDR; using namespace std; using namespace llvm; -extern cl::opt FileMappingPath; +//extern cl::opt FileMappingPath; +extern string FileMappingPath; extern cl::opt DP_MEMORY_PROFILING_SKIP_FUNCTION_ARGUMENTS; namespace dputil { diff --git a/share/lib/DPUtils.cpp b/share/lib/DPUtils.cpp index a9e41e57c..9bb98b5bb 100644 --- a/share/lib/DPUtils.cpp +++ b/share/lib/DPUtils.cpp @@ -14,8 +14,10 @@ #define DP_DEBUG_TYPE "dputils" using namespace std; -cl::opt FileMappingPath("fm-path", cl::init(""), - cl::desc("Specify file mapping location"), cl::Hidden); +//cl::opt FileMappingPath("fm-path", cl::init(""), +// cl::desc("Specify file mapping location"), cl::Hidden); +string FileMappingPath = ".discopop/common_data/FileMapping.txt"; + cl::opt DP_MEMORY_PROFILING_SKIP_FUNCTION_ARGUMENTS("memory-profiling-skip-function-arguments", cl::init(false), cl::desc("disable the memory profiling for allocations which belong to function arguments"), cl::Hidden); From c00c05d20f73f7e12050a0e1436ca090e7849e8f Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Tue, 17 Oct 2023 15:14:44 +0200 Subject: [PATCH 02/14] feat[fmap]: automatic creation of FileMapping.txt during instrumentation --- DiscoPoP/DiscoPoP.cpp | 27 ++++++++++-- share/lib/DPUtils.cpp | 100 +++++++++++++++++++++++++++--------------- 2 files changed, 89 insertions(+), 38 deletions(-) diff --git a/DiscoPoP/DiscoPoP.cpp b/DiscoPoP/DiscoPoP.cpp index 74bf932b8..4b4b43613 100644 --- a/DiscoPoP/DiscoPoP.cpp +++ b/DiscoPoP/DiscoPoP.cpp @@ -112,6 +112,24 @@ bool DiscoPoP::doInitialization(Module &M) { errs() << "DiscoPoP | 190: init pass DiscoPoP \n"; } + // prepare .discopop directory if not present + struct stat st1 = {0}; + if (stat(".discopop", &st1) == -1){ + mkdir(".discopop", 0777); + } + // prepare profiler directory if not present + struct stat st2 = {0}; + if (stat(".discopop/profiler", &st2) == -1){ + mkdir(".discopop/profiler", 0777); + } + + // prepare common_data directory if not present + struct stat st3 = {0}; + if (stat(".discopop/common_data", &st3) == -1){ + mkdir(".discopop/common_data", 0777); + } + + // CUGeneration { CUIDCounter = 0; @@ -925,8 +943,9 @@ bool DiscoPoP::inlinedFunction(Function *F) { void DiscoPoP::instrument_function(llvm::Function *function, map *trueVarNamesFromMetadataMap) { // get the corresponding file id - unsigned file_id = dp_reduction_get_file_id(function); - if (file_id == 0) { + int32_t tmp_file_id; + determineFileID(*function, tmp_file_id); + if (tmp_file_id == 0) { return; } @@ -934,7 +953,7 @@ void DiscoPoP::instrument_function(llvm::Function *function, map open("loop_counter_output.txt", std::ios_base::app); + /* bool success = dp_reduction_init_util(FileMappingPath); if (!success) { llvm::errs() << "could not find the FileMapping file: " << FileMappingPath << "\n"; return false; } + */ instrument_module(&M, &trueVarNamesFromMetadataMap); dp_reduction_insert_functions(); diff --git a/share/lib/DPUtils.cpp b/share/lib/DPUtils.cpp index 9bb98b5bb..65ebcc4c7 100644 --- a/share/lib/DPUtils.cpp +++ b/share/lib/DPUtils.cpp @@ -25,6 +25,39 @@ cl::opt DP_MEMORY_PROFILING_SKIP_FUNCTION_ARGUMENTS("memory-profiling-ski namespace dputil { int32_t getFileID(string fileMapping, string fullPathName) { + int tempfid = 1; + fstream fileMappingFile; + + fileMappingFile.open(".discopop/common_data/FileMapping.txt", ios::in); + if (fileMappingFile) + { + string tp; + while (getline(fileMappingFile, tp)) + { + std::string id = tp.substr(0, tp.find("\t")); + std::string file_name = tp.substr(tp.find("\t") + 1); + if (file_name == fullPathName){ + return stoi(id); + } + tempfid++; + } + fileMappingFile.close(); + + fileMappingFile.open(".discopop/common_data/FileMapping.txt", std::ios_base::app); + fileMappingFile << tempfid << "\t" << fullPathName << "\n"; + fileMappingFile.close(); + errs() << "added fmap entry: " << tempfid << "\t" << fullPathName << "\n"; + return tempfid; + } + else + { + fileMappingFile.open(".discopop/common_data/FileMapping.txt", std::ios_base::app); + fileMappingFile << tempfid << "\t" << fullPathName << "\n"; + fileMappingFile.close(); + errs() << "added fmap entry: " << tempfid << "\t" << fullPathName << "\n"; + return tempfid; + } + /* int32_t index = 0; // if the associated file id is not found, then we return 0 string line; ifstream fileMap(fileMapping.c_str()); @@ -46,6 +79,7 @@ namespace dputil { fileMap.close(); } return index; + */ } // Encode the fileID and line number of BI as LID. @@ -108,47 +142,43 @@ namespace dputil { void determineFileID(Function &F, int32_t &fileID) { fileID = 0; - // if FileMapping.txt is not given, we use 1 as file index - if (!dputil::fexists(FileMappingPath)) { - fileID = 1; - } else { - for (Function::iterator FI = F.begin(), FE = F.end(); FI != FE; ++FI) { - BasicBlock &BB = *FI; - for (BasicBlock::iterator BI = BB.begin(), EI = BB.end(); BI != EI; ++BI) { - int32_t lno; - const DebugLoc &location = BI->getDebugLoc(); - if (location) { - lno = BI->getDebugLoc().getLine(); - } else { - lno = 0; - } + for (Function::iterator FI = F.begin(), FE = F.end(); FI != FE; ++FI) { + BasicBlock &BB = *FI; + for (BasicBlock::iterator BI = BB.begin(), EI = BB.end(); BI != EI; ++BI) { + int32_t lno; + const DebugLoc &location = BI->getDebugLoc(); + if (location) { + lno = BI->getDebugLoc().getLine(); + } else { + lno = 0; + } + + if (lno) { + MDNode *N = BI->getMetadata("dbg"); + // N == NULL means BI is only a helper instruction. + // No metadata is attached to BI. + if (N) { + StringRef File = "", Dir = ""; + const DILocation *Loc = location.get(); + File = Loc->getFilename(); + Dir = Loc->getDirectory(); - if (lno) { - MDNode *N = BI->getMetadata("dbg"); - // N == NULL means BI is only a helper instruction. - // No metadata is attached to BI. - if (N) { - StringRef File = "", Dir = ""; - const DILocation *Loc = location.get(); - File = Loc->getFilename(); - Dir = Loc->getDirectory(); - - char *absolutePathFileName = realpath((Dir.str() + "/" + File.str()).c_str(), NULL); - - if (absolutePathFileName == NULL) { - absolutePathFileName = realpath(File.data(), NULL); - } - - if (absolutePathFileName) { - fileID = dputil::getFileID(FileMappingPath, string(absolutePathFileName)); - delete[] absolutePathFileName; - } - break; + char *absolutePathFileName = realpath((Dir.str() + "/" + File.str()).c_str(), NULL); + + if (absolutePathFileName == NULL) { + absolutePathFileName = realpath(File.data(), NULL); } + + if (absolutePathFileName) { + fileID = dputil::getFileID(FileMappingPath, string(absolutePathFileName)); + delete[] absolutePathFileName; + } + break; } } } } + } string get_exe_dir() { From 4f1f9733791afd56174b7a6664492a1049697489 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Tue, 17 Oct 2023 15:19:09 +0200 Subject: [PATCH 03/14] fix(Instrumentation): added missing includes --- DiscoPoP/DiscoPoP.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DiscoPoP/DiscoPoP.hpp b/DiscoPoP/DiscoPoP.hpp index c46286a16..3099fd978 100644 --- a/DiscoPoP/DiscoPoP.hpp +++ b/DiscoPoP/DiscoPoP.hpp @@ -57,6 +57,10 @@ #include #include +#include +#include + + #define DP_DEBUG false using namespace llvm; From 4243ee78eaa3a2ba27048099c55dd158e239158b Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Tue, 17 Oct 2023 15:21:59 +0200 Subject: [PATCH 04/14] chore: formatting --- DiscoPoP/DiscoPoP.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/DiscoPoP/DiscoPoP.hpp b/DiscoPoP/DiscoPoP.hpp index 3099fd978..9f4241417 100644 --- a/DiscoPoP/DiscoPoP.hpp +++ b/DiscoPoP/DiscoPoP.hpp @@ -60,7 +60,6 @@ #include #include - #define DP_DEBUG false using namespace llvm; From d8c39a61216ad17de4ea1639c5d7bc197fe65803 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Fri, 13 Oct 2023 10:22:22 +0200 Subject: [PATCH 05/14] doc[developer-guide]: added overview of folder structure --- docs/How_to_contribute.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/How_to_contribute.md b/docs/How_to_contribute.md index 7d9e51017..80fdc27e4 100644 --- a/docs/How_to_contribute.md +++ b/docs/How_to_contribute.md @@ -64,3 +64,25 @@ A new version number shall be determined as follows: * Release adds a major new feature or modifies any interface (for example by modifying the format of input or output data) in such a way that it is not fully compatible with the previous version anymore. * ==> Increase first digit by `1`. * ==> Set remaining digits to `0`. + +# Developer hints +## Output folder structure +All tools developed as part of the DiscoPop Project make use of the following folder structure: +``` +- project root/ + - .discopop/ + - common_data/ + - FileMapping.txt + - / + - tool_1 output files + - private/ + - tool_1 intermediate files + - / + - ... + - ... +``` +, where no data should be stored / created outside the `.discopop` folder in order to keep the users build directory as clean as possible. +Files which are read by different tools, e.g. the `FileMapping.txt`, shall be stored in the `.discopop/common_data` folder. +Each tool may create a folder. In case data from these files is required by another tool, think about how to encode the information in a easy-to-use and structured format, preferrably JSON. If possible, please do not rely on exporting to simple `.txt` files as parsing adds a potential point of failure. +Output data for use by other tools should be stored in the folder of the creating tool. Intermediate files may be stored in a folder named `private` and shall not be used by other tools. +Analysis tools, like pattern detection scripts etc. shall be structured in such a way, that a execution from within the `.discopop` folder is intended. From 657fc1b5b0a9d9b270d613ab10b19502a7c09929 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Fri, 13 Oct 2023 10:55:01 +0200 Subject: [PATCH 06/14] feat(structure): updated output structure of the profiler --- DiscoPoP/DiscoPoP.cpp | 16 ++++++++-------- rtlib/iFunctions.cpp | 5 +++-- rtlib/loop_counter.cpp | 4 ++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/DiscoPoP/DiscoPoP.cpp b/DiscoPoP/DiscoPoP.cpp index 4b4b43613..41d585d78 100644 --- a/DiscoPoP/DiscoPoP.cpp +++ b/DiscoPoP/DiscoPoP.cpp @@ -889,7 +889,7 @@ void DiscoPoP::fillStartEndLineNumbers(Node *root, LoopInfo &LI) { } void DiscoPoP::initializeCUIDCounter() { - std::string CUCounterFile = "DP_CUIDCounter.txt"; + std::string CUCounterFile = ".discopop/profiler/DP_CUIDCounter.txt"; if (dputil::fexists(CUCounterFile)) { std::fstream inCUIDCounter(CUCounterFile, std::ios_base::in);; inCUIDCounter >> CUIDCounter; @@ -1830,7 +1830,7 @@ void DiscoPoP::dp_reduction_insert_functions() { // insert function calls to monitor loop iterations std::ofstream loop_metadata_file; - loop_metadata_file.open("loop_meta.txt"); + loop_metadata_file.open(".discopop/profiler/loop_meta.txt"); int loop_id = 1; llvm::Type* loop_incr_fn_arg_type = llvm::Type::getInt32Ty(*ctx_); llvm::ArrayRef loop_incr_fn_args(loop_incr_fn_arg_type); @@ -2018,10 +2018,10 @@ bool DiscoPoP::runOnModule(Module &M) { ctx_ = &module_->getContext(); reduction_file = new std::ofstream(); - reduction_file->open("reduction.txt", std::ios_base::app); + reduction_file->open(".discopop/profiler/reduction.txt", std::ios_base::app); loop_counter_file = new std::ofstream(); - loop_counter_file->open("loop_counter_output.txt", std::ios_base::app); + loop_counter_file->open(".discopop/profiler/loop_counter_output.txt", std::ios_base::app); /* bool success = dp_reduction_init_util(FileMappingPath); @@ -2456,7 +2456,7 @@ bool DiscoPoP::runOnFunction(Function &F) { // Report statically identified dependencies staticDependencyFile = new std::ofstream(); - staticDependencyFile->open("static_dependencies.txt", std::ios_base::app); + staticDependencyFile->open(".discopop/profiler/static_dependencies.txt", std::ios_base::app); for (auto pair: conditionalBBDepMap) { for (auto s: pair.second) { @@ -2866,13 +2866,13 @@ string DiscoPoP::xmlEscape(string data) { void DiscoPoP::secureStream() { outOriginalVariables = new std::ofstream(); - outOriginalVariables->open("OriginalVariables.txt", std::ios_base::app); + outOriginalVariables->open(".discopop/profiler/OriginalVariables.txt", std::ios_base::app); outCUs = new std::ofstream(); - outCUs->open("Data.xml", std::ios_base::app); + outCUs->open(".discopop/profiler/Data.xml", std::ios_base::app); outCUIDCounter = new std::ofstream(); - outCUIDCounter->open("DP_CUIDCounter.txt", std::ios_base::out); + outCUIDCounter->open(".discopop/profiler/DP_CUIDCounter.txt", std::ios_base::out); } string DiscoPoP::getLineNumbersString(set LineNumbers) { diff --git a/rtlib/iFunctions.cpp b/rtlib/iFunctions.cpp index e32c4f959..5583c6898 100755 --- a/rtlib/iFunctions.cpp +++ b/rtlib/iFunctions.cpp @@ -308,7 +308,7 @@ namespace __dp { void outputAllocations() { auto allocationsFileStream = new ofstream(); - allocationsFileStream->open("memory_regions.txt", ios::out); + allocationsFileStream->open(".discopop/profiler/memory_regions.txt", ios::out); for(auto memoryRegion : *allocatedMemoryRegions){ string position = decodeLID(get<0>(memoryRegion)); string id = get<1>(memoryRegion); @@ -1088,7 +1088,8 @@ namespace __dp { selfPath = nullptr; out->open("Output.txt", ios::out); } - out->open(string(selfPath) + "_dep.txt", ios::out); + //out->open(string(selfPath) + "_dep.txt", ios::out); # results in the old _dep.txt + out->open(".discopop/profiler/dynamic_dependencies.txt", ios::out); } #else out->open("Output.txt", ios::out); diff --git a/rtlib/loop_counter.cpp b/rtlib/loop_counter.cpp index 35c3e7e03..ef63cbf01 100644 --- a/rtlib/loop_counter.cpp +++ b/rtlib/loop_counter.cpp @@ -57,7 +57,7 @@ void loop_counter_output() { // get meta information about the loops std::vector loop_infos; loop_infos.push_back(loop_info_t()); // dummy - ifile.open("loop_meta.txt"); + ifile.open(".discopop/profiler/loop_meta.txt"); while (std::getline(ifile, line)) { loop_info_t loop_info; int cnt = sscanf(line.c_str(), "%d %d %d", &loop_info.file_id_, @@ -69,7 +69,7 @@ void loop_counter_output() { ifile.close(); // output information about the loops - ofile.open("loop_counter_output.txt"); + ofile.open(".discopop/profiler/loop_counter_output.txt"); for (auto i = 1; i < lc.loop_counters_.size(); ++i) { loop_info_t &loop_info = loop_infos[i]; ofile << loop_info.file_id_ << " "; From ffe614b5864ae267d7b915867e3d4386e97980ad Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Fri, 13 Oct 2023 11:08:40 +0200 Subject: [PATCH 07/14] feat(structure): updated explorer to use new project folder structure --- discopop_explorer/__main__.py | 20 ++++++++++---------- discopop_explorer/discopop_explorer.py | 4 ++++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/discopop_explorer/__main__.py b/discopop_explorer/__main__.py index befb358b3..7bd7fe9fb 100644 --- a/discopop_explorer/__main__.py +++ b/discopop_explorer/__main__.py @@ -25,24 +25,24 @@ def parse_args() -> ExplorerArguments: # fmt: off parser.add_argument( "--path", type=str, default="./", - help="Directory with input data") + help="Path to the .discopop directory to be analyzed") parser.add_argument( - "--cu-xml", type=str, default="Data.xml", + "--cu-xml", type=str, default="profiler/Data.xml", help="CU node xml file") parser.add_argument( - "--dep-file", type=str, default="dp_run_dep.txt", + "--dep-file", type=str, default="profiler/dynamic_dependencies.txt", help="Dependencies text file" ) parser.add_argument( - "--loop-counter", type=str, default="loop_counter_output.txt", + "--loop-counter", type=str, default="profiler/loop_counter_output.txt", help="Loop counter data" ) parser.add_argument( - "--reduction", type=str, default="reduction.txt", + "--reduction", type=str, default="profiler/reduction.txt", help="Reduction variables file" ) parser.add_argument( - "--fmap", type=str, default="FileMapping.txt", + "--fmap", type=str, default="common_data/FileMapping.txt", help="File mapping") parser.add_argument( "--plugins", type=str, nargs="*", default=[], @@ -53,18 +53,18 @@ def parse_args() -> ExplorerArguments: ) # flags related to output and formatting: parser.add_argument( - "--json", type=str, nargs="?", default=None, const="patterns.json", + "--json", type=str, nargs="?", default=None, const="explorer/patterns.json", help="Json output") parser.add_argument( - "--profiling", type=str, nargs="?", default=None, const="profiling_stats.txt", + "--profiling", type=str, nargs="?", default=None, const="explorer/profiling_stats.txt", help="Enable profiling. If a path is given, the profiling stats are written to the given file, otherwise to profiling_stats.txt", ) parser.add_argument( - "--dump-pet", type=str, nargs="?", default=None, const="pet_dump.json", + "--dump-pet", type=str, nargs="?", default=None, const="explorer/pet_dump.json", help="Dump PET Graph to JSON file. If a path is given, the PET Graph is written to the given file, otherwise to pet_dump.json", ) parser.add_argument( - "--dump-detection-result", type=str, nargs="?", default=None, const="detection_result_dump.json", + "--dump-detection-result", type=str, nargs="?", default=None, const="explorer/detection_result_dump.json", help="Dump DetectionResult object to JSON file. If a path is given, the DetectionResult object is written to the given file, otherwise to detection_result_dump.json. Contents are equivalent to the json output. NOTE: This dump contains a dump of the PET Graph!", ) diff --git a/discopop_explorer/discopop_explorer.py b/discopop_explorer/discopop_explorer.py index 8156c71b3..557512265 100644 --- a/discopop_explorer/discopop_explorer.py +++ b/discopop_explorer/discopop_explorer.py @@ -141,6 +141,10 @@ def __run( def run(arguments: ExplorerArguments): """Run the discopop_explorer with the given arguments""" + # create explorer directory if not already present + if not os.path.exists(os.path.join(arguments.project_path, "explorer")): + os.mkdir(os.path.join(arguments.project_path, "explorer")) + if arguments.enable_profiling_dump_file is not None: profile = cProfile.Profile() profile.enable() From 3720c061d0ead95b1a5e6d804c0f4a270815214b Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Mon, 16 Oct 2023 09:58:54 +0200 Subject: [PATCH 08/14] fix: file paths --- .../combined_gpu_patterns/classes/EntryPoint.py | 2 +- .../combined_gpu_patterns/classes/ExitPoint.py | 2 +- .../combined_gpu_patterns/classes/Update.py | 4 ++-- .../pattern_detectors/simple_gpu_patterns/GPULoop.py | 8 ++++---- .../discopop_optimizer/Variables/Experiment.py | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/EntryPoint.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/EntryPoint.py index 76724ca3d..9c231cc0b 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/EntryPoint.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/EntryPoint.py @@ -133,7 +133,7 @@ def get_as_metadata(self, pet: PETGraphX, project_folder_path: str): # get size of memory region memory_region_sizes = get_sizes_of_memory_regions( - self.memory_regions, os.path.join(project_folder_path, "memory_regions.txt") + self.memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt") ) if len(memory_region_sizes) > 0: max_mem_reg_size = max(memory_region_sizes.values()) diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/ExitPoint.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/ExitPoint.py index 423b520f9..adebf6565 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/ExitPoint.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/ExitPoint.py @@ -131,7 +131,7 @@ def get_as_metadata(self, pet: PETGraphX, project_folder_path: str): # get size of memory region memory_region_sizes = get_sizes_of_memory_regions( - self.memory_regions, os.path.join(project_folder_path, "memory_regions.txt") + self.memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt") ) if len(memory_region_sizes) > 0: max_mem_reg_size = max(memory_region_sizes.values()) diff --git a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/Update.py b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/Update.py index 404866454..f260a350b 100644 --- a/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/Update.py +++ b/discopop_explorer/pattern_detectors/combined_gpu_patterns/classes/Update.py @@ -180,7 +180,7 @@ def get_as_metadata_using_variable_names(self, pet: PETGraphX, project_folder_pa # get size of memory region memory_region_sizes = get_sizes_of_memory_regions( - self.memory_regions, os.path.join(project_folder_path, "memory_regions.txt") + self.memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt") ) if len(memory_region_sizes) > 0: max_mem_reg_size = max(memory_region_sizes.values()) @@ -238,7 +238,7 @@ def get_as_metadata_using_variable_names_and_memory_regions(self, pet: PETGraphX # get size of memory region memory_region_sizes = get_sizes_of_memory_regions( - self.memory_regions, os.path.join(project_folder_path, "memory_regions.txt") + self.memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt") ) if len(memory_region_sizes) > 0: max_mem_reg_size = max(memory_region_sizes.values()) diff --git a/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPULoop.py b/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPULoop.py index 65c98270e..4a14cfaaa 100644 --- a/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPULoop.py +++ b/discopop_explorer/pattern_detectors/simple_gpu_patterns/GPULoop.py @@ -291,7 +291,7 @@ def __get_constructs( # get size of memory region memory_region_sizes = get_sizes_of_memory_regions( - memory_regions, os.path.join(project_folder_path, "memory_regions.txt") + memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt") ) if len(memory_region_sizes) > 0: max_mem_reg_size = max(memory_region_sizes.values()) @@ -318,7 +318,7 @@ def __get_constructs( # get size of memory region memory_region_sizes = get_sizes_of_memory_regions( - memory_regions, os.path.join(project_folder_path, "memory_regions.txt") + memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt") ) if len(memory_region_sizes) > 0: max_mem_reg_size = max(memory_region_sizes.values()) @@ -343,7 +343,7 @@ def __get_constructs( # get size of memory region memory_region_sizes = get_sizes_of_memory_regions( - memory_regions, os.path.join(project_folder_path, "memory_regions.txt") + memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt") ) if len(memory_region_sizes) > 0: max_mem_reg_size = max(memory_region_sizes.values()) @@ -368,7 +368,7 @@ def __get_constructs( # get size of memory region memory_region_sizes = get_sizes_of_memory_regions( - memory_regions, os.path.join(project_folder_path, "memory_regions.txt") + memory_regions, os.path.join(project_folder_path, "profiler/memory_regions.txt") ) if len(memory_region_sizes) > 0: max_mem_reg_size = max(memory_region_sizes.values()) diff --git a/discopop_library/discopop_optimizer/Variables/Experiment.py b/discopop_library/discopop_optimizer/Variables/Experiment.py index 229f1e5e2..d2f95677c 100644 --- a/discopop_library/discopop_optimizer/Variables/Experiment.py +++ b/discopop_library/discopop_optimizer/Variables/Experiment.py @@ -85,7 +85,7 @@ def __init__( self.__memory_region_sizes = get_sizes_of_memory_regions( set(), - os.path.join(discopop_output_path, "memory_regions.txt"), + os.path.join(discopop_output_path, "profiler/memory_regions.txt"), return_all_memory_regions=True, ) From 3dca9c648aea1e30984dbe21c66db4e93847ab23 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Mon, 16 Oct 2023 10:26:03 +0200 Subject: [PATCH 09/14] feat(explorer)[json-output]: enable by default --- discopop_explorer/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discopop_explorer/__main__.py b/discopop_explorer/__main__.py index 7bd7fe9fb..5a93e72ef 100644 --- a/discopop_explorer/__main__.py +++ b/discopop_explorer/__main__.py @@ -53,7 +53,7 @@ def parse_args() -> ExplorerArguments: ) # flags related to output and formatting: parser.add_argument( - "--json", type=str, nargs="?", default=None, const="explorer/patterns.json", + "--json", type=str, nargs="?", default="explorer/patterns.json", help="Json output") parser.add_argument( "--profiling", type=str, nargs="?", default=None, const="explorer/profiling_stats.txt", From 6e366f086554626df45ba23ff46cbad5e8aade7e Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Tue, 17 Oct 2023 15:59:26 +0200 Subject: [PATCH 10/14] feat(explorer)[detection-result-dump]: enable by default --- discopop_explorer/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discopop_explorer/__main__.py b/discopop_explorer/__main__.py index 5a93e72ef..c7a476aae 100644 --- a/discopop_explorer/__main__.py +++ b/discopop_explorer/__main__.py @@ -64,7 +64,7 @@ def parse_args() -> ExplorerArguments: help="Dump PET Graph to JSON file. If a path is given, the PET Graph is written to the given file, otherwise to pet_dump.json", ) parser.add_argument( - "--dump-detection-result", type=str, nargs="?", default=None, const="explorer/detection_result_dump.json", + "--dump-detection-result", type=str, nargs="?", default="explorer/detection_result_dump.json", help="Dump DetectionResult object to JSON file. If a path is given, the DetectionResult object is written to the given file, otherwise to detection_result_dump.json. Contents are equivalent to the json output. NOTE: This dump contains a dump of the PET Graph!", ) From 5e5b601116793de1cddfdae08327e4af01fd2c28 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Tue, 17 Oct 2023 16:13:44 +0200 Subject: [PATCH 11/14] fix(optimizer): corrected default arguments to reflect new folder structure --- discopop_library/discopop_optimizer/__main__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/discopop_library/discopop_optimizer/__main__.py b/discopop_library/discopop_optimizer/__main__.py index 0c2c59061..0a736e2db 100644 --- a/discopop_library/discopop_optimizer/__main__.py +++ b/discopop_library/discopop_optimizer/__main__.py @@ -17,8 +17,8 @@ OPTIONAL ARGUMENTS: --project= Path to the directory that contains your makefile [default: .] - --file-mapping= Path to the FileMapping.txt. [default: FileMapping.txt] - --detection-result-dump= Path to the dumped detection result JSON. [default: detection_result_dump.json] + --file-mapping= Path to the FileMapping.txt. [default: common_data/FileMapping.txt] + --detection-result-dump= Path to the dumped detection result JSON. [default: explorer/detection_result_dump.json] --execute-created-models Compiles, executes and measures models already stored in the project folder. Does not start the optimization pipeline. Required: --executable-name @@ -29,8 +29,8 @@ --execution-append-measurements If set, measurement files from previous executions will be kept and new results will be appended. --clean-created-code Removes all stored code modifications. - --code-export-path= Directory where generated CodeStorageObjects are located. [default: .discopop_optimizer/code_exports] - --dp-output-path= Directory where output files of DiscoPoP are located. [default: .discopop] + --code-export-path= Directory where generated CodeStorageObjects are located. [default: optimizer/code_exports] + --dp-output-path= Directory where output files of DiscoPoP are located. [default: .] --executable-name= Name of the executable generate by your makefile. Must be specified if --execute-created-models is used! [default: ] --executable-arguments= From d0583a4edb573ff785be8764bdabc774b291f6da Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Wed, 18 Oct 2023 11:28:45 +0200 Subject: [PATCH 12/14] fix[todo]: moved FileMapping.txt to .discopop folder --- DiscoPoP/DiscoPoP.cpp | 6 ------ discopop_explorer/__main__.py | 2 +- .../discopop_optimizer/__main__.py | 2 +- docs/How_to_contribute.md | 5 ++--- example/execute_cmake_example.sh | 8 ++------ scripts/CC_wrapper.sh | 2 +- scripts/CXX_wrapper.sh | 2 +- scripts/MPI_CC_wrapper.sh | 18 +++++++++--------- scripts/MPI_CXX_wrapper.sh | 18 +++++++++--------- scripts/runDiscoPoP | 2 +- share/lib/DPUtils.cpp | 8 ++++---- 11 files changed, 31 insertions(+), 42 deletions(-) diff --git a/DiscoPoP/DiscoPoP.cpp b/DiscoPoP/DiscoPoP.cpp index 41d585d78..f3e63a8b0 100644 --- a/DiscoPoP/DiscoPoP.cpp +++ b/DiscoPoP/DiscoPoP.cpp @@ -123,12 +123,6 @@ bool DiscoPoP::doInitialization(Module &M) { mkdir(".discopop/profiler", 0777); } - // prepare common_data directory if not present - struct stat st3 = {0}; - if (stat(".discopop/common_data", &st3) == -1){ - mkdir(".discopop/common_data", 0777); - } - // CUGeneration { diff --git a/discopop_explorer/__main__.py b/discopop_explorer/__main__.py index c7a476aae..da61e6021 100644 --- a/discopop_explorer/__main__.py +++ b/discopop_explorer/__main__.py @@ -42,7 +42,7 @@ def parse_args() -> ExplorerArguments: help="Reduction variables file" ) parser.add_argument( - "--fmap", type=str, default="common_data/FileMapping.txt", + "--fmap", type=str, default="FileMapping.txt", help="File mapping") parser.add_argument( "--plugins", type=str, nargs="*", default=[], diff --git a/discopop_library/discopop_optimizer/__main__.py b/discopop_library/discopop_optimizer/__main__.py index 0a736e2db..08739ddc6 100644 --- a/discopop_library/discopop_optimizer/__main__.py +++ b/discopop_library/discopop_optimizer/__main__.py @@ -17,7 +17,7 @@ OPTIONAL ARGUMENTS: --project= Path to the directory that contains your makefile [default: .] - --file-mapping= Path to the FileMapping.txt. [default: common_data/FileMapping.txt] + --file-mapping= Path to the FileMapping.txt. [default: FileMapping.txt] --detection-result-dump= Path to the dumped detection result JSON. [default: explorer/detection_result_dump.json] --execute-created-models Compiles, executes and measures models already stored in the project folder. Does not start the optimization pipeline. diff --git a/docs/How_to_contribute.md b/docs/How_to_contribute.md index 80fdc27e4..93265905e 100644 --- a/docs/How_to_contribute.md +++ b/docs/How_to_contribute.md @@ -71,8 +71,7 @@ All tools developed as part of the DiscoPop Project make use of the following fo ``` - project root/ - .discopop/ - - common_data/ - - FileMapping.txt + - FileMapping.txt - / - tool_1 output files - private/ @@ -82,7 +81,7 @@ All tools developed as part of the DiscoPop Project make use of the following fo - ... ``` , where no data should be stored / created outside the `.discopop` folder in order to keep the users build directory as clean as possible. -Files which are read by different tools, e.g. the `FileMapping.txt`, shall be stored in the `.discopop/common_data` folder. +Files which are read by different tools, e.g. the `FileMapping.txt`, shall be stored in the `.discopop` folder. Each tool may create a folder. In case data from these files is required by another tool, think about how to encode the information in a easy-to-use and structured format, preferrably JSON. If possible, please do not rely on exporting to simple `.txt` files as parsing adds a potential point of failure. Output data for use by other tools should be stored in the folder of the creating tool. Intermediate files may be stored in a folder named `private` and shall not be used by other tools. Analysis tools, like pattern detection scripts etc. shall be structured in such a way, that a execution from within the `.discopop` folder is intended. diff --git a/example/execute_cmake_example.sh b/example/execute_cmake_example.sh index 453139efc..85c012779 100755 --- a/example/execute_cmake_example.sh +++ b/example/execute_cmake_example.sh @@ -16,18 +16,14 @@ DISCOPOP_BUILD_DIR=$(pwd)/../build rm -rf build mkdir build -# generate FileMapping.txt -${DISCOPOP_BUILD_DIR}/scripts/dp-fmap - # Use DiscoPoP's compiler wrappers to build the cmake example # CMAKE_wrapper.sh acts as a substitute for the `cmake` command cd build ${DISCOPOP_BUILD_DIR}/scripts/CMAKE_wrapper.sh .. -# Set the DP_FM_PATH environment variable -DP_FM_PATH=${EXAMPLE_DIR}/../FileMapping.txt make +make # execute the example ./cmake_example -# Now, profiling results (e.g. cmake_example_dep.txt (dynamically identified data dependencies)) are available +# Now, profiling results (e.g. dynamically identified data dependencies) are available # The created output can be used for pattern detection using the discopop_explorer. # Please refer to the Wiki for detailed information and instructions. diff --git a/scripts/CC_wrapper.sh b/scripts/CC_wrapper.sh index 511840b46..a5f344820 100755 --- a/scripts/CC_wrapper.sh +++ b/scripts/CC_wrapper.sh @@ -39,7 +39,7 @@ echo "DP_FM_PATH: ${DP_FM_PATH}" # exit 1 #fi -echo "${LLVM_CLANG} -g -c -O0 -S -emit-llvm -fno-discard-value-names ${@} -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} " +echo "${LLVM_CLANG} -g -c -O0 -S -emit-llvm -fno-discard-value-names ${@} -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP" #clang-11 -g -c -O0 -S -emit-llvm -fno-discard-value-names "$@" -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} ${LLVM_CLANG} "$@" -g -O0 -fno-discard-value-names -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -fPIC -Xlinker -L${DP_BUILD}/rtlib -Xlinker -lDiscoPoP_RT -Xlinker -lpthread -Xlinker -v -Xlinker -lstdc++ diff --git a/scripts/CXX_wrapper.sh b/scripts/CXX_wrapper.sh index 70ac6eac2..a31cbdbb5 100755 --- a/scripts/CXX_wrapper.sh +++ b/scripts/CXX_wrapper.sh @@ -39,7 +39,7 @@ echo "DP_FM_PATH: ${DP_FM_PATH}" # exit 1 #fi -echo "${LLVM_CLANGPP} -g -c -O0 -S -emit-llvm -fno-discard-value-names ${@} -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} " +echo "${LLVM_CLANGPP} -g -c -O0 -S -emit-llvm -fno-discard-value-names ${@} -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP" #clang++-11 -g -c -O0 -S -emit-llvm -fno-discard-value-names "$@" -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} ${LLVM_CLANGPP} "$@" -g -O0 -fno-discard-value-names -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -fPIC -Xlinker -L${DP_BUILD}/rtlib -Xlinker -lDiscoPoP_RT -Xlinker -lpthread -Xlinker -v # WARNING: OUTPUT IS A .ll FILE, ENDING IS .o diff --git a/scripts/MPI_CC_wrapper.sh b/scripts/MPI_CC_wrapper.sh index f72a14296..1de5a7cb8 100755 --- a/scripts/MPI_CC_wrapper.sh +++ b/scripts/MPI_CC_wrapper.sh @@ -39,15 +39,15 @@ echo "WRAPPED MPI CC COMPILE..." echo "ARGS: ${@}" echo "DP_FM_PATH: ${DP_FM_PATH}" -# check if environment is prepared -if [ -z ${DP_FM_PATH} ]; then - echo "ERROR: DP_FM_PATH unspecified!" - echo " Generate a FileMapping.txt file and create an environment Variable which points to this file." - echo " Please refer to https://discopop-project.github.io/discopop/Profiling/File_Mapping/ for further information." - exit 1 -fi - -echo "${LLVM_CLANG} ${MPI_INCLUDES} -g -c -S -emit-llvm -fno-discard-value-names ${@} -O0 -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH}" +## check if environment is prepared +#if [ -z ${DP_FM_PATH} ]; then +# echo "ERROR: DP_FM_PATH unspecified!" +# echo " Generate a FileMapping.txt file and create an environment Variable which points to this file." +# echo " Please refer to https://discopop-project.github.io/discopop/Profiling/File_Mapping/ for further information." +# exit 1 +#fi + +echo "${LLVM_CLANG} ${MPI_INCLUDES} -g -c -S -emit-llvm -fno-discard-value-names ${@} -O0 -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP" #clang-11 -g -c -O0 -S -emit-llvm -fno-discard-value-names "$@" -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} ${LLVM_CLANG} "${@}" -g -c -O0 -fno-discard-value-names -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -fPIC ${MPI_INCLUDES} -Xlinker -lstdc++ diff --git a/scripts/MPI_CXX_wrapper.sh b/scripts/MPI_CXX_wrapper.sh index b053454bc..c5ef65899 100755 --- a/scripts/MPI_CXX_wrapper.sh +++ b/scripts/MPI_CXX_wrapper.sh @@ -39,15 +39,15 @@ echo "WRAPPED MPI CXX COMPILE..." echo "ARGS: ${@}" echo "DP_FM_PATH: ${DP_FM_PATH}" -# check if environment is prepared -if [ -z ${DP_FM_PATH} ]; then - echo "ERROR: DP_FM_PATH unspecified!" - echo " Generate a FileMapping.txt file and create an environment Variable which points to this file." - echo " Please refer to https://discopop-project.github.io/discopop/Profiling/File_Mapping/ for further information." - exit 1 -fi - -echo "${LLVM_CLANGPP} ${MPI_INCLUDES} -g -c -O0 -S -emit-llvm -fno-discard-value-names ${@} -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH}" +## check if environment is prepared +#if [ -z ${DP_FM_PATH} ]; then +# echo "ERROR: DP_FM_PATH unspecified!" +# echo " Generate a FileMapping.txt file and create an environment Variable which points to this file." +# echo " Please refer to https://discopop-project.github.io/discopop/Profiling/File_Mapping/ for further information." +# exit 1 +#fi + +echo "${LLVM_CLANGPP} ${MPI_INCLUDES} -g -c -O0 -S -emit-llvm -fno-discard-value-names ${@} -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP" #clang++-11 -g -c -O0 -S -emit-llvm -fno-discard-value-names "$@" -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -mllvm --fm-path -mllvm ${DP_FM_PATH} ${LLVM_CLANGPP} "$@" -g -c -O0 -fno-discard-value-names -Xclang -load -Xclang ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP -fPIC ${MPI_INCLUDES} diff --git a/scripts/runDiscoPoP b/scripts/runDiscoPoP index f092ea6ea..b147b5a4e 100755 --- a/scripts/runDiscoPoP +++ b/scripts/runDiscoPoP @@ -372,7 +372,7 @@ fi # Run DiscoPoP Pass log -i "Running DiscoPoP Pass to analyze and instrument your application..." -$LLVM_OPT -S -load ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP ${EXECUTABLE_NAME}.ll -o ${EXECUTABLE_NAME}_dp.ll -fm-path FileMapping.txt ${DP_PASS_FLAGS} +$LLVM_OPT -S -load ${DP_BUILD}/libi/LLVMDiscoPoP.so -DiscoPoP ${EXECUTABLE_NAME}.ll -o ${EXECUTABLE_NAME}_dp.ll ${DP_PASS_FLAGS} log -i "Creating Executable..." $LLVM_LLC -filetype=obj ${EXECUTABLE_NAME}_dp.ll -o ${EXECUTABLE_NAME}_dp.o diff --git a/share/lib/DPUtils.cpp b/share/lib/DPUtils.cpp index 65ebcc4c7..d52b2924b 100644 --- a/share/lib/DPUtils.cpp +++ b/share/lib/DPUtils.cpp @@ -16,7 +16,7 @@ using namespace std; //cl::opt FileMappingPath("fm-path", cl::init(""), // cl::desc("Specify file mapping location"), cl::Hidden); -string FileMappingPath = ".discopop/common_data/FileMapping.txt"; +string FileMappingPath = ".discopop/FileMapping.txt"; cl::opt DP_MEMORY_PROFILING_SKIP_FUNCTION_ARGUMENTS("memory-profiling-skip-function-arguments", cl::init(false), @@ -28,7 +28,7 @@ namespace dputil { int tempfid = 1; fstream fileMappingFile; - fileMappingFile.open(".discopop/common_data/FileMapping.txt", ios::in); + fileMappingFile.open(".discopop/FileMapping.txt", ios::in); if (fileMappingFile) { string tp; @@ -43,7 +43,7 @@ namespace dputil { } fileMappingFile.close(); - fileMappingFile.open(".discopop/common_data/FileMapping.txt", std::ios_base::app); + fileMappingFile.open(".discopop/FileMapping.txt", std::ios_base::app); fileMappingFile << tempfid << "\t" << fullPathName << "\n"; fileMappingFile.close(); errs() << "added fmap entry: " << tempfid << "\t" << fullPathName << "\n"; @@ -51,7 +51,7 @@ namespace dputil { } else { - fileMappingFile.open(".discopop/common_data/FileMapping.txt", std::ios_base::app); + fileMappingFile.open(".discopop/FileMapping.txt", std::ios_base::app); fileMappingFile << tempfid << "\t" << fullPathName << "\n"; fileMappingFile.close(); errs() << "added fmap entry: " << tempfid << "\t" << fullPathName << "\n"; From 4e877506f68a928188c3575a7dee5ff48347a162 Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Wed, 18 Oct 2023 11:41:03 +0200 Subject: [PATCH 13/14] fix(unit-test): update to use new folder structure --- test/do_all/daxpy/src/Makefile | 4 +++- test/do_all/daxpy/test.py | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/test/do_all/daxpy/src/Makefile b/test/do_all/daxpy/src/Makefile index 7a1b8871e..14d72f730 100644 --- a/test/do_all/daxpy/src/Makefile +++ b/test/do_all/daxpy/src/Makefile @@ -1,12 +1,14 @@ all: clean prog prog: daxpy.o - $(LINKER) -o prog daxpy.o $(CXXFLAGS) + $(CXX) -o prog daxpy.o $(CXXFLAGS) daxpy.o: $(CXX) -c -o daxpy.o daxpy.cpp $(CXXFLAGS) clean: + rm -rf .discopop + rm -rf src/.discopop find . -not -name daxpy.cpp -not -name Makefile -not -path **/FileMapping.txt -delete veryclean: clean diff --git a/test/do_all/daxpy/test.py b/test/do_all/daxpy/test.py index 03bdc093d..0d25637d3 100644 --- a/test/do_all/daxpy/test.py +++ b/test/do_all/daxpy/test.py @@ -21,16 +21,19 @@ def test(self): os.system("" + os.path.join(build_dir, "scripts", "dp-fmap")) # build - make_command = "DP_FM_PATH=" + os.path.join(src_dir, "FileMapping.txt") + " " + # make_command = "DP_FM_PATH=" + os.path.join(src_dir, "FileMapping.txt") + " " + make_command = "" make_command += "CC=" + os.path.join(build_dir, "scripts", "CC_wrapper.sh") + " " make_command += "CXX=" + os.path.join(build_dir, "scripts", "CXX_wrapper.sh") + " " - make_command += "LINKER=" + os.path.join(build_dir, "scripts", "LINKER_wrapper.sh") + " " + # make_command += "LINKER=" + os.path.join(build_dir, "scripts", "LINKER_wrapper.sh") + " " make_command += "make " os.system(make_command) # execute instrumented program os.system("./prog") # execute DiscoPoP analysis - os.system("discopop_explorer --dep-file=prog_dep.txt --dump-detection-result") + os.chdir(".discopop") + os.system("discopop_explorer") + os.chdir("..") # validate results self.validate_results(current_dir, src_dir) # clean environment @@ -39,7 +42,7 @@ def test(self): def validate_results(self, test_dir, src_dir): """compare results to gold standard""" gold_standard_file = os.path.join(test_dir, "detection_result_dump.json") - test_output_file = os.path.join(src_dir, "detection_result_dump.json") + test_output_file = os.path.join(src_dir, ".discopop", "explorer", "detection_result_dump.json") # load both detection results with open(gold_standard_file, "r") as f: tmp_str = f.read() From b42391a8dcb506ccd5327ea80a7a12b168161b1b Mon Sep 17 00:00:00 2001 From: Lukas Rothenberger Date: Wed, 18 Oct 2023 11:46:31 +0200 Subject: [PATCH 14/14] fix(ci-test): updated to use new folder structure --- .github/workflows/tests/profiler.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests/profiler.sh b/.github/workflows/tests/profiler.sh index 1faf054a1..ec3f769d3 100755 --- a/.github/workflows/tests/profiler.sh +++ b/.github/workflows/tests/profiler.sh @@ -23,7 +23,7 @@ function test_discopopPass { cp ${DISCOPOP_SRC}/scripts/dp-fmap . ./dp-fmap clang++ -g -c -O0 -S -emit-llvm -fno-discard-value-names "$1" -o out.ll || return 1 - opt-11 -S -load=${DISCOPOP_INSTALL}/libi/LLVMDiscoPoP.so --DiscoPoP out.ll -o out_dp.ll --fm-path FileMapping.txt || return 1 + opt-11 -S -load=${DISCOPOP_INSTALL}/libi/LLVMDiscoPoP.so --DiscoPoP out.ll -o out_dp.ll || return 1 clang++ out_dp.ll -o out_prof -L${DISCOPOP_INSTALL}/rtlib -lDiscoPoP_RT -lpthread || return 1 ./out_prof || return 1 }