From 924a0db435b83243e8db22a5ca8b918f820f17ae Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Thu, 7 Sep 2023 12:46:35 +0200 Subject: [PATCH 01/12] * First skeleton for HyTeg project --- varats/varats/projects/cpp_projects/hyteg.py | 58 ++++++++++++++++++++ varats/varats/tools/bb_config.py | 1 + 2 files changed, 59 insertions(+) create mode 100644 varats/varats/projects/cpp_projects/hyteg.py diff --git a/varats/varats/projects/cpp_projects/hyteg.py b/varats/varats/projects/cpp_projects/hyteg.py new file mode 100644 index 000000000..9b104e07e --- /dev/null +++ b/varats/varats/projects/cpp_projects/hyteg.py @@ -0,0 +1,58 @@ +import typing as tp + +import benchbuild as bb +from benchbuild.utils.cmd import make, cmake, mkdir +from benchbuild.utils.settings import get_number_of_jobs +from plumbum import local + +from varats.project.project_domain import ProjectDomains +from varats.project.project_util import get_local_project_git_path +from varats.project.sources import FeatureSource +from varats.project.varats_project import VProject +from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.settings import bb_cfg + + +class HyTeg(VProject): + NAME = 'HyTeg' + GROUP = 'cpp_projects' + DOMAIN = ProjectDomains.CPP_LIBRARY + + SOURCE = [ + bb.source.Git( + remote="git@github.com:se-sic/hyteg-VaRA.git", + local="HyTeg", + refspec="origin/HEAD", + limit=None, + shallow=False + ), + FeatureSource() + ] + + WORKLOADS = {} + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash + ) -> tp.List['ProjectBinaryWrapper']: + binaries = RevisionBinaryMap(get_local_project_git_path(HyTeg.NAME)) + + return binaries + + def compile(self) -> None: + hyteg_source = local.path(self.source_of(self.primary_source)) + + mkdir("-p", hyteg_source / "build") + + cc_compiler = bb.compiler.cc(self) + cxx_compiler = bb.compiler.cxx(self) + + with local.cwd(hyteg_source / "build"): + with local.env(CC=str(cc_compiler), CXX=str(cxx_compiler)): + bb.watch(cmake)("..") + + with local.cwd(hyteg_source / "build" / "apps"): + bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) + + def run_tests(self) -> None: + pass diff --git a/varats/varats/tools/bb_config.py b/varats/varats/tools/bb_config.py index 5d3b73bd2..8d374709e 100644 --- a/varats/varats/tools/bb_config.py +++ b/varats/varats/tools/bb_config.py @@ -75,6 +75,7 @@ def update_projects( 'varats.projects.cpp_projects.z3', 'varats.projects.cpp_projects.ect', 'varats.projects.cpp_projects.lepton' + 'varats.projects.cpp_projects.hyteg' ] projects_conf.value[:] += [ 'varats.projects.cpp_projects.doxygen', 'varats.projects.cpp_projects' From 33824501d016d889b7d396a34c6b5782bf215522 Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Fri, 15 Sep 2023 14:43:35 +0200 Subject: [PATCH 02/12] * Added Profiling Binary to HyTeg --- varats/varats/projects/cpp_projects/hyteg.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/varats/varats/projects/cpp_projects/hyteg.py b/varats/varats/projects/cpp_projects/hyteg.py index 9b104e07e..902ef6744 100644 --- a/varats/varats/projects/cpp_projects/hyteg.py +++ b/varats/varats/projects/cpp_projects/hyteg.py @@ -2,11 +2,12 @@ import benchbuild as bb from benchbuild.utils.cmd import make, cmake, mkdir +from benchbuild.utils.revision_ranges import SingleRevision from benchbuild.utils.settings import get_number_of_jobs from plumbum import local from varats.project.project_domain import ProjectDomains -from varats.project.project_util import get_local_project_git_path +from varats.project.project_util import get_local_project_git_path, BinaryType from varats.project.sources import FeatureSource from varats.project.varats_project import VProject from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap @@ -37,7 +38,15 @@ def binaries_for_revision( ) -> tp.List['ProjectBinaryWrapper']: binaries = RevisionBinaryMap(get_local_project_git_path(HyTeg.NAME)) - return binaries + binaries.specify_binary( + "build/apps/profiling/ProfilingApp", + BinaryType.EXECUTABLE, + only_valid_in=SingleRevision( + "f4711dadc3f61386e6ccdc704baa783253332db2" + ) + ) + + return binaries[revision] def compile(self) -> None: hyteg_source = local.path(self.source_of(self.primary_source)) @@ -54,5 +63,8 @@ def compile(self) -> None: with local.cwd(hyteg_source / "build" / "apps"): bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) + def recompile(self) -> None: + pass + def run_tests(self) -> None: pass From 3ea74fea20743df8fe6861019b64508d2e4a8f39 Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Tue, 26 Sep 2023 13:00:59 +0200 Subject: [PATCH 03/12] * Added cflags to disable phasar for hyteg --- varats/varats/projects/cpp_projects/hyteg.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/varats/varats/projects/cpp_projects/hyteg.py b/varats/varats/projects/cpp_projects/hyteg.py index 902ef6744..a5b053748 100644 --- a/varats/varats/projects/cpp_projects/hyteg.py +++ b/varats/varats/projects/cpp_projects/hyteg.py @@ -6,6 +6,7 @@ from benchbuild.utils.settings import get_number_of_jobs from plumbum import local +from varats.paper.paper_config import PaperConfigSpecificGit from varats.project.project_domain import ProjectDomains from varats.project.project_util import get_local_project_git_path, BinaryType from varats.project.sources import FeatureSource @@ -20,7 +21,8 @@ class HyTeg(VProject): DOMAIN = ProjectDomains.CPP_LIBRARY SOURCE = [ - bb.source.Git( + PaperConfigSpecificGit( + project_name="HyTeg", remote="git@github.com:se-sic/hyteg-VaRA.git", local="HyTeg", refspec="origin/HEAD", @@ -53,6 +55,10 @@ def compile(self) -> None: mkdir("-p", hyteg_source / "build") + # Currently Phasar passes crash the compiler + # This limits us to analysing compile time variability + self.cflags += ["-mllvm", "--vara-disable-phasar"] + cc_compiler = bb.compiler.cc(self) cxx_compiler = bb.compiler.cxx(self) From 52fbf5673eb78f217112ee87cbeffd7a9064929a Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Wed, 4 Oct 2023 09:05:01 +0200 Subject: [PATCH 04/12] * Disabled MPI build for HyTeG/Walberla --- varats/varats/projects/cpp_projects/hyteg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varats/varats/projects/cpp_projects/hyteg.py b/varats/varats/projects/cpp_projects/hyteg.py index a5b053748..f56aa834d 100644 --- a/varats/varats/projects/cpp_projects/hyteg.py +++ b/varats/varats/projects/cpp_projects/hyteg.py @@ -64,7 +64,7 @@ def compile(self) -> None: with local.cwd(hyteg_source / "build"): with local.env(CC=str(cc_compiler), CXX=str(cxx_compiler)): - bb.watch(cmake)("..") + bb.watch(cmake)("..", "-DWALBERLA_BUILD_WITH_MPI=OFF") with local.cwd(hyteg_source / "build" / "apps"): bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) From 85c986c96789af164f28c4e8fe0f7837e268b687 Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Wed, 4 Oct 2023 10:42:54 +0200 Subject: [PATCH 05/12] * Disabled Doc build for HyTeG --- varats/varats/projects/cpp_projects/hyteg.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/varats/varats/projects/cpp_projects/hyteg.py b/varats/varats/projects/cpp_projects/hyteg.py index f56aa834d..cfde582cd 100644 --- a/varats/varats/projects/cpp_projects/hyteg.py +++ b/varats/varats/projects/cpp_projects/hyteg.py @@ -64,7 +64,10 @@ def compile(self) -> None: with local.cwd(hyteg_source / "build"): with local.env(CC=str(cc_compiler), CXX=str(cxx_compiler)): - bb.watch(cmake)("..", "-DWALBERLA_BUILD_WITH_MPI=OFF") + bb.watch(cmake)( + "..", "-DWALBERLA_BUILD_WITH_MPI=OFF", + "-DHYTEG_BUILD_DOC=OFF" + ) with local.cwd(hyteg_source / "build" / "apps"): bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) From ef967efc0d9aea63a8168433f19bc5e682f0c14c Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Fri, 6 Oct 2023 08:24:37 +0200 Subject: [PATCH 06/12] * Add HyTeG Workload --- varats/varats/projects/cpp_projects/hyteg.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/varats/varats/projects/cpp_projects/hyteg.py b/varats/varats/projects/cpp_projects/hyteg.py index cfde582cd..fcd08994b 100644 --- a/varats/varats/projects/cpp_projects/hyteg.py +++ b/varats/varats/projects/cpp_projects/hyteg.py @@ -1,9 +1,11 @@ import typing as tp import benchbuild as bb +from benchbuild.command import WorkloadSet, Command, SourceRoot from benchbuild.utils.cmd import make, cmake, mkdir from benchbuild.utils.revision_ranges import SingleRevision from benchbuild.utils.settings import get_number_of_jobs +from experiment.workload_util import WorkloadCategory, RSBinary from plumbum import local from varats.paper.paper_config import PaperConfigSpecificGit @@ -18,7 +20,7 @@ class HyTeg(VProject): NAME = 'HyTeg' GROUP = 'cpp_projects' - DOMAIN = ProjectDomains.CPP_LIBRARY + DOMAIN = ProjectDomains.TEST SOURCE = [ PaperConfigSpecificGit( @@ -32,7 +34,15 @@ class HyTeg(VProject): FeatureSource() ] - WORKLOADS = {} + WORKLOADS = { + WorkloadSet(WorkloadCategory.EXAMPLE): [ + Command( + SourceRoot("HyTeG/build/apps/profiling") / + RSBinary('ProfilingApp'), + label='ProfilingApp' + ) + ] + } @staticmethod def binaries_for_revision( @@ -69,7 +79,7 @@ def compile(self) -> None: "-DHYTEG_BUILD_DOC=OFF" ) - with local.cwd(hyteg_source / "build" / "apps"): + with local.cwd(hyteg_source / "build" / "apps" / "profiling"): bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) def recompile(self) -> None: From 932b02b3e51f157c3cf7db0ae3b7f1d834ecb48f Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Wed, 11 Oct 2023 14:34:36 +0200 Subject: [PATCH 07/12] - Added suggestions from code review --- varats/varats/projects/cpp_projects/hyteg.py | 25 +++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/varats/varats/projects/cpp_projects/hyteg.py b/varats/varats/projects/cpp_projects/hyteg.py index fcd08994b..985e7643e 100644 --- a/varats/varats/projects/cpp_projects/hyteg.py +++ b/varats/varats/projects/cpp_projects/hyteg.py @@ -18,6 +18,20 @@ class HyTeg(VProject): + """ + C++ framework for large scale high performance finite element simulations + based on (but not limited to) matrix-free geometric multigrid. + + Note: + Currently HyTeg CANNOT be compiled with the Phasar passes activated + in vara. + Trying to do so will crash the compiler + + If you use Dune with an experiment that uses the vara compiler, + add `-mllvm --vara-disable-phasar` to the projects `cflags` to + disable phasar passes. + This will still allow to analyse compile-time variability. + """ NAME = 'HyTeg' GROUP = 'cpp_projects' DOMAIN = ProjectDomains.TEST @@ -37,7 +51,7 @@ class HyTeg(VProject): WORKLOADS = { WorkloadSet(WorkloadCategory.EXAMPLE): [ Command( - SourceRoot("HyTeG/build/apps/profiling") / + SourceRoot("HyTeG") / "build" / "apps" / "profiling" / RSBinary('ProfilingApp'), label='ProfilingApp' ) @@ -65,10 +79,6 @@ def compile(self) -> None: mkdir("-p", hyteg_source / "build") - # Currently Phasar passes crash the compiler - # This limits us to analysing compile time variability - self.cflags += ["-mllvm", "--vara-disable-phasar"] - cc_compiler = bb.compiler.cc(self) cxx_compiler = bb.compiler.cxx(self) @@ -83,7 +93,10 @@ def compile(self) -> None: bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) def recompile(self) -> None: - pass + hyteg_source = local.path(self.source_of(self.primary_source)) + + with local.cwd(hyteg_source / "build" / "apps" / "profiling"): + bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) def run_tests(self) -> None: pass From 9c98e59f5e9ded73342878e0bbd9d6e6597fa48f Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Wed, 11 Oct 2023 14:37:24 +0200 Subject: [PATCH 08/12] - Added suggestions from code review --- varats-core/varats/project/project_domain.py | 1 + varats/varats/projects/cpp_projects/hyteg.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/varats-core/varats/project/project_domain.py b/varats-core/varats/project/project_domain.py index d7108ecaf..071ed9605 100644 --- a/varats-core/varats/project/project_domain.py +++ b/varats-core/varats/project/project_domain.py @@ -18,6 +18,7 @@ class ProjectDomains(Enum): EDITOR = "Editor" FILE_FORMAT = "File format" HW_EMULATOR = "Hardware emulator" + HPC = "High Performance Applications" PARSER = "Parser" PLANNING = "Planning" PROG_LANG = "Programming language" diff --git a/varats/varats/projects/cpp_projects/hyteg.py b/varats/varats/projects/cpp_projects/hyteg.py index 985e7643e..9e7b9a7ae 100644 --- a/varats/varats/projects/cpp_projects/hyteg.py +++ b/varats/varats/projects/cpp_projects/hyteg.py @@ -5,9 +5,9 @@ from benchbuild.utils.cmd import make, cmake, mkdir from benchbuild.utils.revision_ranges import SingleRevision from benchbuild.utils.settings import get_number_of_jobs -from experiment.workload_util import WorkloadCategory, RSBinary from plumbum import local +from varats.experiment.workload_util import WorkloadCategory, RSBinary from varats.paper.paper_config import PaperConfigSpecificGit from varats.project.project_domain import ProjectDomains from varats.project.project_util import get_local_project_git_path, BinaryType @@ -34,7 +34,7 @@ class HyTeg(VProject): """ NAME = 'HyTeg' GROUP = 'cpp_projects' - DOMAIN = ProjectDomains.TEST + DOMAIN = ProjectDomains.HPC SOURCE = [ PaperConfigSpecificGit( From ebd6767f011913855780a86a73c27c1dfa035b03 Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Wed, 11 Oct 2023 16:49:18 +0200 Subject: [PATCH 09/12] - Pylint and MyPy --- varats/varats/projects/cpp_projects/hyteg.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/varats/varats/projects/cpp_projects/hyteg.py b/varats/varats/projects/cpp_projects/hyteg.py index 9e7b9a7ae..3ea9e323a 100644 --- a/varats/varats/projects/cpp_projects/hyteg.py +++ b/varats/varats/projects/cpp_projects/hyteg.py @@ -1,3 +1,4 @@ +"""Adds the HyTeg framework as a project to VaRA-TS.""" import typing as tp import benchbuild as bb @@ -10,7 +11,11 @@ from varats.experiment.workload_util import WorkloadCategory, RSBinary from varats.paper.paper_config import PaperConfigSpecificGit from varats.project.project_domain import ProjectDomains -from varats.project.project_util import get_local_project_git_path, BinaryType +from varats.project.project_util import ( + get_local_project_git_path, + BinaryType, + ProjectBinaryWrapper, +) from varats.project.sources import FeatureSource from varats.project.varats_project import VProject from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap @@ -75,6 +80,7 @@ def binaries_for_revision( return binaries[revision] def compile(self) -> None: + """Compile HyTeg with irrelevant settings disabled.""" hyteg_source = local.path(self.source_of(self.primary_source)) mkdir("-p", hyteg_source / "build") @@ -93,6 +99,7 @@ def compile(self) -> None: bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) def recompile(self) -> None: + """Recompiles HyTeg e.g. after a patch has been applied.""" hyteg_source = local.path(self.source_of(self.primary_source)) with local.cwd(hyteg_source / "build" / "apps" / "profiling"): From f569a36a57e3f7cafb61f304151c82ca70ea8e34 Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Tue, 17 Oct 2023 19:58:13 +0200 Subject: [PATCH 10/12] Hyteg fixes --- varats/varats/projects/cpp_projects/hyteg.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/varats/varats/projects/cpp_projects/hyteg.py b/varats/varats/projects/cpp_projects/hyteg.py index 3ea9e323a..20f1de139 100644 --- a/varats/varats/projects/cpp_projects/hyteg.py +++ b/varats/varats/projects/cpp_projects/hyteg.py @@ -17,6 +17,7 @@ ProjectBinaryWrapper, ) from varats.project.sources import FeatureSource +from varats.project.varats_command import VCommand from varats.project.varats_project import VProject from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap from varats.utils.settings import bb_cfg @@ -55,8 +56,8 @@ class HyTeg(VProject): WORKLOADS = { WorkloadSet(WorkloadCategory.EXAMPLE): [ - Command( - SourceRoot("HyTeG") / "build" / "apps" / "profiling" / + VCommand( + SourceRoot("HyTeg") / "build" / "apps" / "profiling" / RSBinary('ProfilingApp'), label='ProfilingApp' ) @@ -70,7 +71,7 @@ def binaries_for_revision( binaries = RevisionBinaryMap(get_local_project_git_path(HyTeg.NAME)) binaries.specify_binary( - "build/apps/profiling/ProfilingApp", + "ProfilingApp", BinaryType.EXECUTABLE, only_valid_in=SingleRevision( "f4711dadc3f61386e6ccdc704baa783253332db2" From b80140585b5e3f1f0f907e73dad2b77954195fdf Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Mon, 23 Oct 2023 17:57:52 +0200 Subject: [PATCH 11/12] Update varats/varats/projects/cpp_projects/hyteg.py --- varats/varats/projects/cpp_projects/hyteg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varats/varats/projects/cpp_projects/hyteg.py b/varats/varats/projects/cpp_projects/hyteg.py index 20f1de139..3e8311b30 100644 --- a/varats/varats/projects/cpp_projects/hyteg.py +++ b/varats/varats/projects/cpp_projects/hyteg.py @@ -45,7 +45,7 @@ class HyTeg(VProject): SOURCE = [ PaperConfigSpecificGit( project_name="HyTeg", - remote="git@github.com:se-sic/hyteg-VaRA.git", + remote="https://github.com/se-sic/hyteg-VaRA.git", local="HyTeg", refspec="origin/HEAD", limit=None, From b26ee1c9dae68607a43a0b023a1afa9623b60290 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Mon, 23 Oct 2023 20:43:31 +0200 Subject: [PATCH 12/12] Apply suggestions from code review --- varats/varats/projects/cpp_projects/hyteg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varats/varats/projects/cpp_projects/hyteg.py b/varats/varats/projects/cpp_projects/hyteg.py index 3e8311b30..5de8f13b4 100644 --- a/varats/varats/projects/cpp_projects/hyteg.py +++ b/varats/varats/projects/cpp_projects/hyteg.py @@ -2,7 +2,7 @@ import typing as tp import benchbuild as bb -from benchbuild.command import WorkloadSet, Command, SourceRoot +from benchbuild.command import WorkloadSet, SourceRoot from benchbuild.utils.cmd import make, cmake, mkdir from benchbuild.utils.revision_ranges import SingleRevision from benchbuild.utils.settings import get_number_of_jobs