From ea91e581ab20f3e0a9cde080ff17140996d6e564 Mon Sep 17 00:00:00 2001
From: Freifrau von Bleifrei <freifrauvonbleifrei@users.noreply.github.com>
Date: Mon, 18 Sep 2023 16:44:48 +0200
Subject: [PATCH] Subspace vote for more groups (#106)

* have a single (but hard-coded) place to change the maximum
number of groups for subspace and outgroup reduce,
as determined by the length of the datatype used
---
 src/sparsegrid/AnyDistributedSparseGrid.cpp | 30 +++++++++++----------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/src/sparsegrid/AnyDistributedSparseGrid.cpp b/src/sparsegrid/AnyDistributedSparseGrid.cpp
index 3781064b..a0eaf1c6 100644
--- a/src/sparsegrid/AnyDistributedSparseGrid.cpp
+++ b/src/sparsegrid/AnyDistributedSparseGrid.cpp
@@ -116,21 +116,23 @@ void AnyDistributedSparseGrid::setDataSize(SubspaceIndexType i, SubspaceSizeType
   subspacesDataSizes_[i] = newSize;
 }
 
-std::vector<boost::multiprecision::uint256_t> getSubspaceVote(
+using UIntForGroupReductionType = boost::multiprecision::uint256_t;
+
+std::vector<UIntForGroupReductionType> getSubspaceVote(
     CommunicatorType comm, RankType rankInComm,
     const std::vector<SubspaceSizeType>& subspacesDataSizes) {
-  RankType maxNumRanks = std::numeric_limits<boost::multiprecision::uint256_t>::digits;
+  RankType maxNumRanks = std::numeric_limits<UIntForGroupReductionType>::digits;
   if (rankInComm >= maxNumRanks) {
     throw std::runtime_error("Rank in communicator is too large; try fewer process groups? " +
                              std::to_string(maxNumRanks) + " allowed");
     // or implement things with bitsets or a veeery long string altogether
   }
 
-  boost::multiprecision::uint256_t mySummand = static_cast<boost::multiprecision::uint256_t>(1)
+  UIntForGroupReductionType mySummand = static_cast<UIntForGroupReductionType>(1)
                                                << rankInComm;
 
   // allocate vector of long long
-  std::vector<boost::multiprecision::uint256_t> subspaceVote(subspacesDataSizes.size(), 0);
+  std::vector<UIntForGroupReductionType> subspaceVote(subspacesDataSizes.size(), 0);
   // set to mySummand if we have data in this subspace
   for (AnyDistributedSparseGrid::SubspaceIndexType i = 0; i < subspacesDataSizes.size(); ++i) {
     if (subspacesDataSizes[i] > 0) {
@@ -140,7 +142,7 @@ std::vector<boost::multiprecision::uint256_t> getSubspaceVote(
 
   // vote by binary or
   MPI_Allreduce(MPI_IN_PLACE, subspaceVote.data(),
-                static_cast<int>(subspaceVote.size()) * sizeof(boost::multiprecision::uint256_t),
+                static_cast<int>(subspaceVote.size()) * sizeof(UIntForGroupReductionType),
                 MPI_CHAR, MPI_BOR, comm);
   return subspaceVote;
 }
@@ -151,13 +153,13 @@ void AnyDistributedSparseGrid::setOutgroupCommunicator(CommunicatorType comm, Ra
   // free previous communicator, if any
   clearSubspaceCommunicators();
 
-  RankType maxNumRanks = std::numeric_limits<boost::multiprecision::uint256_t>::digits;
+  RankType maxNumRanks = std::numeric_limits<UIntForGroupReductionType>::digits;
 
-  std::vector<boost::multiprecision::uint256_t> subspaceVote =
+  std::vector<UIntForGroupReductionType> subspaceVote =
       getSubspaceVote(comm, rankInComm, subspacesDataSizes_);
 
   // use this data to get the subspaces by the reduced values
-  boost::multiprecision::uint256_t manyGroups = 0;
+  UIntForGroupReductionType manyGroups = 0;
   std::vector<SubspaceIndexType> subspacesForMany;
   for (SubspaceIndexType i = 0; i < this->getNumSubspaces(); ++i) {
     // only add if the vote has > 1 bit set (i.e. more than one rank has data in this subspace)
@@ -171,7 +173,7 @@ void AnyDistributedSparseGrid::setOutgroupCommunicator(CommunicatorType comm, Ra
   std::vector<RankType> ranks;
   // find all ranks that have voted
   for (RankType r = 0; r < maxNumRanks; ++r) {
-    if (manyGroups & (static_cast<boost::multiprecision::uint256_t>(1) << r)) {
+    if (manyGroups & (static_cast<UIntForGroupReductionType>(1) << r)) {
       ranks.push_back(r);
     }
   }
@@ -216,13 +218,13 @@ void AnyDistributedSparseGrid::setSubspaceCommunicators(CommunicatorType comm,
                                                         RankType rankInComm) {
   clearSubspaceCommunicators();
   assert(this->getNumSubspaces() > 0);
-  RankType maxNumRanks = std::numeric_limits<boost::multiprecision::uint256_t>::digits;
+  RankType maxNumRanks = std::numeric_limits<UIntForGroupReductionType>::digits;
 
-  std::vector<boost::multiprecision::uint256_t> subspaceVote =
+  std::vector<UIntForGroupReductionType> subspaceVote =
       getSubspaceVote(comm, rankInComm, subspacesDataSizes_);
 
   // use this data to get the subspaces by the reduced values
-  std::map<boost::multiprecision::uint256_t, std::vector<SubspaceIndexType>> subspacesByVote;
+  std::map<UIntForGroupReductionType, std::vector<SubspaceIndexType>> subspacesByVote;
   for (SubspaceIndexType i = 0; i < this->getNumSubspaces(); ++i) {
     // only add if the vote has > 1 bit set (i.e. more than one rank has data in this subspace)
     if (subspaceVote[i] != 0 && (subspaceVote[i] & (subspaceVote[i] - 1)) != 0) {
@@ -236,11 +238,11 @@ void AnyDistributedSparseGrid::setSubspaceCommunicators(CommunicatorType comm,
   for (const auto& kv : subspacesByVote) {
     // get a new group communicator that includes all that have voted
     // for this subspace
-    boost::multiprecision::uint256_t vote = kv.first;
+    UIntForGroupReductionType vote = kv.first;
     std::vector<RankType> ranks;
     // find all ranks that have voted
     for (RankType r = 0; r < maxNumRanks; ++r) {
-      if (vote & (static_cast<boost::multiprecision::uint256_t>(1) << r)) {
+      if (vote & (static_cast<UIntForGroupReductionType>(1) << r)) {
         ranks.push_back(r);
       }
     }