[WIP] Graph Concepts and Containers

dart-if/include/dash/dart/if/dart_communication.h

@@ -165,6 +165,60 @@ dart_ret_t dart_allgatherv(
   const size_t    * recvdispls,
   dart_team_t       teamid) DART_NOTHROW;
 
+/**
+ * DART Equivalent to MPI alltoall.
+ *
+ * \param sendbuf The buffer containing the data to be sent by each unit.
+ * \param recvbuf The buffer to hold the received data.
+ * \param nelem   Number of values sent by each process and received from
+ *                each unit.
+ * \param dtype   The data type of values in \c sendbuf and \c recvbuf.
+ * \param team    The team to participate in the allgather.
+ *
+ * \return \c DART_OK on success, any other of \ref dart_ret_t otherwise.
+ *
+ * \threadsafe_data{team}
+ * \ingroup DartCommunication
+ */
+dart_ret_t dart_alltoall(
+  const void      * sendbuf,
+  void            * recvbuf,
+  size_t            nelem,
+  dart_datatype_t   dtype,
+	dart_team_t       team) DART_NOTHROW;
+
+/**
+ * DART Equivalent to MPI alltoallv.
+ *
+ * \param sendbuf     The buffer containing the data to be sent by each unit.
+ * \param nsendcounts Array containing the number of values to be sent by 
+ *                    each unit.
+ * \param senddispls  Array containing the displacements of data to be sent
+ *                    by each unit.
+ * \param dtype       The data type of values in \c sendbuf and \c recvbuf.
+ * \param recvbuf     The buffer to hold the received data.
+ * \param nrecvelem   Array containing the number of values to receive from
+ *                    each unit.
+ * \param recvdispls  Array containing the displacements of data received
+ *                    from each unit in \c recvbuf.
+ * \param teamid      The team to participate in the allgatherv.
+ *
+ * \return \c DART_OK on success, any other of \ref dart_ret_t otherwise.
+ *
+ * \threadsafe_data{team}
+ * \ingroup DartCommunication
+ */
+dart_ret_t dart_alltoallv(
+  const void      * sendbuf,
+  const size_t    * nsendcounts,
+  const size_t    * senddispls,
+  dart_datatype_t   dtype,
+  void            * recvbuf,
+  const size_t    * nrecvelem,
+  const size_t    * recvdispls,
+  dart_team_t       teamid) DART_NOTHROW;
+
+
 /**
  * DART Equivalent to MPI allreduce.
  *

dart-impl/mpi/src/dart_communication.c

@@ -2120,7 +2120,7 @@ dart_ret_t dart_allgatherv(
     DART_LOG_ERROR("dart_allgatherv ! unknown teamid %d", teamid);
     return DART_ERR_INVAL;
   }
-  if (sendbuf == recvbuf || NULL == sendbuf) {
+  if (sendbuf == recvbuf) {
     sendbuf = MPI_IN_PLACE;
   }
   MPI_Comm comm      = team_data->comm;
@@ -2168,6 +2168,150 @@ dart_ret_t dart_allgatherv(
   return DART_OK;
 }
 
+#if 0
+/*
+ * Implementation from branch feat-graph, to be discussed
+ */
+dart_ret_t dart_alltoall(
+  const void      * sendbuf,
+  void            * recvbuf,
+  size_t            nelem,
+  dart_datatype_t   dtype,
+  dart_team_t       teamid)
+{
+  MPI_Datatype mpi_dtype = dart__mpi__datatype_struct(dtype)->basic.mpi_type;
+  MPI_Comm     comm;
+  DART_LOG_TRACE("dart_alltoall() team:%d nelem:%"PRIu64"",
+                 teamid, nelem);
+
+  if (teamid == DART_UNDEFINED_TEAM_ID) {
+    DART_LOG_ERROR("dart_alltoall ! failed: team may not be DART_UNDEFINED_TEAM_ID");
+    return DART_ERR_INVAL;
+  }
+
+  /*
+   * MPI uses offset type int, do not copy more than INT_MAX elements:
+   */
+  if (nelem > INT_MAX) {
+    DART_LOG_ERROR("dart_alltoall ! failed: nelem > INT_MAX");
+    return DART_ERR_INVAL;
+  }
+
+  dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid);
+  if (team_data == NULL) {
+    DART_LOG_ERROR("dart_alltoall ! team:%d "
+                   "dart_adapt_teamlist_convert failed", teamid);
+    return DART_ERR_INVAL;
+  }
+  if (sendbuf == recvbuf || NULL == sendbuf) {
+    sendbuf = MPI_IN_PLACE;
+  }
+  comm = team_data->comm;
+  if (MPI_Alltoall(
+           sendbuf,
+           nelem,
+           mpi_dtype,
+           recvbuf,
+           nelem,
+           mpi_dtype,
+           comm) != MPI_SUCCESS) {
+    DART_LOG_ERROR("dart_alltoall ! team:%d nelem:%"PRIu64" failed",
+                   teamid, nelem);
+    return DART_ERR_INVAL;
+  }
+  DART_LOG_TRACE("dart_alltoall > team:%d nelem:%"PRIu64"",
+                 teamid, nelem);
+  return DART_OK;
+}
+#endif
+
+/*
+ * Implementation from branch feat-graph, to be discussed
+ */
+dart_ret_t dart_alltoallv(
+  const void      * sendbuf,
+  const size_t    * nsendcounts,
+  const size_t    * senddispls,
+  dart_datatype_t   dtype,
+  void            * recvbuf,
+  const size_t    * nrecvcounts,
+  const size_t    * recvdispls,
+  dart_team_t       teamid)
+{
+  MPI_Datatype mpi_dtype = dart__mpi__datatype_struct(dtype)->contiguous.mpi_type;
+  MPI_Comm     comm;
+  int          comm_size;
+  DART_LOG_TRACE("dart_alltoallv() team:%d nsendelem:%"PRIu64"",
+                 teamid, nsendcounts);
+
+  if (teamid == DART_UNDEFINED_TEAM_ID) {
+    DART_LOG_ERROR(
+      "dart_alltoallv ! failed: team may not be DART_UNDEFINED_TEAM_ID");
+    return DART_ERR_INVAL;
+  }
+
+  dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid);
+  if (team_data == NULL) {
+    DART_LOG_ERROR("dart_alltoallv ! team:%d "
+                   "dart_adapt_teamlist_convert failed", teamid);
+    return DART_ERR_INVAL;
+  }
+  comm = team_data->comm;
+
+  // convert nrecvcounts and recvdispls
+  MPI_Comm_size(comm, &comm_size);
+  int *insendcounts = malloc(sizeof(int) * comm_size);
+  int *isenddispls  = malloc(sizeof(int) * comm_size);
+  int *inrecvcounts = malloc(sizeof(int) * comm_size);
+  int *irecvdispls  = malloc(sizeof(int) * comm_size);
+  for (int i = 0; i < comm_size; i++) {
+    if (nsendcounts[i] > INT_MAX || senddispls[i] > INT_MAX) {
+      DART_LOG_ERROR(
+        "dart_alltoallv ! failed: "
+        "nsendcounts[%i] > INT_MAX || senddispls[%i] > INT_MAX", i, i);
+      free(insendcounts);
+      free(isenddispls);
+      return DART_ERR_INVAL;
+    }
+    if (nrecvcounts[i] > INT_MAX || recvdispls[i] > INT_MAX) {
+      DART_LOG_ERROR(
+        "dart_alltoallv ! failed: "
+        "nrecvcounts[%i] > INT_MAX || recvdispls[%i] > INT_MAX", i, i);
+      free(inrecvcounts);
+      free(irecvdispls);
+      return DART_ERR_INVAL;
+    }
+    insendcounts[i] = nsendcounts[i];
+    isenddispls[i]  = senddispls[i];
+    inrecvcounts[i] = nrecvcounts[i];
+    irecvdispls[i]  = recvdispls[i];
+  }
+
+  if (MPI_Alltoallv(
+           sendbuf,
+           insendcounts,
+           isenddispls,
+           mpi_dtype,
+           recvbuf,
+           inrecvcounts,
+           irecvdispls,
+           mpi_dtype,
+           comm) != MPI_SUCCESS) {
+    DART_LOG_ERROR("dart_alltoallv ! team:%d failed", teamid);
+    free(insendcounts);
+    free(isenddispls);
+    free(inrecvcounts);
+    free(irecvdispls);
+    return DART_ERR_INVAL;
+  }
+  free(insendcounts);
+  free(isenddispls);
+  free(inrecvcounts);
+  free(irecvdispls);
+  DART_LOG_TRACE("dart_alltoallv > team:%d", teamid);
+  return DART_OK;
+}
+
 dart_ret_t dart_allreduce(
   const void       * sendbuf,
   void             * recvbuf,

dash/examples/ex.02.graph-deref/main.cpp

@@ -0,0 +1,121 @@
+#include <libdash.h>
+#include <iostream>
+#include <ctime>
+#include <math.h>
+// #include "rmatrandom.h"
+
+struct vprop {
+  int comp;
+};
+
+struct eprop {
+  int comp;
+};
+
+typedef dash::Graph<dash::DirectedGraph, vprop, eprop>       graph_t;
+
+int main(int argc, char* argv[]) {
+  dash::init(&argc, &argv);
+
+  int n_unit_edges = 1;
+  int n_vertices_full = 448;
+  int n_edges_full = n_vertices_full * n_unit_edges;
+  // not exactly n_vertices_full are generated due to rounding
+  int n_vertices_start = n_vertices_full / dash::size();
+  int n_size_rounds = 5;
+  int n_rounds = 1;
+  for(int rounds = 0; rounds < n_size_rounds; ++rounds) {
+    for(int i = 0; i < n_rounds; ++i) {
+      int scale = rounds + 1;
+      int n_vertices = n_vertices_start * scale;
+      graph_t g(n_vertices, n_unit_edges);
+
+      for(int j = 0; j < n_vertices; ++j) {
+        g.add_vertex();
+      }
+
+      g.commit();
+
+      int n_edges = n_vertices * n_unit_edges;
+      auto src = g.vertices().lbegin();
+      auto trg = src + 1;
+      auto end = g.vertices().lend();
+      for(int j = 0; j < n_edges; ++j) {
+        g.add_edge(src, trg);
+        ++src;
+        ++trg;
+        if(trg == end) {
+          src = g.vertices().lbegin();
+          trg = src + 1;
+        }
+      }
+
+      g.commit();
+
+      std::clock_t g_begin_time = clock();
+      for(auto it = g.vertices().lbegin(); it != g.vertices().lend(); ++it) {
+        auto v = g[it];
+        v.attributes();
+      }
+      std::clock_t g_end_time = clock();
+      double time = double(g_end_time - g_begin_time) / CLOCKS_PER_SEC;
+
+      double all_time;
+      dart_reduce(&time, &all_time, 1, DART_TYPE_DOUBLE, DART_OP_SUM, 0, 
+          g.team().dart_id());
+
+      if(dash::myid() == 0) {
+        std::cout << "[round " << i << "] " << n_vertices_full * scale << 
+          " vertices per node dereferenced (local): " << all_time << 
+          std::endl;
+      }
+
+      if(dash::myid() == 0) {
+        std::clock_t g_begin_time = clock();
+        for(auto it = g.vertices().begin(); it != g.vertices().end(); ++it) {
+         auto v = g[it];
+         v.attributes();
+        }
+        std::clock_t g_end_time = clock();
+        std::cout << "[round " << i << "] " << n_vertices_full * scale << 
+          " vertices per node dereferenced (global): " << 
+          double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl;
+      }
+
+      g_begin_time = clock();
+      for(auto it = g.out_edges().lbegin(); it != g.out_edges().lend(); ++it) {
+        auto e = g[it];
+        e.attributes();
+      }
+      g_end_time = clock();
+      time = double(g_end_time - g_begin_time) / CLOCKS_PER_SEC;
+
+      dart_reduce(&time, &all_time, 1, DART_TYPE_DOUBLE, DART_OP_SUM, 0, 
+          g.team().dart_id());
+
+      if(dash::myid() == 0) {
+        std::cout << "[round " << i << "] " << n_edges_full * scale << 
+          " edges per node dereferenced (local): " << all_time << std::endl;
+      }
+
+      if(dash::myid() == 0) {
+        std::clock_t g_begin_time = clock();
+        for(auto it = g.out_edges().begin(); it != g.out_edges().end(); ++it) {
+          auto e = g[it];
+          e.attributes();
+        }
+        std::clock_t g_end_time = clock();
+        std::cout << "[round " << i << "] " << n_edges_full * scale << 
+          " edges per node dereferenced (global): " << 
+          double(g_end_time - g_begin_time) / CLOCKS_PER_SEC << std::endl;
+      }
+      dash::barrier();
+    }
+    if(dash::myid() == 0) std::cout << "-----------------" << std::endl;
+  }
+
+  dash::finalize();
+  return 0;
+} 
+
+