Merge pull request #18 from yosefe/topic/rndv-defer-sched

UCP/RNDV: Add defer sched config
yosefe · Jul 22, 2020 · 9c7a466 · 9c7a466
2 parents e514f4f + d895597
commit 9c7a466
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 4 deletions.
diff --git a/src/ucp/core/ucp_context.c b/src/ucp/core/ucp_context.c
@@ -279,6 +279,10 @@ static ucs_config_field_t ucp_config_table[] = {
    "require out of band synchronization before destroying UCP resources.",
    ucs_offsetof(ucp_config_t, ctx.sockaddr_cm_enable), UCS_CONFIG_TYPE_TERNARY},
 
+  {"RNDV_DEFER_SCHED", "n",
+   "Try to push rndv to pending queue first",
+   ucs_offsetof(ucp_config_t, ctx.rdnv_defer_sched), UCS_CONFIG_TYPE_BOOL},
+
   {NULL}
 };
 UCS_CONFIG_REGISTER_TABLE(ucp_config_table, "UCP context", NULL, ucp_config_t)

diff --git a/src/ucp/core/ucp_context.h b/src/ucp/core/ucp_context.h
@@ -97,6 +97,8 @@ typedef struct ucp_context_config {
     int                                    unified_mode;
     /** Enable cm wireup-and-close protocol for client-server connections */
     ucs_ternary_value_t                    sockaddr_cm_enable;
+    /* Try to post rndv to pending first */
+    int                                    rdnv_defer_sched;
 } ucp_context_config_t;
 
 

diff --git a/src/ucp/tag/rndv.c b/src/ucp/tag/rndv.c
@@ -715,6 +715,7 @@ ucp_rndv_req_init_zcopy_lane_map(ucp_request_t *rndv_req)
 static void ucp_rndv_req_send_rma_get(ucp_request_t *rndv_req, ucp_request_t *rreq,
                                       const ucp_rndv_rts_hdr_t *rndv_rts_hdr)
 {
+    ucp_worker_h worker = rndv_req->send.ep->worker;
     ucs_status_t status;
     uct_rkey_t uct_rkey;
 
@@ -741,11 +742,16 @@ static void ucp_rndv_req_send_rma_get(ucp_request_t *rndv_req, ucp_request_t *rr
     ucp_request_send_state_reset(rndv_req, ucp_rndv_get_completion,
                                  UCP_REQUEST_SEND_PROTO_RNDV_GET);
     ucp_rndv_req_init_zcopy_lane_map(rndv_req);
-    rndv_req->send.lane         = ucp_rndv_get_zcopy_get_lane(rndv_req, &uct_rkey);
-    rndv_req->send.pending_lane = UCP_NULL_LANE;
 
-    if ((rndv_req->send.lane == UCP_NULL_LANE) ||
-        !ucp_request_pending_add(rndv_req, &status, 0)) {
+    if (worker->context->config.ext.rdnv_defer_sched) {
+        rndv_req->send.lane         = ucp_rndv_get_zcopy_get_lane(rndv_req, &uct_rkey);
+        rndv_req->send.pending_lane = UCP_NULL_LANE;
+
+        if ((rndv_req->send.lane == UCP_NULL_LANE) ||
+            !ucp_request_pending_add(rndv_req, &status, 0)) {
+            ucp_request_send(rndv_req, 0);
+        }
+    } else {
         ucp_request_send(rndv_req, 0);
     }
 }