From 9d750047beea04fb9f4cc3924ad3cb09fed94d99 Mon Sep 17 00:00:00 2001 From: Logan Lamb Date: Mon, 6 May 2024 16:18:07 +0000 Subject: [PATCH] Add two fields to fd_topo_run_tile_t: int for_tpool; when set this a call to run_tile_thread() will not spawn a thread so the tpool can handle the thread spawning later int (*main )( void ); When not NULL this function is called instead of fd_mux_tile(). --- src/app/fddev/main1.c | 2 ++ src/disco/topo/fd_topo.h | 2 ++ src/disco/topo/fd_topo_run.c | 43 +++++++++++++++++++++--------------- 3 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/app/fddev/main1.c b/src/app/fddev/main1.c index 7e4f5527d7..27836b542c 100644 --- a/src/app/fddev/main1.c +++ b/src/app/fddev/main1.c @@ -51,6 +51,8 @@ extern fd_topo_run_tile_t fd_tile_bencho; extern fd_topo_run_tile_t fd_tile_benchg; extern fd_topo_run_tile_t fd_tile_benchs; +fd_topo_run_tile_t fd_tile_tvu_thread = { .name = "thread", .for_tpool = 1 }; + fd_topo_run_tile_t * TILES[] = { &fd_tile_net, &fd_tile_netmux, diff --git a/src/disco/topo/fd_topo.h b/src/disco/topo/fd_topo.h index 337beb8740..9d03f28028 100644 --- a/src/disco/topo/fd_topo.h +++ b/src/disco/topo/fd_topo.h @@ -256,6 +256,7 @@ typedef struct { ulong mux_flags; ulong burst; ulong rlimit_file_cnt; + int for_tpool; void * (*mux_ctx )( void * scratch ); fd_mux_during_housekeeping_fn * mux_during_housekeeping; @@ -274,6 +275,7 @@ typedef struct { ulong (*loose_footprint )( fd_topo_tile_t const * tile ); void (*privileged_init )( fd_topo_t * topo, fd_topo_tile_t * tile, void * scratch ); void (*unprivileged_init )( fd_topo_t * topo, fd_topo_tile_t * tile, void * scratch ); + int (*main )( void ); } fd_topo_run_tile_t; FD_PROTOTYPES_BEGIN diff --git a/src/disco/topo/fd_topo_run.c b/src/disco/topo/fd_topo_run.c index 5eabe129d4..80de93f03f 100644 --- a/src/disco/topo/fd_topo_run.c +++ b/src/disco/topo/fd_topo_run.c @@ -161,22 +161,27 @@ fd_topo_run_tile( fd_topo_t * topo, if( FD_UNLIKELY( tile_run->lazy ) ) lazy = tile_run->lazy( tile_mem ); fd_rng_t rng[1]; - fd_mux_tile( tile->cnc, - tile_run->mux_flags, - polled_in_cnt, - in_mcache, - in_fseq, - tile->out_link_id_primary == ULONG_MAX ? NULL : topo->links[ tile->out_link_id_primary ].mcache, - out_cnt_reliable, - out_fseq, - tile_run->burst, - 0, - lazy, - fd_rng_join( fd_rng_new( rng, 0, 0UL ) ), - fd_alloca( FD_MUX_TILE_SCRATCH_ALIGN, FD_MUX_TILE_SCRATCH_FOOTPRINT( polled_in_cnt, out_cnt_reliable ) ), - ctx, - &callbacks ); - FD_LOG_ERR(( "tile run loop returned" )); + int ret = 0; + if( FD_LIKELY( tile_run->main == NULL ) ) { + ret = fd_mux_tile( tile->cnc, + tile_run->mux_flags, + polled_in_cnt, + in_mcache, + in_fseq, + tile->out_link_id_primary == ULONG_MAX ? NULL : topo->links[ tile->out_link_id_primary ].mcache, + out_cnt_reliable, + out_fseq, + tile_run->burst, + 0, + lazy, + fd_rng_join( fd_rng_new( rng, 0, 0UL ) ), + fd_alloca( FD_MUX_TILE_SCRATCH_ALIGN, FD_MUX_TILE_SCRATCH_FOOTPRINT( polled_in_cnt, out_cnt_reliable ) ), + ctx, + &callbacks ); + } else { + ret = tile_run->main(); + } + FD_LOG_ERR(( "tile run loop returned: %d", ret )); } typedef struct { @@ -279,7 +284,7 @@ fd_topo_tile_stack_new( int optimize, return stack; } -static inline pthread_t +static inline void run_tile_thread( fd_topo_t * topo, fd_topo_tile_t * tile, fd_topo_run_tile_t tile_run, @@ -288,6 +293,9 @@ run_tile_thread( fd_topo_t * topo, int * done_futex, fd_cpuset_t const * floating_cpu_set, int floating_priority ) { + /* tpool will assign a thread later */ + if( FD_UNLIKELY( tile_run.for_tpool ) ) return; + /* TODO: Use a better CPU idx for the stack if tile is floating */ ulong stack_cpu_idx = 0UL; if( FD_LIKELY( tile->cpu_idx<65535UL ) ) stack_cpu_idx = tile->cpu_idx; @@ -333,7 +341,6 @@ run_tile_thread( fd_topo_t * topo, if( FD_UNLIKELY( pthread_create( &pthread, attr, run_tile_thread_main, &args ) ) ) FD_LOG_ERR(( "pthread_create() failed (%i-%s)", errno, fd_io_strerror( errno ) )); while( !FD_VOLATILE( args.copied ) ) FD_SPIN_PAUSE(); - return pthread; } void