fdctl: support to affine labs to cores

src/app/fdctl/config.c

@@ -227,6 +227,7 @@ static int parse_key_value( config_t *   config,
   ENTRY_STR   ( ., snapshots,           path                                                      );
 
   ENTRY_STR   ( ., layout,              affinity                                                  );
+  ENTRY_STR   ( ., layout,              solana_labs_affinity                                      );
   ENTRY_UINT  ( ., layout,              net_tile_count                                            );
   ENTRY_UINT  ( ., layout,              quic_tile_count                                           );
   ENTRY_UINT  ( ., layout,              verify_tile_count                                         );

src/app/fdctl/config.h

@@ -116,6 +116,8 @@ typedef struct {
 
   struct {
     char affinity[ AFFINITY_SZ ];
+    char solana_labs_affinity[ AFFINITY_SZ ];
+
     uint net_tile_count;
     uint quic_tile_count;
     uint verify_tile_count;

src/app/fdctl/config/default.toml

@@ -498,6 +498,23 @@ dynamic_port_range = "8900-9000"
     # that the Solana network can run as fast as possible.
     affinity = "0-21"
 
+    # In addition to the Firedancer tiles which use a core each, the
+    # current version of Firedancer hosts a Solana Labs (Agave)
+    # validator as a subprocess.
+    #
+    # This affinity congtrols which logical CPU cores the Solana Labs
+    # subprocess and all of its threads are allowed to run on.  This is
+    # specified in the same format as the above Firedancer affinity.
+    #
+    # It is strongly suggested that you do not overlap the Firedancer
+    # affinity with the Solana Labs affinity, as Firedancer tiles expect
+    # to have exclusive use of their core.  Unexpected latency  spikes
+    # due to context switching may decrease performance overall.
+    #
+    # An empty string here will not affine the process, and it will run
+    # be free to run on any floating core.
+    solana_labs_affinity = "22-31"
+
     # How many net tiles to run.  Each networking tile will service
     # exactly one queue from a network device being listened to.  If
     # there are less net tiles than queues, some queues will not get

src/app/fdctl/run/run_solana.c

@@ -140,6 +140,20 @@ solana_labs_boot( config_t * config ) {
   FD_LOG_INFO(( "Running Solana Labs validator with the following arguments:" ));
   for( ulong j=0UL; j<idx; j++ ) FD_LOG_INFO(( "%s", argv[j] ));
 
+  FD_CPUSET_DECL( floating_cpu_set );
+  if( FD_UNLIKELY( fd_cpuset_getaffinity( 0, floating_cpu_set ) ) )
+    FD_LOG_ERR(( "sched_getaffinity failed (%i-%s)", errno, fd_io_strerror( errno ) ));
+
+  ushort solana_labs_cpu[ FD_TILE_MAX ];
+  ulong labs_cpu_cnt = fd_tile_private_cpus_parse( config->layout.solana_labs_affinity, solana_labs_cpu );
+  FD_CPUSET_DECL( cpu_set );
+  for( ulong i=0UL; i<labs_cpu_cnt; i++ ) {
+    fd_cpuset_insert( cpu_set, solana_labs_cpu[ i ] );
+  }
+
+  if( FD_UNLIKELY( fd_cpuset_setaffinity( 0, cpu_set ) ) )
+    FD_LOG_ERR(( "sched_setaffinity failed (%i-%s)", errno, fd_io_strerror( errno ) ));
+
   /* solana labs main will exit(1) if it fails, so no return code */
   fd_ext_validator_main( (const char **)argv );
 }