verify: increase tcache size

firedancer-io · Nov 18, 2024 · ebc27b6 · ebc27b6
1 parent a786f70
commit ebc27b6
Show file tree

Hide file tree

Showing 10 changed files with 25 additions and 12 deletions.
diff --git a/src/app/fdctl/config.h b/src/app/fdctl/config.h
@@ -222,6 +222,7 @@ typedef struct {
     } quic;
 
     struct {
+      uint signature_cache_size;
       uint receive_buffer_size;
       uint mtu;
     } verify;

diff --git a/src/app/fdctl/config/default.toml b/src/app/fdctl/config/default.toml
@@ -928,6 +928,12 @@ dynamic_port_range = "8900-9000"
     # transactions, making sure that the data is well-formed, and that
     # it is signed by the appropriate private key.
     [tiles.verify]
+        # The verify tiles have a cache of signatures they have seen
+        # recently, used to discard duplicate transactions before they
+        # get verified to save signature verification resources.  See
+        # [tiles.dedup.signature_cache_size] below for more information.
+        signature_cache_size = 4194302
+
         # The maximum number of messages in-flight between a QUIC tile
         # and associated verify tile, after which earlier messages might
         # start being overwritten, and get dropped so that the system
@@ -949,7 +955,10 @@ dynamic_port_range = "8900-9000"
         # is available, it can make sense to increase this cache size to
         # protect against denial of service from high volumes of
         # transaction spam.
-        signature_cache_size = 4194302
+        #
+        # The default value here, 2^26 - 2 is a good balance, as it fits
+        # in 1GiB of memory using a single gigantic page.
+        signature_cache_size = 33554430
 
     # The pack tile takes incoming transactions that have been verified
     # by the verify tile and then deduplicated, and attempts to order

diff --git a/src/app/fdctl/config_parse.c b/src/app/fdctl/config_parse.c
@@ -298,6 +298,7 @@ fdctl_pod_to_cfg( config_t * config,
   CFG_POP      ( uint,   tiles.quic.ack_delay_millis                      );
   CFG_POP      ( bool,   tiles.quic.retry                                 );
 
+  CFG_POP      ( uint,   tiles.verify.signature_cache_size                );
   CFG_POP      ( uint,   tiles.verify.receive_buffer_size                 );
   CFG_POP      ( uint,   tiles.verify.mtu                                 );
 
@@ -455,6 +456,7 @@ fdctl_cfg_validate( config_t * cfg ) {
   CFG_HAS_NON_ZERO( tiles.quic.max_inflight_quic_packets );
   CFG_HAS_NON_ZERO( tiles.quic.idle_timeout_millis );
 
+  CFG_HAS_NON_ZERO( tiles.verify.signature_cache_size );
   CFG_HAS_NON_ZERO( tiles.verify.receive_buffer_size );
 
   CFG_HAS_NON_ZERO( tiles.dedup.signature_cache_size );

diff --git a/src/app/fdctl/run/tiles/fd_dedup.c b/src/app/fdctl/run/tiles/fd_dedup.c
@@ -62,10 +62,9 @@ scratch_align( void ) {
 
 FD_FN_PURE static inline ulong
 scratch_footprint( fd_topo_tile_t const * tile ) {
-  (void)tile;
   ulong l = FD_LAYOUT_INIT;
   l = FD_LAYOUT_APPEND( l, alignof( fd_dedup_ctx_t ), sizeof( fd_dedup_ctx_t ) );
-  l = FD_LAYOUT_APPEND( l, fd_tcache_align(), fd_tcache_footprint( tile->dedup.tcache_depth, 0 ) );
+  l = FD_LAYOUT_APPEND( l, fd_tcache_align(), fd_tcache_footprint( tile->dedup.tcache_depth, 0UL ) );
   return FD_LAYOUT_FINI( l, scratch_align() );
 }
 

diff --git a/src/app/fdctl/run/tiles/fd_verify.c b/src/app/fdctl/run/tiles/fd_verify.c
@@ -19,10 +19,9 @@ scratch_align( void ) {
 
 FD_FN_PURE static inline ulong
 scratch_footprint( fd_topo_tile_t const * tile ) {
-  (void)tile;
   ulong l = FD_LAYOUT_INIT;
   l = FD_LAYOUT_APPEND( l, alignof( fd_verify_ctx_t ), sizeof( fd_verify_ctx_t ) );
-  l = FD_LAYOUT_APPEND( l, fd_tcache_align(), fd_tcache_footprint( VERIFY_TCACHE_DEPTH, VERIFY_TCACHE_MAP_CNT ) );
+  l = FD_LAYOUT_APPEND( l, fd_tcache_align(), fd_tcache_footprint( tile->verify.tcache_depth, 0UL ) );
   for( ulong i=0; i<FD_TXN_ACTUAL_SIG_MAX; i++ ) {
     l = FD_LAYOUT_APPEND( l, fd_sha512_align(), fd_sha512_footprint() );
   }
@@ -160,7 +159,7 @@ unprivileged_init( fd_topo_t *      topo,
 
   FD_SCRATCH_ALLOC_INIT( l, scratch );
   fd_verify_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_verify_ctx_t ), sizeof( fd_verify_ctx_t ) );
-  fd_tcache_t * tcache = fd_tcache_join( fd_tcache_new( FD_SCRATCH_ALLOC_APPEND( l, FD_TCACHE_ALIGN, FD_TCACHE_FOOTPRINT( VERIFY_TCACHE_DEPTH, VERIFY_TCACHE_MAP_CNT ) ), VERIFY_TCACHE_DEPTH, VERIFY_TCACHE_MAP_CNT ) );
+  fd_tcache_t * tcache = fd_tcache_join( fd_tcache_new( FD_SCRATCH_ALLOC_APPEND( l, FD_TCACHE_ALIGN, FD_TCACHE_FOOTPRINT( tile->verify.tcache_depth, 0UL ) ), tile->verify.tcache_depth, 0UL ) );
   if( FD_UNLIKELY( !tcache ) ) FD_LOG_ERR(( "fd_tcache_join failed" ));
 
   ctx->round_robin_cnt = fd_topo_tile_name_cnt( topo, tile->name );

diff --git a/src/app/fdctl/run/tiles/fd_verify.h b/src/app/fdctl/run/tiles/fd_verify.h
@@ -3,9 +3,6 @@
 
 #include "../../../../disco/tiles.h"
 
-#define VERIFY_TCACHE_DEPTH   16UL
-#define VERIFY_TCACHE_MAP_CNT 64UL
-
 #define FD_TXN_VERIFY_SUCCESS  0
 #define FD_TXN_VERIFY_FAILED  -1
 #define FD_TXN_VERIFY_DEDUP   -2

diff --git a/src/app/fdctl/run/tiles/test_verify.c b/src/app/fdctl/run/tiles/test_verify.c
@@ -127,13 +127,13 @@ static void
 setup_verify_ctx( fd_verify_ctx_t * ctx, void ** mem ) {
   fd_memset( ctx, 0, sizeof(fd_verify_ctx_t) );
   /* tcache - note: using aligned_alloc for tests */
-  ulong depth     = VERIFY_TCACHE_DEPTH;
-  ulong map_cnt   = VERIFY_TCACHE_MAP_CNT;
+  ulong depth     = 16UL;
+  ulong map_cnt   = 64UL;
   ulong align     = fd_tcache_align();
   ulong footprint = fd_tcache_footprint( depth, map_cnt );
   if( FD_UNLIKELY( !footprint ) ) FD_LOG_ERR(( "bad depth / map_cnt" ));
   *mem = aligned_alloc( align, footprint ); FD_TEST( *mem );
-  fd_tcache_t * tcache = fd_tcache_join( fd_tcache_new( *mem, VERIFY_TCACHE_DEPTH, VERIFY_TCACHE_MAP_CNT ) );
+  fd_tcache_t * tcache = fd_tcache_join( fd_tcache_new( *mem, depth, map_cnt ) );
   if( FD_UNLIKELY( !tcache ) ) FD_LOG_ERR(( "fd_tcache_join failed" ));
   ctx->tcache_depth   = fd_tcache_depth       ( tcache );
   ctx->tcache_map_cnt = fd_tcache_map_cnt     ( tcache );

diff --git a/src/app/fdctl/run/topos/fd_firedancer.c b/src/app/fdctl/run/topos/fd_firedancer.c
@@ -487,6 +487,7 @@ fd_topo_initialize( config_t * config ) {
       tile->quic.max_concurrent_streams_per_connection = config->tiles.quic.max_concurrent_streams_per_connection;
 
     } else if( FD_UNLIKELY( !strcmp( tile->name, "verify" ) ) ) {
+      tile->verify.tcache_depth = config->tiles.verify.signature_cache_size;
 
     } else if( FD_UNLIKELY( !strcmp( tile->name, "dedup" ) ) ) {
       tile->dedup.tcache_depth = config->tiles.dedup.signature_cache_size;

diff --git a/src/app/fdctl/run/topos/fd_frankendancer.c b/src/app/fdctl/run/topos/fd_frankendancer.c
@@ -375,6 +375,7 @@ fd_topo_initialize( config_t * config ) {
       tile->quic.max_concurrent_streams_per_connection = config->tiles.quic.max_concurrent_streams_per_connection;
 
     } else if( FD_UNLIKELY( !strcmp( tile->name, "verify" ) ) ) {
+      tile->verify.tcache_depth = config->tiles.verify.signature_cache_size;
 
     } else if( FD_UNLIKELY( !strcmp( tile->name, "dedup" ) ) ) {
       tile->dedup.tcache_depth = config->tiles.dedup.signature_cache_size;

diff --git a/src/disco/topo/fd_topo.h b/src/disco/topo/fd_topo.h
@@ -167,6 +167,10 @@ typedef struct {
       int    retry;
     } quic;
 
+    struct {
+      ulong tcache_depth;
+    } verify;
+
     struct {
       ulong tcache_depth;
     } dedup;