From ebc27b6893f11a0219a36af7a17c762c01a0772d Mon Sep 17 00:00:00 2001 From: Michael McGee Date: Mon, 18 Nov 2024 17:44:32 +0000 Subject: [PATCH] verify: increase tcache size --- src/app/fdctl/config.h | 1 + src/app/fdctl/config/default.toml | 11 ++++++++++- src/app/fdctl/config_parse.c | 2 ++ src/app/fdctl/run/tiles/fd_dedup.c | 3 +-- src/app/fdctl/run/tiles/fd_verify.c | 5 ++--- src/app/fdctl/run/tiles/fd_verify.h | 3 --- src/app/fdctl/run/tiles/test_verify.c | 6 +++--- src/app/fdctl/run/topos/fd_firedancer.c | 1 + src/app/fdctl/run/topos/fd_frankendancer.c | 1 + src/disco/topo/fd_topo.h | 4 ++++ 10 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/app/fdctl/config.h b/src/app/fdctl/config.h index 709d1d5ae6..e54d1a5a2c 100644 --- a/src/app/fdctl/config.h +++ b/src/app/fdctl/config.h @@ -222,6 +222,7 @@ typedef struct { } quic; struct { + uint signature_cache_size; uint receive_buffer_size; uint mtu; } verify; diff --git a/src/app/fdctl/config/default.toml b/src/app/fdctl/config/default.toml index 8345bb78ea..ab6165cf6d 100644 --- a/src/app/fdctl/config/default.toml +++ b/src/app/fdctl/config/default.toml @@ -928,6 +928,12 @@ dynamic_port_range = "8900-9000" # transactions, making sure that the data is well-formed, and that # it is signed by the appropriate private key. [tiles.verify] + # The verify tiles have a cache of signatures they have seen + # recently, used to discard duplicate transactions before they + # get verified to save signature verification resources. See + # [tiles.dedup.signature_cache_size] below for more information. + signature_cache_size = 4194302 + # The maximum number of messages in-flight between a QUIC tile # and associated verify tile, after which earlier messages might # start being overwritten, and get dropped so that the system @@ -949,7 +955,10 @@ dynamic_port_range = "8900-9000" # is available, it can make sense to increase this cache size to # protect against denial of service from high volumes of # transaction spam. - signature_cache_size = 4194302 + # + # The default value here, 2^26 - 2 is a good balance, as it fits + # in 1GiB of memory using a single gigantic page. + signature_cache_size = 33554430 # The pack tile takes incoming transactions that have been verified # by the verify tile and then deduplicated, and attempts to order diff --git a/src/app/fdctl/config_parse.c b/src/app/fdctl/config_parse.c index db349dfd70..cd8ed0bb63 100644 --- a/src/app/fdctl/config_parse.c +++ b/src/app/fdctl/config_parse.c @@ -298,6 +298,7 @@ fdctl_pod_to_cfg( config_t * config, CFG_POP ( uint, tiles.quic.ack_delay_millis ); CFG_POP ( bool, tiles.quic.retry ); + CFG_POP ( uint, tiles.verify.signature_cache_size ); CFG_POP ( uint, tiles.verify.receive_buffer_size ); CFG_POP ( uint, tiles.verify.mtu ); @@ -455,6 +456,7 @@ fdctl_cfg_validate( config_t * cfg ) { CFG_HAS_NON_ZERO( tiles.quic.max_inflight_quic_packets ); CFG_HAS_NON_ZERO( tiles.quic.idle_timeout_millis ); + CFG_HAS_NON_ZERO( tiles.verify.signature_cache_size ); CFG_HAS_NON_ZERO( tiles.verify.receive_buffer_size ); CFG_HAS_NON_ZERO( tiles.dedup.signature_cache_size ); diff --git a/src/app/fdctl/run/tiles/fd_dedup.c b/src/app/fdctl/run/tiles/fd_dedup.c index 392a653fd8..7ddd401517 100644 --- a/src/app/fdctl/run/tiles/fd_dedup.c +++ b/src/app/fdctl/run/tiles/fd_dedup.c @@ -62,10 +62,9 @@ scratch_align( void ) { FD_FN_PURE static inline ulong scratch_footprint( fd_topo_tile_t const * tile ) { - (void)tile; ulong l = FD_LAYOUT_INIT; l = FD_LAYOUT_APPEND( l, alignof( fd_dedup_ctx_t ), sizeof( fd_dedup_ctx_t ) ); - l = FD_LAYOUT_APPEND( l, fd_tcache_align(), fd_tcache_footprint( tile->dedup.tcache_depth, 0 ) ); + l = FD_LAYOUT_APPEND( l, fd_tcache_align(), fd_tcache_footprint( tile->dedup.tcache_depth, 0UL ) ); return FD_LAYOUT_FINI( l, scratch_align() ); } diff --git a/src/app/fdctl/run/tiles/fd_verify.c b/src/app/fdctl/run/tiles/fd_verify.c index db76b083b7..445b4d313b 100644 --- a/src/app/fdctl/run/tiles/fd_verify.c +++ b/src/app/fdctl/run/tiles/fd_verify.c @@ -19,10 +19,9 @@ scratch_align( void ) { FD_FN_PURE static inline ulong scratch_footprint( fd_topo_tile_t const * tile ) { - (void)tile; ulong l = FD_LAYOUT_INIT; l = FD_LAYOUT_APPEND( l, alignof( fd_verify_ctx_t ), sizeof( fd_verify_ctx_t ) ); - l = FD_LAYOUT_APPEND( l, fd_tcache_align(), fd_tcache_footprint( VERIFY_TCACHE_DEPTH, VERIFY_TCACHE_MAP_CNT ) ); + l = FD_LAYOUT_APPEND( l, fd_tcache_align(), fd_tcache_footprint( tile->verify.tcache_depth, 0UL ) ); for( ulong i=0; iverify.tcache_depth, 0UL ) ), tile->verify.tcache_depth, 0UL ) ); if( FD_UNLIKELY( !tcache ) ) FD_LOG_ERR(( "fd_tcache_join failed" )); ctx->round_robin_cnt = fd_topo_tile_name_cnt( topo, tile->name ); diff --git a/src/app/fdctl/run/tiles/fd_verify.h b/src/app/fdctl/run/tiles/fd_verify.h index 9ef8f48fde..fef8d94450 100644 --- a/src/app/fdctl/run/tiles/fd_verify.h +++ b/src/app/fdctl/run/tiles/fd_verify.h @@ -3,9 +3,6 @@ #include "../../../../disco/tiles.h" -#define VERIFY_TCACHE_DEPTH 16UL -#define VERIFY_TCACHE_MAP_CNT 64UL - #define FD_TXN_VERIFY_SUCCESS 0 #define FD_TXN_VERIFY_FAILED -1 #define FD_TXN_VERIFY_DEDUP -2 diff --git a/src/app/fdctl/run/tiles/test_verify.c b/src/app/fdctl/run/tiles/test_verify.c index 2e7d76c4c1..3b3cbad9ff 100644 --- a/src/app/fdctl/run/tiles/test_verify.c +++ b/src/app/fdctl/run/tiles/test_verify.c @@ -127,13 +127,13 @@ static void setup_verify_ctx( fd_verify_ctx_t * ctx, void ** mem ) { fd_memset( ctx, 0, sizeof(fd_verify_ctx_t) ); /* tcache - note: using aligned_alloc for tests */ - ulong depth = VERIFY_TCACHE_DEPTH; - ulong map_cnt = VERIFY_TCACHE_MAP_CNT; + ulong depth = 16UL; + ulong map_cnt = 64UL; ulong align = fd_tcache_align(); ulong footprint = fd_tcache_footprint( depth, map_cnt ); if( FD_UNLIKELY( !footprint ) ) FD_LOG_ERR(( "bad depth / map_cnt" )); *mem = aligned_alloc( align, footprint ); FD_TEST( *mem ); - fd_tcache_t * tcache = fd_tcache_join( fd_tcache_new( *mem, VERIFY_TCACHE_DEPTH, VERIFY_TCACHE_MAP_CNT ) ); + fd_tcache_t * tcache = fd_tcache_join( fd_tcache_new( *mem, depth, map_cnt ) ); if( FD_UNLIKELY( !tcache ) ) FD_LOG_ERR(( "fd_tcache_join failed" )); ctx->tcache_depth = fd_tcache_depth ( tcache ); ctx->tcache_map_cnt = fd_tcache_map_cnt ( tcache ); diff --git a/src/app/fdctl/run/topos/fd_firedancer.c b/src/app/fdctl/run/topos/fd_firedancer.c index 1871b5b277..9477f10924 100644 --- a/src/app/fdctl/run/topos/fd_firedancer.c +++ b/src/app/fdctl/run/topos/fd_firedancer.c @@ -487,6 +487,7 @@ fd_topo_initialize( config_t * config ) { tile->quic.max_concurrent_streams_per_connection = config->tiles.quic.max_concurrent_streams_per_connection; } else if( FD_UNLIKELY( !strcmp( tile->name, "verify" ) ) ) { + tile->verify.tcache_depth = config->tiles.verify.signature_cache_size; } else if( FD_UNLIKELY( !strcmp( tile->name, "dedup" ) ) ) { tile->dedup.tcache_depth = config->tiles.dedup.signature_cache_size; diff --git a/src/app/fdctl/run/topos/fd_frankendancer.c b/src/app/fdctl/run/topos/fd_frankendancer.c index a6354f0a3b..ab34af9bb1 100644 --- a/src/app/fdctl/run/topos/fd_frankendancer.c +++ b/src/app/fdctl/run/topos/fd_frankendancer.c @@ -375,6 +375,7 @@ fd_topo_initialize( config_t * config ) { tile->quic.max_concurrent_streams_per_connection = config->tiles.quic.max_concurrent_streams_per_connection; } else if( FD_UNLIKELY( !strcmp( tile->name, "verify" ) ) ) { + tile->verify.tcache_depth = config->tiles.verify.signature_cache_size; } else if( FD_UNLIKELY( !strcmp( tile->name, "dedup" ) ) ) { tile->dedup.tcache_depth = config->tiles.dedup.signature_cache_size; diff --git a/src/disco/topo/fd_topo.h b/src/disco/topo/fd_topo.h index 5435eec6ac..4994aa91c3 100644 --- a/src/disco/topo/fd_topo.h +++ b/src/disco/topo/fd_topo.h @@ -167,6 +167,10 @@ typedef struct { int retry; } quic; + struct { + ulong tcache_depth; + } verify; + struct { ulong tcache_depth; } dedup;