From e79754b4d8504c7d7db016551f2ba6d4f2a6298e Mon Sep 17 00:00:00 2001 From: Ravyu Sivakumaran Date: Fri, 13 Dec 2024 11:17:32 -0600 Subject: [PATCH] tile, gossip: rewire dedup out to verify --- src/app/fdctl/run/tiles/fd_dedup.c | 51 +++++++++-------------- src/app/fdctl/run/tiles/fd_gossip.c | 54 ++++++++++++++----------- src/app/fdctl/run/topos/fd_firedancer.c | 8 ++-- 3 files changed, 54 insertions(+), 59 deletions(-) diff --git a/src/app/fdctl/run/tiles/fd_dedup.c b/src/app/fdctl/run/tiles/fd_dedup.c index 3890d56c65..69f266ec79 100644 --- a/src/app/fdctl/run/tiles/fd_dedup.c +++ b/src/app/fdctl/run/tiles/fd_dedup.c @@ -16,9 +16,6 @@ checks the transaction signature field for duplicates and filters them out. */ -#define GOSSIP_IN_IDX (0UL) /* Frankendancer and Firedancer */ -#define VOTER_IN_IDX (1UL) /* Firedancer only */ - /* fd_dedup_in_ctx_t is a context object for each in (producer) mcache connected to the dedup tile. */ @@ -41,6 +38,7 @@ typedef struct { /* The first unparsed_in_cnt in links do not have the parsed fd_txn_t in the payload trailer. */ ulong unparsed_in_cnt; + ulong gossip_in_idx; fd_dedup_in_ctx_t in[ 64UL ]; fd_wksp_t * out_mem; @@ -141,7 +139,7 @@ after_frag( fd_dedup_ctx_t * ctx, ulong txn_off = fd_ulong_align_up( payload_sz, 2UL ); fd_txn_t * txn = (fd_txn_t *)(dcache_entry + txn_off); - if ( FD_UNLIKELY( in_idx < ctx->unparsed_in_cnt ) ) { + if( FD_UNLIKELY( in_idx < ctx->unparsed_in_cnt ) ) { /* Transactions coming in from these links are not parsed. We'll need to parse it so it's ready for downstream consumers. @@ -163,7 +161,7 @@ after_frag( fd_dedup_ctx_t * ctx, if( FD_UNLIKELY( !txn_t_sz ) ) FD_LOG_ERR(( "fd_txn_parse failed for vote transactions that should have been sigverified" )); /* Increment on GOSSIP_IN_IDX but not VOTER_IN_IDX */ - FD_MCNT_INC( DEDUP, GOSSIPED_VOTES_RECEIVED, 1UL - in_idx ); + FD_MCNT_INC( DEDUP, GOSSIPED_VOTES_RECEIVED, ( in_idx == ctx->gossip_in_idx ) ); /* Write payload_sz into trailer. fd_txn_parse always returns a multiple of 2 so this sz is @@ -212,32 +210,8 @@ unprivileged_init( fd_topo_t * topo, fd_topo_tile_t * tile ) { void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id ); - /* Frankendancer has gossip_dedup, verify_dedup+ - Firedancer has gossip_dedup, voter_dedup, verify_dedup+ */ - ulong unparsed_in_cnt = 1; - if( FD_UNLIKELY( tile->in_cnt<2UL ) ) { - FD_LOG_ERR(( "dedup tile needs at least two input links, got %lu", tile->in_cnt )); - } else if( FD_UNLIKELY( strcmp( topo->links[ tile->in_link_id[ GOSSIP_IN_IDX ] ].name, "gossip_dedup" ) ) ) { - /* We have one link for gossip messages... */ - FD_LOG_ERR(( "dedup tile has unexpected input links %lu %lu %s", - tile->in_cnt, GOSSIP_IN_IDX, topo->links[ tile->in_link_id[ GOSSIP_IN_IDX ] ].name )); - } else { - /* ...followed by a voter_dedup link if it were the Firedancer topology */ - ulong voter_dedup_idx = fd_topo_find_tile_in_link( topo, tile, "voter_dedup", 0 ); - if( voter_dedup_idx!=ULONG_MAX ) { - FD_TEST( voter_dedup_idx == VOTER_IN_IDX ); - unparsed_in_cnt = 2; - } else { - unparsed_in_cnt = 1; - } - - /* ...followed by a sequence of verify_dedup links */ - for( ulong i=unparsed_in_cnt; iin_cnt; i++ ) { - if( FD_UNLIKELY( strcmp( topo->links[ tile->in_link_id[ i ] ].name, "verify_dedup" ) ) ) { - FD_LOG_ERR(( "dedup tile has unexpected input links %lu %lu %s", - tile->in_cnt, i, topo->links[ tile->in_link_id[ i ] ].name )); - } - } + if( FD_UNLIKELY( tile->in_cnt == 0UL ) ) { + FD_LOG_ERR(( "dedup tile needs at least one input link, got zero" )); } FD_SCRATCH_ALLOC_INIT( l, scratch ); @@ -252,7 +226,8 @@ unprivileged_init( fd_topo_t * topo, ctx->tcache_map = fd_tcache_map_laddr ( tcache ); FD_TEST( tile->in_cnt<=sizeof( ctx->in )/sizeof( ctx->in[ 0 ] ) ); - ctx->unparsed_in_cnt = unparsed_in_cnt; + ulong parsed_in_start_idx = ULONG_MAX; + ctx->gossip_in_idx = ULONG_MAX; for( ulong i=0; iin_cnt; i++ ) { fd_topo_link_t * link = &topo->links[ tile->in_link_id[ i ] ]; fd_topo_wksp_t * link_wksp = &topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ]; @@ -260,7 +235,19 @@ unprivileged_init( fd_topo_t * topo, ctx->in[i].mem = link_wksp->wksp; ctx->in[i].chunk0 = fd_dcache_compact_chunk0( ctx->in[i].mem, link->dcache ); ctx->in[i].wmark = fd_dcache_compact_wmark ( ctx->in[i].mem, link->dcache, link->mtu ); + + if( FD_UNLIKELY( !strcmp( link->name, "gossip_dedup" ) ) ) { + if( FD_UNLIKELY( i > parsed_in_start_idx ) ) FD_LOG_ERR(( "unparsed links must go before parsed ones" )); + ctx->gossip_in_idx = i; + } else if( FD_UNLIKELY( !strcmp( link->name, "voter_dedup" ) ) ) { + if( FD_UNLIKELY( i > parsed_in_start_idx ) ) FD_LOG_ERR(( "unparsed links must go before parsed ones" )); + } else if( FD_LIKELY( !strcmp( link->name, "verify_dedup" ) ) ) { + if( FD_UNLIKELY( parsed_in_start_idx == ULONG_MAX ) ) parsed_in_start_idx = i; + } else { + FD_LOG_ERR(( "dedup tile has unexpected input link name %s", link->name )); + } } + ctx->unparsed_in_cnt = parsed_in_start_idx; ctx->out_mem = topo->workspaces[ topo->objs[ topo->links[ tile->out_link_id[ 0 ] ].dcache_obj_id ].wksp_id ].wksp; ctx->out_chunk0 = fd_dcache_compact_chunk0( ctx->out_mem, topo->links[ tile->out_link_id[ 0 ] ].dcache ); diff --git a/src/app/fdctl/run/tiles/fd_gossip.c b/src/app/fdctl/run/tiles/fd_gossip.c index 19fbc1de63..9af930f272 100644 --- a/src/app/fdctl/run/tiles/fd_gossip.c +++ b/src/app/fdctl/run/tiles/fd_gossip.c @@ -36,7 +36,7 @@ #define NET_OUT_IDX 0 #define SHRED_OUT_IDX 1 #define REPAIR_OUT_IDX 2 -#define DEDUP_OUT_IDX 3 +#define VERIFY_OUT_IDX 3 #define SIGN_OUT_IDX 4 #define VOTER_OUT_IDX 5 #define REPLAY_OUT_IDX 6 @@ -104,15 +104,15 @@ struct fd_gossip_tile_ctx { ulong voter_contact_out_wmark; ulong voter_contact_out_chunk; - fd_frag_meta_t * dedup_out_mcache; - ulong * dedup_out_sync; - ulong dedup_out_depth; - ulong dedup_out_seq; + fd_frag_meta_t * verify_out_mcache; + ulong * verify_out_sync; + ulong verify_out_depth; + ulong verify_out_seq; - fd_wksp_t * dedup_out_mem; - ulong dedup_out_chunk0; - ulong dedup_out_wmark; - ulong dedup_out_chunk; + fd_wksp_t * verify_out_mem; + ulong verify_out_chunk0; + ulong verify_out_wmark; + ulong verify_out_chunk; fd_frag_meta_t * eqvoc_out_mcache; ulong * eqvoc_out_sync; @@ -317,15 +317,23 @@ gossip_deliver_fun( fd_crds_data_t * data, void * arg ) { return; } - uchar * vote_txn_msg = fd_chunk_to_laddr( ctx->dedup_out_mem, ctx->dedup_out_chunk ); + uchar * vote_txn_msg = fd_chunk_to_laddr( ctx->verify_out_mem, ctx->verify_out_chunk ); ulong vote_txn_sz = gossip_vote->txn.raw_sz; memcpy( vote_txn_msg, gossip_vote->txn.raw, vote_txn_sz ); + /* DEFENSE-IN-DEPTH: While the underlying txn message is technically verifed + at this point (by virtue of being a substring of the full CRDS message we + verify in fd_gossip_recv_crds_value), we still send it to the verify tile(s) + to avoid having the single point of verification failure lie in the gossip tile. + + TODO: monitor gossip vote traffic to determine potential impact this might + have on verify traffic */ + ulong sig = 1UL; - fd_mcache_publish( ctx->dedup_out_mcache, ctx->dedup_out_depth, ctx->dedup_out_seq, sig, ctx->dedup_out_chunk, + fd_mcache_publish( ctx->verify_out_mcache, ctx->verify_out_depth, ctx->verify_out_seq, sig, ctx->verify_out_chunk, vote_txn_sz, 0UL, 0, 0 ); - ctx->dedup_out_seq = fd_seq_inc( ctx->dedup_out_seq, 1UL ); - ctx->dedup_out_chunk = fd_dcache_compact_next( ctx->dedup_out_chunk, vote_txn_sz, ctx->dedup_out_chunk0, ctx->dedup_out_wmark ); + ctx->verify_out_seq = fd_seq_inc( ctx->verify_out_seq, 1UL ); + ctx->verify_out_chunk = fd_dcache_compact_next( ctx->verify_out_chunk, vote_txn_sz, ctx->verify_out_chunk0, ctx->verify_out_wmark ); } else if( fd_crds_data_is_contact_info_v1( data ) ) { fd_gossip_contact_info_v1_t const * contact_info = &data->inner.contact_info_v1; @@ -797,7 +805,7 @@ unprivileged_init( fd_topo_t * topo, strcmp( topo->links[ tile->out_link_id[ NET_OUT_IDX ] ].name, "gossip_net" ) || strcmp( topo->links[ tile->out_link_id[ SHRED_OUT_IDX ] ].name, "crds_shred" ) || strcmp( topo->links[ tile->out_link_id[ REPAIR_OUT_IDX ] ].name, "gossip_repai" ) || - strcmp( topo->links[ tile->out_link_id[ DEDUP_OUT_IDX ] ].name, "gossip_dedup" ) || + strcmp( topo->links[ tile->out_link_id[ VERIFY_OUT_IDX ] ].name, "gossip_verif" ) || strcmp( topo->links[ tile->out_link_id[ SIGN_OUT_IDX ] ].name, "gossip_sign" ) || strcmp( topo->links[ tile->out_link_id[ VOTER_OUT_IDX ] ].name, "gossip_voter" ) || strcmp( topo->links[ tile->out_link_id[ REPLAY_OUT_IDX ] ].name, "gossip_repla" ) || @@ -947,15 +955,15 @@ unprivileged_init( fd_topo_t * topo, ctx->repair_contact_out_chunk = ctx->repair_contact_out_chunk0; /* Set up dedup tile output */ - fd_topo_link_t * dedup_out = &topo->links[ tile->out_link_id[ DEDUP_OUT_IDX ] ]; - ctx->dedup_out_mcache = dedup_out->mcache; - ctx->dedup_out_sync = fd_mcache_seq_laddr( ctx->dedup_out_mcache ); - ctx->dedup_out_depth = fd_mcache_depth( ctx->dedup_out_mcache ); - ctx->dedup_out_seq = fd_mcache_seq_query( ctx->dedup_out_sync ); - ctx->dedup_out_mem = topo->workspaces[ topo->objs[ dedup_out->dcache_obj_id ].wksp_id ].wksp; - ctx->dedup_out_chunk0 = fd_dcache_compact_chunk0( ctx->dedup_out_mem, dedup_out->dcache ); - ctx->dedup_out_wmark = fd_dcache_compact_wmark ( ctx->dedup_out_mem, dedup_out->dcache, dedup_out->mtu ); - ctx->dedup_out_chunk = ctx->dedup_out_chunk0; + fd_topo_link_t * verify_out = &topo->links[ tile->out_link_id[ VERIFY_OUT_IDX ] ]; + ctx->verify_out_mcache = verify_out->mcache; + ctx->verify_out_sync = fd_mcache_seq_laddr( ctx->verify_out_mcache ); + ctx->verify_out_depth = fd_mcache_depth( ctx->verify_out_mcache ); + ctx->verify_out_seq = fd_mcache_seq_query( ctx->verify_out_sync ); + ctx->verify_out_mem = topo->workspaces[ topo->objs[ verify_out->dcache_obj_id ].wksp_id ].wksp; + ctx->verify_out_chunk0 = fd_dcache_compact_chunk0( ctx->verify_out_mem, verify_out->dcache ); + ctx->verify_out_wmark = fd_dcache_compact_wmark ( ctx->verify_out_mem, verify_out->dcache, verify_out->mtu ); + ctx->verify_out_chunk = ctx->verify_out_chunk0; fd_topo_link_t * eqvoc_out = &topo->links[ tile->out_link_id[ EQVOC_OUT_IDX ] ]; ctx->eqvoc_out_mcache = eqvoc_out->mcache; diff --git a/src/app/fdctl/run/topos/fd_firedancer.c b/src/app/fdctl/run/topos/fd_firedancer.c index 7cd894e0c8..e2112cfe01 100644 --- a/src/app/fdctl/run/topos/fd_firedancer.c +++ b/src/app/fdctl/run/topos/fd_firedancer.c @@ -108,7 +108,7 @@ fd_topo_initialize( config_t * config ) { fd_topob_wksp( topo, "crds_shred" ); fd_topob_wksp( topo, "gossip_repai" ); - fd_topob_wksp( topo, "gossip_dedup" ); + fd_topob_wksp( topo, "gossip_verif" ); fd_topob_wksp( topo, "gossip_eqvoc" ); fd_topob_wksp( topo, "store_repair" ); @@ -178,7 +178,7 @@ fd_topo_initialize( config_t * config ) { /**/ fd_topob_link( topo, "replay_store", "replay_store", 128UL, sizeof(ulong) * 2, 1UL ); /* gossip_dedup could be FD_TPU_MTU, since txns are not parsed, but better to just share one size for all the ins of dedup */ - /**/ fd_topob_link( topo, "gossip_dedup", "gossip_dedup", config->tiles.verify.receive_buffer_size, FD_TPU_DCACHE_MTU, 1UL ); + /**/ fd_topob_link( topo, "gossip_verif", "gossip_verif", config->tiles.verify.receive_buffer_size, FD_TPU_DCACHE_MTU, 1UL ); /**/ fd_topob_link( topo, "gossip_eqvoc", "gossip_eqvoc", 128UL, FD_TPU_MTU, 1UL ); /**/ fd_topob_link( topo, "crds_shred", "crds_shred", 128UL, 8UL + 40200UL * 38UL, 1UL ); @@ -343,7 +343,7 @@ fd_topo_initialize( config_t * config ) { FOR(verify_tile_cnt) for( ulong j=0UL; j