Skip to content

Commit

Permalink
tile, gossip: funnel incoming vote txns to verify instead of dedup
Browse files Browse the repository at this point in the history
  • Loading branch information
@ravyu-jump
ravyu-jump committed Dec 12, 2024
1 parent e6fa66e commit af5fe49
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 40 deletions.
19 changes: 6 additions & 13 deletions src/app/fdctl/run/tiles/fd_dedup.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,7 @@
checks the transaction signature field for duplicates and filters
them out. */

#define GOSSIP_IN_IDX (0UL) /* Frankendancer and Firedancer */
#define VOTER_IN_IDX (1UL) /* Firedancer only */
#define VOTER_IN_IDX (0UL) /* Firedancer only */

/* fd_dedup_in_ctx_t is a context object for each in (producer) mcache
connected to the dedup tile. */
Expand Down Expand Up @@ -162,8 +161,6 @@ after_frag( fd_dedup_ctx_t * ctx,
ulong txn_t_sz = fd_txn_parse( dcache_entry, payload_sz, txn, NULL );
if( FD_UNLIKELY( !txn_t_sz ) ) FD_LOG_ERR(( "fd_txn_parse failed for vote transactions that should have been sigverified" ));

/* Increment on GOSSIP_IN_IDX but not VOTER_IN_IDX */
FD_MCNT_INC( DEDUP, GOSSIPED_VOTES_RECEIVED, 1UL - in_idx );

/* Write payload_sz into trailer.
fd_txn_parse always returns a multiple of 2 so this sz is
Expand Down Expand Up @@ -214,21 +211,17 @@ unprivileged_init( fd_topo_t * topo,

/* Frankendancer has gossip_dedup, verify_dedup+
Firedancer has gossip_dedup, voter_dedup, verify_dedup+ */
ulong unparsed_in_cnt = 1;
if( FD_UNLIKELY( tile->in_cnt<2UL ) ) {
FD_LOG_ERR(( "dedup tile needs at least two input links, got %lu", tile->in_cnt ));
} else if( FD_UNLIKELY( strcmp( topo->links[ tile->in_link_id[ GOSSIP_IN_IDX ] ].name, "gossip_dedup" ) ) ) {
/* We have one link for gossip messages... */
FD_LOG_ERR(( "dedup tile has unexpected input links %lu %lu %s",
tile->in_cnt, GOSSIP_IN_IDX, topo->links[ tile->in_link_id[ GOSSIP_IN_IDX ] ].name ));
ulong unparsed_in_cnt = 0;
if( FD_UNLIKELY( tile->in_cnt<1UL ) ) {
FD_LOG_ERR(( "dedup tile needs at least one input link, got %lu", tile->in_cnt ));
} else {
/* ...followed by a voter_dedup link if it were the Firedancer topology */
ulong voter_dedup_idx = fd_topo_find_tile_in_link( topo, tile, "voter_dedup", 0 );
if( voter_dedup_idx!=ULONG_MAX ) {
FD_TEST( voter_dedup_idx == VOTER_IN_IDX );
unparsed_in_cnt = 2;
} else {
unparsed_in_cnt = 1;
} else {
unparsed_in_cnt = 0;
}

/* ...followed by a sequence of verify_dedup links */
Expand Down
54 changes: 31 additions & 23 deletions src/app/fdctl/run/tiles/fd_gossip.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
#define NET_OUT_IDX 0
#define SHRED_OUT_IDX 1
#define REPAIR_OUT_IDX 2
#define DEDUP_OUT_IDX 3
#define VERIFY_OUT_IDX 3
#define SIGN_OUT_IDX 4
#define VOTER_OUT_IDX 5
#define REPLAY_OUT_IDX 6
Expand Down Expand Up @@ -104,15 +104,15 @@ struct fd_gossip_tile_ctx {
ulong voter_contact_out_wmark;
ulong voter_contact_out_chunk;

fd_frag_meta_t * dedup_out_mcache;
ulong * dedup_out_sync;
ulong dedup_out_depth;
ulong dedup_out_seq;
fd_frag_meta_t * verify_out_mcache;
ulong * verify_out_sync;
ulong verify_out_depth;
ulong verify_out_seq;

fd_wksp_t * dedup_out_mem;
ulong dedup_out_chunk0;
ulong dedup_out_wmark;
ulong dedup_out_chunk;
fd_wksp_t * verify_out_mem;
ulong verify_out_chunk0;
ulong verify_out_wmark;
ulong verify_out_chunk;

fd_frag_meta_t * eqvoc_out_mcache;
ulong * eqvoc_out_sync;
Expand Down Expand Up @@ -317,15 +317,23 @@ gossip_deliver_fun( fd_crds_data_t * data, void * arg ) {
return;
}

uchar * vote_txn_msg = fd_chunk_to_laddr( ctx->dedup_out_mem, ctx->dedup_out_chunk );
uchar * vote_txn_msg = fd_chunk_to_laddr( ctx->verify_out_mem, ctx->verify_out_chunk );
ulong vote_txn_sz = gossip_vote->txn.raw_sz;
memcpy( vote_txn_msg, gossip_vote->txn.raw, vote_txn_sz );

/* DEFENSE-IN-DEPTH: While the underlying txn message is technically verifed
at this point (by virtue of being a substring of the full CRDS message we
verify in fd_gossip_recv_crds_value), we still send it to the verify tile(s)
to avoid having the single point of verification failure lie in the gossip tile.
TODO: monitor gossip vote traffic to determine potential impact this might
have on verify traffic */

ulong sig = 1UL;
fd_mcache_publish( ctx->dedup_out_mcache, ctx->dedup_out_depth, ctx->dedup_out_seq, sig, ctx->dedup_out_chunk,
fd_mcache_publish( ctx->verify_out_mcache, ctx->verify_out_depth, ctx->verify_out_seq, sig, ctx->verify_out_chunk,
vote_txn_sz, 0UL, 0, 0 );
ctx->dedup_out_seq = fd_seq_inc( ctx->dedup_out_seq, 1UL );
ctx->dedup_out_chunk = fd_dcache_compact_next( ctx->dedup_out_chunk, vote_txn_sz, ctx->dedup_out_chunk0, ctx->dedup_out_wmark );
ctx->verify_out_seq = fd_seq_inc( ctx->verify_out_seq, 1UL );
ctx->verify_out_chunk = fd_dcache_compact_next( ctx->verify_out_chunk, vote_txn_sz, ctx->verify_out_chunk0, ctx->verify_out_wmark );

} else if( fd_crds_data_is_contact_info_v1( data ) ) {
fd_gossip_contact_info_v1_t const * contact_info = &data->inner.contact_info_v1;
Expand Down Expand Up @@ -797,7 +805,7 @@ unprivileged_init( fd_topo_t * topo,
strcmp( topo->links[ tile->out_link_id[ NET_OUT_IDX ] ].name, "gossip_net" ) ||
strcmp( topo->links[ tile->out_link_id[ SHRED_OUT_IDX ] ].name, "crds_shred" ) ||
strcmp( topo->links[ tile->out_link_id[ REPAIR_OUT_IDX ] ].name, "gossip_repai" ) ||
strcmp( topo->links[ tile->out_link_id[ DEDUP_OUT_IDX ] ].name, "gossip_dedup" ) ||
strcmp( topo->links[ tile->out_link_id[ VERIFY_OUT_IDX ] ].name, "gossip_verif" ) ||
strcmp( topo->links[ tile->out_link_id[ SIGN_OUT_IDX ] ].name, "gossip_sign" ) ||
strcmp( topo->links[ tile->out_link_id[ VOTER_OUT_IDX ] ].name, "gossip_voter" ) ||
strcmp( topo->links[ tile->out_link_id[ REPLAY_OUT_IDX ] ].name, "gossip_repla" ) ||
Expand Down Expand Up @@ -947,15 +955,15 @@ unprivileged_init( fd_topo_t * topo,
ctx->repair_contact_out_chunk = ctx->repair_contact_out_chunk0;

/* Set up dedup tile output */
fd_topo_link_t * dedup_out = &topo->links[ tile->out_link_id[ DEDUP_OUT_IDX ] ];
ctx->dedup_out_mcache = dedup_out->mcache;
ctx->dedup_out_sync = fd_mcache_seq_laddr( ctx->dedup_out_mcache );
ctx->dedup_out_depth = fd_mcache_depth( ctx->dedup_out_mcache );
ctx->dedup_out_seq = fd_mcache_seq_query( ctx->dedup_out_sync );
ctx->dedup_out_mem = topo->workspaces[ topo->objs[ dedup_out->dcache_obj_id ].wksp_id ].wksp;
ctx->dedup_out_chunk0 = fd_dcache_compact_chunk0( ctx->dedup_out_mem, dedup_out->dcache );
ctx->dedup_out_wmark = fd_dcache_compact_wmark ( ctx->dedup_out_mem, dedup_out->dcache, dedup_out->mtu );
ctx->dedup_out_chunk = ctx->dedup_out_chunk0;
fd_topo_link_t * verify_out = &topo->links[ tile->out_link_id[ VERIFY_OUT_IDX ] ];
ctx->verify_out_mcache = verify_out->mcache;
ctx->verify_out_sync = fd_mcache_seq_laddr( ctx->verify_out_mcache );
ctx->verify_out_depth = fd_mcache_depth( ctx->verify_out_mcache );
ctx->verify_out_seq = fd_mcache_seq_query( ctx->verify_out_sync );
ctx->verify_out_mem = topo->workspaces[ topo->objs[ verify_out->dcache_obj_id ].wksp_id ].wksp;
ctx->verify_out_chunk0 = fd_dcache_compact_chunk0( ctx->verify_out_mem, verify_out->dcache );
ctx->verify_out_wmark = fd_dcache_compact_wmark ( ctx->verify_out_mem, verify_out->dcache, verify_out->mtu );
ctx->verify_out_chunk = ctx->verify_out_chunk0;

fd_topo_link_t * eqvoc_out = &topo->links[ tile->out_link_id[ EQVOC_OUT_IDX ] ];
ctx->eqvoc_out_mcache = eqvoc_out->mcache;
Expand Down
8 changes: 4 additions & 4 deletions src/app/fdctl/run/topos/fd_firedancer.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ fd_topo_initialize( config_t * config ) {

fd_topob_wksp( topo, "crds_shred" );
fd_topob_wksp( topo, "gossip_repai" );
fd_topob_wksp( topo, "gossip_dedup" );
fd_topob_wksp( topo, "gossip_verif" );
fd_topob_wksp( topo, "gossip_eqvoc" );

fd_topob_wksp( topo, "store_repair" );
Expand Down Expand Up @@ -178,7 +178,7 @@ fd_topo_initialize( config_t * config ) {
/**/ fd_topob_link( topo, "replay_store", "replay_store", 128UL, sizeof(ulong) * 2, 1UL );
/* gossip_dedup could be FD_TPU_MTU, since txns are not parsed, but better to just share one size for all the ins of dedup */
/**/ fd_topob_link( topo, "gossip_dedup", "gossip_dedup", config->tiles.verify.receive_buffer_size, FD_TPU_DCACHE_MTU, 1UL );
/**/ fd_topob_link( topo, "gossip_verif", "gossip_verif", config->tiles.verify.receive_buffer_size, FD_TPU_DCACHE_MTU, 1UL );
/**/ fd_topob_link( topo, "gossip_eqvoc", "gossip_eqvoc", 128UL, FD_TPU_MTU, 1UL );
/**/ fd_topob_link( topo, "crds_shred", "crds_shred", 128UL, 8UL + 40200UL * 38UL, 1UL );
Expand Down Expand Up @@ -343,7 +343,7 @@ fd_topo_initialize( config_t * config ) {
FOR(verify_tile_cnt) for( ulong j=0UL; j<quic_tile_cnt; j++ )
fd_topob_tile_in( topo, "verify", i, "metric_in", "quic_verify", j, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers, verify tiles may be overrun */
FOR(verify_tile_cnt) fd_topob_tile_out( topo, "verify", i, "verify_dedup", i );
/**/ fd_topob_tile_in( topo, "dedup", 0UL, "metric_in", "gossip_dedup", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */
FOR(verify_tile_cnt) fd_topob_tile_in( topo, "verify", i, "metric_in", "gossip_verif", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */
/**/ fd_topob_tile_in( topo, "dedup", 0UL, "metric_in", "voter_dedup", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
FOR(verify_tile_cnt) fd_topob_tile_in( topo, "dedup", 0UL, "metric_in", "verify_dedup", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_out( topo, "dedup", 0UL, "dedup_pack", 0UL );
Expand Down Expand Up @@ -387,7 +387,7 @@ fd_topo_initialize( config_t * config ) {
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_net", 0UL );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "crds_shred", 0UL );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_repai", 0UL );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_dedup", 0UL );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_verif", 0UL );
/**/ fd_topob_tile_in( topo, "sign", 0UL, "metric_in", "gossip_sign", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_sign", 0UL );
/**/ fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "voter_gossip", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
Expand Down

0 comments on commit af5fe49

Please sign in to comment.