Skip to content

Commit

Permalink
tile, gossip: rewire dedup out to verify
Browse files Browse the repository at this point in the history
  • Loading branch information
@ravyu-jump
ravyu-jump committed Dec 13, 2024
1 parent e6fa66e commit e79754b
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 59 deletions.
51 changes: 19 additions & 32 deletions src/app/fdctl/run/tiles/fd_dedup.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,6 @@
checks the transaction signature field for duplicates and filters
them out. */

#define GOSSIP_IN_IDX (0UL) /* Frankendancer and Firedancer */
#define VOTER_IN_IDX (1UL) /* Firedancer only */

/* fd_dedup_in_ctx_t is a context object for each in (producer) mcache
connected to the dedup tile. */

Expand All @@ -41,6 +38,7 @@ typedef struct {
/* The first unparsed_in_cnt in links do not have the parsed fd_txn_t
in the payload trailer. */
ulong unparsed_in_cnt;
ulong gossip_in_idx;
fd_dedup_in_ctx_t in[ 64UL ];

fd_wksp_t * out_mem;
Expand Down Expand Up @@ -141,7 +139,7 @@ after_frag( fd_dedup_ctx_t * ctx,
ulong txn_off = fd_ulong_align_up( payload_sz, 2UL );
fd_txn_t * txn = (fd_txn_t *)(dcache_entry + txn_off);

if ( FD_UNLIKELY( in_idx < ctx->unparsed_in_cnt ) ) {
if( FD_UNLIKELY( in_idx < ctx->unparsed_in_cnt ) ) {
/* Transactions coming in from these links are not parsed.
We'll need to parse it so it's ready for downstream consumers.
Expand All @@ -163,7 +161,7 @@ after_frag( fd_dedup_ctx_t * ctx,
if( FD_UNLIKELY( !txn_t_sz ) ) FD_LOG_ERR(( "fd_txn_parse failed for vote transactions that should have been sigverified" ));

/* Increment on GOSSIP_IN_IDX but not VOTER_IN_IDX */
FD_MCNT_INC( DEDUP, GOSSIPED_VOTES_RECEIVED, 1UL - in_idx );
FD_MCNT_INC( DEDUP, GOSSIPED_VOTES_RECEIVED, ( in_idx == ctx->gossip_in_idx ) );

/* Write payload_sz into trailer.
fd_txn_parse always returns a multiple of 2 so this sz is
Expand Down Expand Up @@ -212,32 +210,8 @@ unprivileged_init( fd_topo_t * topo,
fd_topo_tile_t * tile ) {
void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );

/* Frankendancer has gossip_dedup, verify_dedup+
Firedancer has gossip_dedup, voter_dedup, verify_dedup+ */
ulong unparsed_in_cnt = 1;
if( FD_UNLIKELY( tile->in_cnt<2UL ) ) {
FD_LOG_ERR(( "dedup tile needs at least two input links, got %lu", tile->in_cnt ));
} else if( FD_UNLIKELY( strcmp( topo->links[ tile->in_link_id[ GOSSIP_IN_IDX ] ].name, "gossip_dedup" ) ) ) {
/* We have one link for gossip messages... */
FD_LOG_ERR(( "dedup tile has unexpected input links %lu %lu %s",
tile->in_cnt, GOSSIP_IN_IDX, topo->links[ tile->in_link_id[ GOSSIP_IN_IDX ] ].name ));
} else {
/* ...followed by a voter_dedup link if it were the Firedancer topology */
ulong voter_dedup_idx = fd_topo_find_tile_in_link( topo, tile, "voter_dedup", 0 );
if( voter_dedup_idx!=ULONG_MAX ) {
FD_TEST( voter_dedup_idx == VOTER_IN_IDX );
unparsed_in_cnt = 2;
} else {
unparsed_in_cnt = 1;
}

/* ...followed by a sequence of verify_dedup links */
for( ulong i=unparsed_in_cnt; i<tile->in_cnt; i++ ) {
if( FD_UNLIKELY( strcmp( topo->links[ tile->in_link_id[ i ] ].name, "verify_dedup" ) ) ) {
FD_LOG_ERR(( "dedup tile has unexpected input links %lu %lu %s",
tile->in_cnt, i, topo->links[ tile->in_link_id[ i ] ].name ));
}
}
if( FD_UNLIKELY( tile->in_cnt == 0UL ) ) {
FD_LOG_ERR(( "dedup tile needs at least one input link, got zero" ));
}

FD_SCRATCH_ALLOC_INIT( l, scratch );
Expand All @@ -252,15 +226,28 @@ unprivileged_init( fd_topo_t * topo,
ctx->tcache_map = fd_tcache_map_laddr ( tcache );

FD_TEST( tile->in_cnt<=sizeof( ctx->in )/sizeof( ctx->in[ 0 ] ) );
ctx->unparsed_in_cnt = unparsed_in_cnt;
ulong parsed_in_start_idx = ULONG_MAX;
ctx->gossip_in_idx = ULONG_MAX;
for( ulong i=0; i<tile->in_cnt; i++ ) {
fd_topo_link_t * link = &topo->links[ tile->in_link_id[ i ] ];
fd_topo_wksp_t * link_wksp = &topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ];

ctx->in[i].mem = link_wksp->wksp;
ctx->in[i].chunk0 = fd_dcache_compact_chunk0( ctx->in[i].mem, link->dcache );
ctx->in[i].wmark = fd_dcache_compact_wmark ( ctx->in[i].mem, link->dcache, link->mtu );

if( FD_UNLIKELY( !strcmp( link->name, "gossip_dedup" ) ) ) {
if( FD_UNLIKELY( i > parsed_in_start_idx ) ) FD_LOG_ERR(( "unparsed links must go before parsed ones" ));
ctx->gossip_in_idx = i;
} else if( FD_UNLIKELY( !strcmp( link->name, "voter_dedup" ) ) ) {
if( FD_UNLIKELY( i > parsed_in_start_idx ) ) FD_LOG_ERR(( "unparsed links must go before parsed ones" ));
} else if( FD_LIKELY( !strcmp( link->name, "verify_dedup" ) ) ) {
if( FD_UNLIKELY( parsed_in_start_idx == ULONG_MAX ) ) parsed_in_start_idx = i;
} else {
FD_LOG_ERR(( "dedup tile has unexpected input link name %s", link->name ));
}
}
ctx->unparsed_in_cnt = parsed_in_start_idx;

ctx->out_mem = topo->workspaces[ topo->objs[ topo->links[ tile->out_link_id[ 0 ] ].dcache_obj_id ].wksp_id ].wksp;
ctx->out_chunk0 = fd_dcache_compact_chunk0( ctx->out_mem, topo->links[ tile->out_link_id[ 0 ] ].dcache );
Expand Down
54 changes: 31 additions & 23 deletions src/app/fdctl/run/tiles/fd_gossip.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
#define NET_OUT_IDX 0
#define SHRED_OUT_IDX 1
#define REPAIR_OUT_IDX 2
#define DEDUP_OUT_IDX 3
#define VERIFY_OUT_IDX 3
#define SIGN_OUT_IDX 4
#define VOTER_OUT_IDX 5
#define REPLAY_OUT_IDX 6
Expand Down Expand Up @@ -104,15 +104,15 @@ struct fd_gossip_tile_ctx {
ulong voter_contact_out_wmark;
ulong voter_contact_out_chunk;

fd_frag_meta_t * dedup_out_mcache;
ulong * dedup_out_sync;
ulong dedup_out_depth;
ulong dedup_out_seq;
fd_frag_meta_t * verify_out_mcache;
ulong * verify_out_sync;
ulong verify_out_depth;
ulong verify_out_seq;

fd_wksp_t * dedup_out_mem;
ulong dedup_out_chunk0;
ulong dedup_out_wmark;
ulong dedup_out_chunk;
fd_wksp_t * verify_out_mem;
ulong verify_out_chunk0;
ulong verify_out_wmark;
ulong verify_out_chunk;

fd_frag_meta_t * eqvoc_out_mcache;
ulong * eqvoc_out_sync;
Expand Down Expand Up @@ -317,15 +317,23 @@ gossip_deliver_fun( fd_crds_data_t * data, void * arg ) {
return;
}

uchar * vote_txn_msg = fd_chunk_to_laddr( ctx->dedup_out_mem, ctx->dedup_out_chunk );
uchar * vote_txn_msg = fd_chunk_to_laddr( ctx->verify_out_mem, ctx->verify_out_chunk );
ulong vote_txn_sz = gossip_vote->txn.raw_sz;
memcpy( vote_txn_msg, gossip_vote->txn.raw, vote_txn_sz );

/* DEFENSE-IN-DEPTH: While the underlying txn message is technically verifed
at this point (by virtue of being a substring of the full CRDS message we
verify in fd_gossip_recv_crds_value), we still send it to the verify tile(s)
to avoid having the single point of verification failure lie in the gossip tile.
TODO: monitor gossip vote traffic to determine potential impact this might
have on verify traffic */

ulong sig = 1UL;
fd_mcache_publish( ctx->dedup_out_mcache, ctx->dedup_out_depth, ctx->dedup_out_seq, sig, ctx->dedup_out_chunk,
fd_mcache_publish( ctx->verify_out_mcache, ctx->verify_out_depth, ctx->verify_out_seq, sig, ctx->verify_out_chunk,
vote_txn_sz, 0UL, 0, 0 );
ctx->dedup_out_seq = fd_seq_inc( ctx->dedup_out_seq, 1UL );
ctx->dedup_out_chunk = fd_dcache_compact_next( ctx->dedup_out_chunk, vote_txn_sz, ctx->dedup_out_chunk0, ctx->dedup_out_wmark );
ctx->verify_out_seq = fd_seq_inc( ctx->verify_out_seq, 1UL );
ctx->verify_out_chunk = fd_dcache_compact_next( ctx->verify_out_chunk, vote_txn_sz, ctx->verify_out_chunk0, ctx->verify_out_wmark );

} else if( fd_crds_data_is_contact_info_v1( data ) ) {
fd_gossip_contact_info_v1_t const * contact_info = &data->inner.contact_info_v1;
Expand Down Expand Up @@ -797,7 +805,7 @@ unprivileged_init( fd_topo_t * topo,
strcmp( topo->links[ tile->out_link_id[ NET_OUT_IDX ] ].name, "gossip_net" ) ||
strcmp( topo->links[ tile->out_link_id[ SHRED_OUT_IDX ] ].name, "crds_shred" ) ||
strcmp( topo->links[ tile->out_link_id[ REPAIR_OUT_IDX ] ].name, "gossip_repai" ) ||
strcmp( topo->links[ tile->out_link_id[ DEDUP_OUT_IDX ] ].name, "gossip_dedup" ) ||
strcmp( topo->links[ tile->out_link_id[ VERIFY_OUT_IDX ] ].name, "gossip_verif" ) ||
strcmp( topo->links[ tile->out_link_id[ SIGN_OUT_IDX ] ].name, "gossip_sign" ) ||
strcmp( topo->links[ tile->out_link_id[ VOTER_OUT_IDX ] ].name, "gossip_voter" ) ||
strcmp( topo->links[ tile->out_link_id[ REPLAY_OUT_IDX ] ].name, "gossip_repla" ) ||
Expand Down Expand Up @@ -947,15 +955,15 @@ unprivileged_init( fd_topo_t * topo,
ctx->repair_contact_out_chunk = ctx->repair_contact_out_chunk0;

/* Set up dedup tile output */
fd_topo_link_t * dedup_out = &topo->links[ tile->out_link_id[ DEDUP_OUT_IDX ] ];
ctx->dedup_out_mcache = dedup_out->mcache;
ctx->dedup_out_sync = fd_mcache_seq_laddr( ctx->dedup_out_mcache );
ctx->dedup_out_depth = fd_mcache_depth( ctx->dedup_out_mcache );
ctx->dedup_out_seq = fd_mcache_seq_query( ctx->dedup_out_sync );
ctx->dedup_out_mem = topo->workspaces[ topo->objs[ dedup_out->dcache_obj_id ].wksp_id ].wksp;
ctx->dedup_out_chunk0 = fd_dcache_compact_chunk0( ctx->dedup_out_mem, dedup_out->dcache );
ctx->dedup_out_wmark = fd_dcache_compact_wmark ( ctx->dedup_out_mem, dedup_out->dcache, dedup_out->mtu );
ctx->dedup_out_chunk = ctx->dedup_out_chunk0;
fd_topo_link_t * verify_out = &topo->links[ tile->out_link_id[ VERIFY_OUT_IDX ] ];
ctx->verify_out_mcache = verify_out->mcache;
ctx->verify_out_sync = fd_mcache_seq_laddr( ctx->verify_out_mcache );
ctx->verify_out_depth = fd_mcache_depth( ctx->verify_out_mcache );
ctx->verify_out_seq = fd_mcache_seq_query( ctx->verify_out_sync );
ctx->verify_out_mem = topo->workspaces[ topo->objs[ verify_out->dcache_obj_id ].wksp_id ].wksp;
ctx->verify_out_chunk0 = fd_dcache_compact_chunk0( ctx->verify_out_mem, verify_out->dcache );
ctx->verify_out_wmark = fd_dcache_compact_wmark ( ctx->verify_out_mem, verify_out->dcache, verify_out->mtu );
ctx->verify_out_chunk = ctx->verify_out_chunk0;

fd_topo_link_t * eqvoc_out = &topo->links[ tile->out_link_id[ EQVOC_OUT_IDX ] ];
ctx->eqvoc_out_mcache = eqvoc_out->mcache;
Expand Down
8 changes: 4 additions & 4 deletions src/app/fdctl/run/topos/fd_firedancer.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ fd_topo_initialize( config_t * config ) {

fd_topob_wksp( topo, "crds_shred" );
fd_topob_wksp( topo, "gossip_repai" );
fd_topob_wksp( topo, "gossip_dedup" );
fd_topob_wksp( topo, "gossip_verif" );
fd_topob_wksp( topo, "gossip_eqvoc" );

fd_topob_wksp( topo, "store_repair" );
Expand Down Expand Up @@ -178,7 +178,7 @@ fd_topo_initialize( config_t * config ) {
/**/ fd_topob_link( topo, "replay_store", "replay_store", 128UL, sizeof(ulong) * 2, 1UL );
/* gossip_dedup could be FD_TPU_MTU, since txns are not parsed, but better to just share one size for all the ins of dedup */
/**/ fd_topob_link( topo, "gossip_dedup", "gossip_dedup", config->tiles.verify.receive_buffer_size, FD_TPU_DCACHE_MTU, 1UL );
/**/ fd_topob_link( topo, "gossip_verif", "gossip_verif", config->tiles.verify.receive_buffer_size, FD_TPU_DCACHE_MTU, 1UL );
/**/ fd_topob_link( topo, "gossip_eqvoc", "gossip_eqvoc", 128UL, FD_TPU_MTU, 1UL );
/**/ fd_topob_link( topo, "crds_shred", "crds_shred", 128UL, 8UL + 40200UL * 38UL, 1UL );
Expand Down Expand Up @@ -343,7 +343,7 @@ fd_topo_initialize( config_t * config ) {
FOR(verify_tile_cnt) for( ulong j=0UL; j<quic_tile_cnt; j++ )
fd_topob_tile_in( topo, "verify", i, "metric_in", "quic_verify", j, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers, verify tiles may be overrun */
FOR(verify_tile_cnt) fd_topob_tile_out( topo, "verify", i, "verify_dedup", i );
/**/ fd_topob_tile_in( topo, "dedup", 0UL, "metric_in", "gossip_dedup", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */
FOR(verify_tile_cnt) fd_topob_tile_in( topo, "verify", i, "metric_in", "gossip_verif", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* No reliable consumers of networking fragments, may be dropped or overrun */
/**/ fd_topob_tile_in( topo, "dedup", 0UL, "metric_in", "voter_dedup", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
FOR(verify_tile_cnt) fd_topob_tile_in( topo, "dedup", 0UL, "metric_in", "verify_dedup", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_out( topo, "dedup", 0UL, "dedup_pack", 0UL );
Expand Down Expand Up @@ -387,7 +387,7 @@ fd_topo_initialize( config_t * config ) {
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_net", 0UL );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "crds_shred", 0UL );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_repai", 0UL );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_dedup", 0UL );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_verif", 0UL );
/**/ fd_topob_tile_in( topo, "sign", 0UL, "metric_in", "gossip_sign", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
/**/ fd_topob_tile_out( topo, "gossip", 0UL, "gossip_sign", 0UL );
/**/ fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "voter_gossip", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
Expand Down

0 comments on commit e79754b

Please sign in to comment.