Skip to content

Commit

Permalink
flamenco: back input regions and bincode valloc with spad
Browse files Browse the repository at this point in the history 
Also back miscellaneous transaction execution scoped dynamic
allocations with spad.
  • Loading branch information
@yufeng-jump
yufeng-jump committed Dec 17, 2024
1 parent 2ae4921 commit 4715366
Show file tree
Hide file tree
Showing 28 changed files with 477 additions and 194 deletions.
9 changes: 5 additions & 4 deletions src/app/fdctl/run/tiles/fd_replay.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,8 @@ scratch_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) {
l = FD_LAYOUT_APPEND( l, FD_BMTREE_COMMIT_ALIGN, FD_BMTREE_COMMIT_FOOTPRINT(0) );
}
l = FD_LAYOUT_APPEND( l, FD_SCRATCH_ALIGN_DEFAULT, tile->replay.tpool_thread_count * TPOOL_WORKER_MEM_SZ );
l = FD_LAYOUT_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * fd_spad_footprint( MAX_TX_ACCOUNT_LOCKS * fd_ulong_align_up( FD_ACC_TOT_SZ_MAX, FD_ACCOUNT_REC_ALIGN ) ) );
ulong thread_spad_size = fd_spad_footprint( FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_DEFAULT );
l = FD_LAYOUT_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * fd_ulong_align_up( thread_spad_size, fd_spad_align() ) );
l = FD_LAYOUT_APPEND( l, fd_scratch_smem_align(), fd_scratch_smem_footprint( SCRATCH_MAX ) );
l = FD_LAYOUT_APPEND( l, fd_scratch_fmem_align(), fd_scratch_fmem_footprint( SCRATCH_DEPTH ) );
l = FD_LAYOUT_FINI ( l, scratch_align() );
Expand Down Expand Up @@ -1723,8 +1724,8 @@ unprivileged_init( fd_topo_t * topo,
ctx->bmtree[i] = FD_SCRATCH_ALLOC_APPEND( l, FD_BMTREE_COMMIT_ALIGN, FD_BMTREE_COMMIT_FOOTPRINT(0) );
}
void * tpool_worker_mem = FD_SCRATCH_ALLOC_APPEND( l, FD_SCRATCH_ALIGN_DEFAULT, tile->replay.tpool_thread_count * TPOOL_WORKER_MEM_SZ );
ulong thread_spad_size = fd_spad_footprint( MAX_TX_ACCOUNT_LOCKS * fd_ulong_align_up( FD_ACC_TOT_SZ_MAX, FD_ACCOUNT_REC_ALIGN ) );
void * spad_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * thread_spad_size );
ulong thread_spad_size = fd_spad_footprint( FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_DEFAULT );
void * spad_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * fd_ulong_align_up( thread_spad_size, fd_spad_align() ) );
void * scratch_smem = FD_SCRATCH_ALLOC_APPEND( l, fd_scratch_smem_align(), fd_scratch_smem_footprint( SCRATCH_MAX ) );
void * scratch_fmem = FD_SCRATCH_ALLOC_APPEND( l, fd_scratch_fmem_align(), fd_scratch_fmem_footprint( SCRATCH_DEPTH ) );
ulong scratch_alloc_mem = FD_SCRATCH_ALLOC_FINI ( l, scratch_align() );
Expand Down Expand Up @@ -1950,7 +1951,7 @@ unprivileged_init( fd_topo_t * topo,
for( ulong i=0UL; i<tile->replay.tpool_thread_count; i++ ) {
fd_spad_t * spad = fd_spad_join( fd_spad_new( spad_mem_cur, thread_spad_size ) );
ctx->spads[ ctx->spad_cnt++ ] = spad;
spad_mem_cur += thread_spad_size;
spad_mem_cur += fd_ulong_align_up( thread_spad_size, fd_spad_align() );
}

/**********************************************************************/
Expand Down
2 changes: 1 addition & 1 deletion src/app/ledger/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -467,7 +467,7 @@ fd_ledger_main_setup( fd_ledger_args_t * args ) {

args->spad_cnt = fd_tpool_worker_cnt( args->tpool );
for( ulong i=0UL; i<args->spad_cnt; i++ ) {
ulong total_mem_sz = fd_spad_footprint( MAX_TX_ACCOUNT_LOCKS * fd_ulong_align_up( FD_ACC_TOT_SZ_MAX, FD_ACCOUNT_REC_ALIGN ) );
ulong total_mem_sz = FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_FUZZ;
uchar * mem = fd_wksp_alloc_laddr( args->wksp, FD_SPAD_ALIGN, total_mem_sz, 999UL );
fd_spad_t * spad = fd_spad_join( fd_spad_new( mem, total_mem_sz ) );
if( FD_UNLIKELY( !spad ) ) {
Expand Down
3 changes: 3 additions & 0 deletions src/flamenco/runtime/Local.mk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ $(call add-objs,fd_pubkey_utils,fd_flamenco)

$(call add-hdrs,fd_rent_lists.h)

$(call add-hdrs,fd_runtime_spad.h)
$(call add-objs,fd_runtime_spad,fd_flamenco)

ifdef FD_HAS_ATOMIC
$(call add-hdrs,fd_runtime.h fd_runtime_init.h fd_runtime_err.h)
$(call add-objs,fd_runtime fd_runtime_init,fd_flamenco)
Expand Down
2 changes: 1 addition & 1 deletion src/flamenco/runtime/context/fd_exec_instr_ctx.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ struct __attribute__((aligned(8UL))) fd_exec_instr_ctx {

fd_funk_txn_t * funk_txn;
fd_acc_mgr_t * acc_mgr;
fd_valloc_t valloc;
fd_valloc_t valloc; /* Scratch-backed valloc TODO: migrate to transaction spad */

/* Most instructions log the base58 program id multiple times, so it's
convenient to compute it once and reuse it. */
Expand Down
2 changes: 1 addition & 1 deletion src/flamenco/runtime/context/fd_exec_slot_ctx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ recover_clock( fd_exec_slot_ctx_t * slot_ctx ) {

/* Record timestamp */
if( vote_state_timestamp.slot != 0 || n->elem.stake != 0 ) {
fd_vote_record_timestamp_vote_with_slot(slot_ctx, &n->elem.key, vote_state_timestamp.timestamp, vote_state_timestamp.slot);
fd_vote_record_timestamp_vote_with_slot( slot_ctx, &n->elem.key, vote_state_timestamp.timestamp, vote_state_timestamp.slot, slot_ctx->valloc );
}
}

Expand Down
5 changes: 2 additions & 3 deletions src/flamenco/runtime/context/fd_exec_txn_ctx.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ struct __attribute__((aligned(8UL))) fd_exec_txn_ctx {

fd_funk_txn_t * funk_txn;
fd_acc_mgr_t * acc_mgr;
fd_valloc_t valloc;
fd_valloc_t valloc; /* Workspace-backed valloc TODO: migrate to spad */
fd_spad_t * spad; /* Sized out to handle the worst case footprint of single transaction execution. */

ulong paid_fees;
ulong compute_unit_limit; /* Compute unit limit for this transaction. */
Expand Down Expand Up @@ -126,8 +127,6 @@ struct __attribute__((aligned(8UL))) fd_exec_txn_ctx {
int exec_err;
int exec_err_kind;

fd_spad_t * spad;

/* The has_program_id flag is used to indicate if the current transaction has valid program indices or not.
It will be set in fd_executor_load_transaction_accounts similar to how program_indices is used in
load_transaction_accounts on the agave side */
Expand Down
2 changes: 2 additions & 0 deletions src/flamenco/runtime/fd_executor.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1163,6 +1163,7 @@ fd_instr_stack_pop( fd_exec_txn_ctx_t * txn_ctx,
int
fd_execute_instr( fd_exec_txn_ctx_t * txn_ctx,
fd_instr_info_t * instr ) {
FD_RUNTIME_TXN_SPAD_FRAME_BEGIN( txn_ctx->spad, txn_ctx ) {
FD_SCRATCH_SCOPE_BEGIN {
fd_exec_instr_ctx_t * parent = NULL;
if( txn_ctx->instr_stack_sz ) {
Expand Down Expand Up @@ -1245,6 +1246,7 @@ fd_execute_instr( fd_exec_txn_ctx_t * txn_ctx,

return exec_result;
} FD_SCRATCH_SCOPE_END;
} FD_RUNTIME_TXN_SPAD_FRAME_END;
}

void
Expand Down
62 changes: 41 additions & 21 deletions src/flamenco/runtime/fd_runtime.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1104,7 +1104,7 @@ fd_runtime_prepare_execute_finalize_txn( fd_exec_slot_ctx_t * slot_ctx,
fd_capture_ctx_t * capture_ctx,
fd_txn_p_t * txn,
fd_execute_txn_task_info_t * task_info ) {

FD_SPAD_FRAME_BEGIN( spad ) {
FD_SCRATCH_SCOPE_BEGIN {

int res = 0;
Expand Down Expand Up @@ -1152,6 +1152,7 @@ fd_runtime_prepare_execute_finalize_txn( fd_exec_slot_ctx_t * slot_ctx,
return res;

} FD_SCRATCH_SCOPE_END;
} FD_SPAD_FRAME_END;
}


Expand Down Expand Up @@ -1445,13 +1446,13 @@ fd_runtime_finalize_txn( fd_exec_slot_ctx_t * slot_ctx,
if( dirty_vote_acc && 0==memcmp( acc_rec->const_meta->info.owner, &fd_solana_vote_program_id, sizeof(fd_pubkey_t) ) ) {
/* lock for inserting/modifying vote accounts in slot ctx. */
fd_funk_start_write( slot_ctx->acc_mgr->funk );
fd_vote_store_account( slot_ctx, acc_rec );
FD_SCRATCH_SCOPE_BEGIN {
fd_vote_store_account( slot_ctx, acc_rec, txn_ctx->spad );
FD_SPAD_FRAME_BEGIN( txn_ctx->spad ) {
fd_vote_state_versioned_t vsv[1];
fd_bincode_decode_ctx_t decode_vsv =
{ .data = acc_rec->const_data,
.dataend = acc_rec->const_data + acc_rec->const_meta->dlen,
.valloc = fd_scratch_virtual() };
.valloc = fd_runtime_spad_virtual( txn_ctx->spad ) };

int err = fd_vote_state_versioned_decode( vsv, &decode_vsv );
if( err ) break; /* out of scratch scope */
Expand All @@ -1471,9 +1472,9 @@ fd_runtime_finalize_txn( fd_exec_slot_ctx_t * slot_ctx,
__builtin_unreachable();
}

fd_vote_record_timestamp_vote_with_slot( slot_ctx, acc_rec->pubkey, ts->timestamp, ts->slot );
}
FD_SCRATCH_SCOPE_END;
fd_valloc_t valloc = fd_runtime_spad_virtual( txn_ctx->spad );
fd_vote_record_timestamp_vote_with_slot( slot_ctx, acc_rec->pubkey, ts->timestamp, ts->slot, valloc );
} FD_SPAD_FRAME_END;
fd_funk_end_write( slot_ctx->acc_mgr->funk );
}

Expand Down Expand Up @@ -1734,7 +1735,7 @@ fd_runtime_finalize_txns_tpool( fd_exec_slot_ctx_t * slot_ctx,
results = fd_scratch_alloc( alignof(uchar), txn_cnt * sizeof(uchar) );
}

fd_borrowed_account_t * * accounts_to_save = fd_scratch_alloc( 8UL, 128UL * txn_cnt * sizeof(fd_borrowed_account_t *) );
fd_borrowed_account_t * * accounts_to_save = fd_scratch_alloc( 8UL, MAX_TX_ACCOUNT_LOCKS * txn_cnt * sizeof(fd_borrowed_account_t *) );
ulong acc_idx = 0UL;
ulong nonvote_txn_count = 0;
ulong failed_txn_count = 0;
Expand Down Expand Up @@ -1837,13 +1838,13 @@ fd_runtime_finalize_txns_tpool( fd_exec_slot_ctx_t * slot_ctx,
fd_borrowed_account_t * acc_rec = &txn_ctx->borrowed_accounts[i];

if( dirty_vote_acc && !memcmp( acc_rec->const_meta->info.owner, &fd_solana_vote_program_id, sizeof(fd_pubkey_t) ) ) {
fd_vote_store_account( slot_ctx, acc_rec );
FD_SCRATCH_SCOPE_BEGIN {
fd_vote_store_account( slot_ctx, acc_rec, txn_ctx->spad );
FD_SPAD_FRAME_BEGIN( txn_ctx->spad ) {
fd_vote_state_versioned_t vsv[1];
fd_bincode_decode_ctx_t decode_vsv =
{ .data = acc_rec->const_data,
.dataend = acc_rec->const_data + acc_rec->const_meta->dlen,
.valloc = fd_scratch_virtual() };
.valloc = fd_runtime_spad_virtual( txn_ctx->spad ) };

int err = fd_vote_state_versioned_decode( vsv, &decode_vsv );
if( err ) break; /* out of scratch scope */
Expand All @@ -1863,9 +1864,9 @@ fd_runtime_finalize_txns_tpool( fd_exec_slot_ctx_t * slot_ctx,
__builtin_unreachable();
}

fd_vote_record_timestamp_vote_with_slot( slot_ctx, acc_rec->pubkey, ts->timestamp, ts->slot );
}
FD_SCRATCH_SCOPE_END;
fd_valloc_t valloc = fd_runtime_spad_virtual( txn_ctx->spad );
fd_vote_record_timestamp_vote_with_slot( slot_ctx, acc_rec->pubkey, ts->timestamp, ts->slot, valloc );
} FD_SPAD_FRAME_END;
}

if( dirty_stake_acc && !memcmp( acc_rec->const_meta->info.owner, &fd_solana_stake_program_id, sizeof(fd_pubkey_t) ) ) {
Expand Down Expand Up @@ -2120,18 +2121,27 @@ fd_runtime_execute_txns_in_waves_tpool( fd_exec_slot_ctx_t * slot_ctx,
next_incomplete_txn_idxs = temp_incomplete_txn_idxs;
incomplete_txn_idxs_cnt = next_incomplete_txn_idxs_cnt;

// Dump txns in waves
if( dump_txn ) {
for( ulong i = 0; i < wave_task_infos_cnt; ++i ) {
fd_dump_txn_to_protobuf( wave_task_infos[i].txn_ctx, spads[0] );
}
for( ulong i=0UL; i<spad_cnt; i++ ) {
/* Borrowed accounts are allocated during prep and need to
persist till the end of finalize. This initial frame will
be holding that. */
fd_spad_push( spads[ i ] );
}

/* Assign out spads to the transaction contexts */
for( ulong i=0UL; i<wave_task_infos_cnt; i++ ) {
wave_task_infos[i].spads = spads;
}

// Dump txns in waves
if( dump_txn ) {
for( ulong i = 0; i < wave_task_infos_cnt; ++i ) {
// TODO this dumper still relies on scratch, move to spad?
/* Manual push/pop on the spad within the callee. */
fd_dump_txn_to_protobuf( wave_task_infos[i].txn_ctx, spads[0] );
}
}

res |= fd_runtime_verify_txn_signatures_tpool( wave_task_infos, wave_task_infos_cnt, tpool );
if( res != 0 ) {
FD_LOG_WARNING(("Fail signature verification"));
Expand All @@ -2147,9 +2157,19 @@ fd_runtime_execute_txns_in_waves_tpool( fd_exec_slot_ctx_t * slot_ctx,
FD_LOG_ERR(("Fail finalize"));
}

/* Resetting the spad is a O(1) operation */
for( ulong i=0UL; i<spad_cnt; i++ ) {
fd_spad_reset( spads[i] );
/* The first frame, which holds borrowed accounts, can be
pretty big, and there are additional dynamic allocations
during finalize. */
if( FD_UNLIKELY( fd_spad_verify( spads[ i ] ) ) ) {
FD_LOG_ERR(( "spad corrupted or overflown" ));
}
/* We indiscriminately pushed a frame to every spad.
So it should be safe to indiscriminately pop here. */
fd_spad_pop( spads[ i ] );
if( FD_UNLIKELY( fd_spad_frame_used( spads[ i ] )!=0 ) ) {
FD_LOG_ERR(( "stray spad frame frame_used=%lu", fd_spad_frame_used( spads[ i ] ) ));
}
}

// wave_time += fd_log_wallclock();
Expand Down
Loading

0 comments on commit 4715366

Please sign in to comment.