Skip to content

Commit

Permalink
flamenco: back input regions and bincode valloc with spad
Browse files Browse the repository at this point in the history 
Also back miscellaneous transaction execution scoped dynamic
allocations with spad.

The valloc handle in txn_ctx is gone.
  • Loading branch information
@yufeng-jump
yufeng-jump committed Dec 18, 2024
1 parent 08e3339 commit 2a82177
Show file tree
Hide file tree
Showing 32 changed files with 482 additions and 262 deletions.
9 changes: 5 additions & 4 deletions src/app/fdctl/run/tiles/fd_replay.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,8 @@ scratch_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) {
l = FD_LAYOUT_APPEND( l, FD_BMTREE_COMMIT_ALIGN, FD_BMTREE_COMMIT_FOOTPRINT(0) );
}
l = FD_LAYOUT_APPEND( l, FD_SCRATCH_ALIGN_DEFAULT, tile->replay.tpool_thread_count * TPOOL_WORKER_MEM_SZ );
l = FD_LAYOUT_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * fd_spad_footprint( MAX_TX_ACCOUNT_LOCKS * fd_ulong_align_up( FD_ACC_TOT_SZ_MAX, FD_ACCOUNT_REC_ALIGN ) ) );
ulong thread_spad_size = fd_spad_footprint( FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_DEFAULT );
l = FD_LAYOUT_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * fd_ulong_align_up( thread_spad_size, fd_spad_align() ) );
l = FD_LAYOUT_APPEND( l, fd_scratch_smem_align(), fd_scratch_smem_footprint( SCRATCH_MAX ) );
l = FD_LAYOUT_APPEND( l, fd_scratch_fmem_align(), fd_scratch_fmem_footprint( SCRATCH_DEPTH ) );
l = FD_LAYOUT_FINI ( l, scratch_align() );
Expand Down Expand Up @@ -1723,8 +1724,8 @@ unprivileged_init( fd_topo_t * topo,
ctx->bmtree[i] = FD_SCRATCH_ALLOC_APPEND( l, FD_BMTREE_COMMIT_ALIGN, FD_BMTREE_COMMIT_FOOTPRINT(0) );
}
void * tpool_worker_mem = FD_SCRATCH_ALLOC_APPEND( l, FD_SCRATCH_ALIGN_DEFAULT, tile->replay.tpool_thread_count * TPOOL_WORKER_MEM_SZ );
ulong thread_spad_size = fd_spad_footprint( MAX_TX_ACCOUNT_LOCKS * fd_ulong_align_up( FD_ACC_TOT_SZ_MAX, FD_ACCOUNT_REC_ALIGN ) );
void * spad_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * thread_spad_size );
ulong thread_spad_size = fd_spad_footprint( FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_DEFAULT );
void * spad_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * fd_ulong_align_up( thread_spad_size, fd_spad_align() ) );
void * scratch_smem = FD_SCRATCH_ALLOC_APPEND( l, fd_scratch_smem_align(), fd_scratch_smem_footprint( SCRATCH_MAX ) );
void * scratch_fmem = FD_SCRATCH_ALLOC_APPEND( l, fd_scratch_fmem_align(), fd_scratch_fmem_footprint( SCRATCH_DEPTH ) );
ulong scratch_alloc_mem = FD_SCRATCH_ALLOC_FINI ( l, scratch_align() );
Expand Down Expand Up @@ -1950,7 +1951,7 @@ unprivileged_init( fd_topo_t * topo,
for( ulong i=0UL; i<tile->replay.tpool_thread_count; i++ ) {
fd_spad_t * spad = fd_spad_join( fd_spad_new( spad_mem_cur, thread_spad_size ) );
ctx->spads[ ctx->spad_cnt++ ] = spad;
spad_mem_cur += thread_spad_size;
spad_mem_cur += fd_ulong_align_up( thread_spad_size, fd_spad_align() );
}

/**********************************************************************/
Expand Down
2 changes: 1 addition & 1 deletion src/app/ledger/main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -467,7 +467,7 @@ fd_ledger_main_setup( fd_ledger_args_t * args ) {

args->spad_cnt = fd_tpool_worker_cnt( args->tpool );
for( ulong i=0UL; i<args->spad_cnt; i++ ) {
ulong total_mem_sz = fd_spad_footprint( MAX_TX_ACCOUNT_LOCKS * fd_ulong_align_up( FD_ACC_TOT_SZ_MAX, FD_ACCOUNT_REC_ALIGN ) );
ulong total_mem_sz = FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_FUZZ;
uchar * mem = fd_wksp_alloc_laddr( args->wksp, FD_SPAD_ALIGN, total_mem_sz, 999UL );
fd_spad_t * spad = fd_spad_join( fd_spad_new( mem, total_mem_sz ) );
if( FD_UNLIKELY( !spad ) ) {
Expand Down
6 changes: 5 additions & 1 deletion src/flamenco/runtime/context/fd_exec_instr_ctx.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,11 @@ struct __attribute__((aligned(8UL))) fd_exec_instr_ctx {

fd_funk_txn_t * funk_txn;
fd_acc_mgr_t * acc_mgr;
fd_valloc_t valloc;
/* This is for instruction-level test harnesses, and really just
test_vm_util.c at the moment.
Offline and live instruction execution should use the spad in
txn_ctx. */
fd_valloc_t test_only_valloc;

/* Most instructions log the base58 program id multiple times, so it's
convenient to compute it once and reuse it. */
Expand Down
2 changes: 1 addition & 1 deletion src/flamenco/runtime/context/fd_exec_slot_ctx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ recover_clock( fd_exec_slot_ctx_t * slot_ctx ) {

/* Record timestamp */
if( vote_state_timestamp.slot != 0 || n->elem.stake != 0 ) {
fd_vote_record_timestamp_vote_with_slot(slot_ctx, &n->elem.key, vote_state_timestamp.timestamp, vote_state_timestamp.slot);
fd_vote_record_timestamp_vote_with_slot( slot_ctx, &n->elem.key, vote_state_timestamp.timestamp, vote_state_timestamp.slot, slot_ctx->valloc );
}
}

Expand Down
1 change: 0 additions & 1 deletion src/flamenco/runtime/context/fd_exec_txn_ctx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,6 @@ fd_exec_txn_ctx_from_exec_slot_ctx( fd_exec_slot_ctx_t * slot_ctx,
fd_exec_txn_ctx_t * txn_ctx ) {
txn_ctx->slot_ctx = slot_ctx;
txn_ctx->epoch_ctx = slot_ctx->epoch_ctx;
txn_ctx->valloc = slot_ctx->valloc;
txn_ctx->funk_txn = NULL;
txn_ctx->acc_mgr = slot_ctx->acc_mgr;
}
Expand Down
4 changes: 1 addition & 3 deletions src/flamenco/runtime/context/fd_exec_txn_ctx.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ struct __attribute__((aligned(8UL))) fd_exec_txn_ctx {

fd_funk_txn_t * funk_txn;
fd_acc_mgr_t * acc_mgr;
fd_valloc_t valloc;
fd_spad_t * spad; /* Sized out to handle the worst case footprint of single transaction execution. */

ulong paid_fees;
ulong compute_unit_limit; /* Compute unit limit for this transaction. */
Expand Down Expand Up @@ -126,8 +126,6 @@ struct __attribute__((aligned(8UL))) fd_exec_txn_ctx {
int exec_err;
int exec_err_kind;

fd_spad_t * spad;

/* The has_program_id flag is used to indicate if the current transaction has valid program indices or not.
It will be set in fd_executor_load_transaction_accounts similar to how program_indices is used in
load_transaction_accounts on the agave side */
Expand Down
47 changes: 4 additions & 43 deletions src/flamenco/runtime/fd_executor.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1163,6 +1163,7 @@ fd_instr_stack_pop( fd_exec_txn_ctx_t * txn_ctx,
int
fd_execute_instr( fd_exec_txn_ctx_t * txn_ctx,
fd_instr_info_t * instr ) {
FD_RUNTIME_TXN_SPAD_FRAME_BEGIN( txn_ctx->spad, txn_ctx ) {
FD_SCRATCH_SCOPE_BEGIN {
fd_exec_instr_ctx_t * parent = NULL;
if( txn_ctx->instr_stack_sz ) {
Expand All @@ -1181,7 +1182,7 @@ fd_execute_instr( fd_exec_txn_ctx_t * txn_ctx,
.txn_ctx = txn_ctx,
.epoch_ctx = txn_ctx->epoch_ctx,
.slot_ctx = txn_ctx->slot_ctx,
.valloc = fd_scratch_virtual(),
.test_only_valloc = (fd_valloc_t){NULL, NULL},
.acc_mgr = txn_ctx->acc_mgr,
.funk_txn = txn_ctx->funk_txn,
.parent = parent,
Expand Down Expand Up @@ -1245,6 +1246,7 @@ fd_execute_instr( fd_exec_txn_ctx_t * txn_ctx,

return exec_result;
} FD_SCRATCH_SCOPE_END;
} FD_RUNTIME_TXN_SPAD_FRAME_END;
}

void
Expand Down Expand Up @@ -1452,45 +1454,6 @@ fd_execute_txn_prepare_phase3( fd_exec_slot_ctx_t * slot_ctx,
return 0;
}

/* Stuff to be done after multithreading ends */
int
fd_execute_txn_finalize( fd_exec_txn_ctx_t * txn_ctx,
int exec_txn_err ) {
if( exec_txn_err != 0 ) {
for( ulong i = 0; i < txn_ctx->accounts_cnt; i++ ) {
fd_borrowed_account_t * acc_rec = &txn_ctx->borrowed_accounts[i];
void * acc_rec_data = fd_borrowed_account_destroy( acc_rec );
if( acc_rec_data != NULL ) {
fd_valloc_free( txn_ctx->valloc, acc_rec_data );
}
}

// fd_funk_txn_cancel( slot_ctx->acc_mgr->funk, txn_ctx->funk_txn, 0 );
return 0;
}

for( ulong i = 0; i < txn_ctx->accounts_cnt; i++ ) {
if( !fd_txn_account_is_writable_idx( txn_ctx, (int)i ) ) {
continue;
}

fd_borrowed_account_t * acc_rec = &txn_ctx->borrowed_accounts[i];

int ret = fd_acc_mgr_save_non_tpool( txn_ctx->acc_mgr, txn_ctx->funk_txn, acc_rec );
if( ret != FD_ACC_MGR_SUCCESS ) {
FD_LOG_ERR(( "failed to save edits to accounts" ));
return -1;
}

void * borrow_account_data = fd_borrowed_account_destroy( acc_rec );
if( borrow_account_data != NULL ) {
fd_valloc_free( txn_ctx->valloc, borrow_account_data );
}
}

return 0;
}

/* Creates a TxnContext Protobuf message from a provided txn_ctx.
- The transaction is assumed to have just finished phase 1 of preparation
- Caller of this function should have a scratch frame ready
Expand Down Expand Up @@ -1826,10 +1789,8 @@ fd_dump_txn_to_protobuf( fd_exec_txn_ctx_t *txn_ctx, fd_spad_t * spad ) {
fd_base58_encode_64( signature, &out_size, encoded_signature );

if( txn_ctx->capture_ctx->dump_proto_sig_filter ) {
ulong filter_strlen = (ulong) strlen(txn_ctx->capture_ctx->dump_proto_sig_filter);

// Terminate early if the signature does not match
if( memcmp( txn_ctx->capture_ctx->dump_proto_sig_filter, encoded_signature, filter_strlen < out_size ? filter_strlen : out_size ) ) {
if( strcmp( txn_ctx->capture_ctx->dump_proto_sig_filter, encoded_signature ) ) {
return;
}
}
Expand Down
4 changes: 0 additions & 4 deletions src/flamenco/runtime/fd_executor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,6 @@ fd_execute_txn_prepare_phase3( fd_exec_slot_ctx_t * slot_ctx,
fd_exec_txn_ctx_t * txn_ctx,
fd_txn_p_t * txn );

int
fd_execute_txn_finalize( fd_exec_txn_ctx_t * txn_ctx,
int exec_txn_err );

/*
Execute the given transaction.
Expand Down
67 changes: 43 additions & 24 deletions src/flamenco/runtime/fd_runtime.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1104,7 +1104,7 @@ fd_runtime_prepare_execute_finalize_txn( fd_exec_slot_ctx_t * slot_ctx,
fd_capture_ctx_t * capture_ctx,
fd_txn_p_t * txn,
fd_execute_txn_task_info_t * task_info ) {

FD_SPAD_FRAME_BEGIN( spad ) {
FD_SCRATCH_SCOPE_BEGIN {

int res = 0;
Expand All @@ -1125,8 +1125,6 @@ fd_runtime_prepare_execute_finalize_txn( fd_exec_slot_ctx_t * slot_ctx,
return -1;
}

txn_ctx->valloc = fd_scratch_virtual();

/* NOTE: This intentionally does not have sigverify */

fd_runtime_pre_execute_check( task_info );
Expand All @@ -1152,6 +1150,7 @@ fd_runtime_prepare_execute_finalize_txn( fd_exec_slot_ctx_t * slot_ctx,
return res;

} FD_SCRATCH_SCOPE_END;
} FD_SPAD_FRAME_END;
}


Expand Down Expand Up @@ -1445,13 +1444,13 @@ fd_runtime_finalize_txn( fd_exec_slot_ctx_t * slot_ctx,
if( dirty_vote_acc && 0==memcmp( acc_rec->const_meta->info.owner, &fd_solana_vote_program_id, sizeof(fd_pubkey_t) ) ) {
/* lock for inserting/modifying vote accounts in slot ctx. */
fd_funk_start_write( slot_ctx->acc_mgr->funk );
fd_vote_store_account( slot_ctx, acc_rec );
FD_SCRATCH_SCOPE_BEGIN {
fd_vote_store_account( slot_ctx, acc_rec, txn_ctx->spad );
FD_SPAD_FRAME_BEGIN( txn_ctx->spad ) {
fd_vote_state_versioned_t vsv[1];
fd_bincode_decode_ctx_t decode_vsv =
{ .data = acc_rec->const_data,
.dataend = acc_rec->const_data + acc_rec->const_meta->dlen,
.valloc = fd_scratch_virtual() };
.valloc = fd_spad_virtual( txn_ctx->spad ) };

int err = fd_vote_state_versioned_decode( vsv, &decode_vsv );
if( err ) break; /* out of scratch scope */
Expand All @@ -1471,9 +1470,9 @@ fd_runtime_finalize_txn( fd_exec_slot_ctx_t * slot_ctx,
__builtin_unreachable();
}

fd_vote_record_timestamp_vote_with_slot( slot_ctx, acc_rec->pubkey, ts->timestamp, ts->slot );
}
FD_SCRATCH_SCOPE_END;
fd_valloc_t valloc = fd_spad_virtual( txn_ctx->spad );
fd_vote_record_timestamp_vote_with_slot( slot_ctx, acc_rec->pubkey, ts->timestamp, ts->slot, valloc );
} FD_SPAD_FRAME_END;
fd_funk_end_write( slot_ctx->acc_mgr->funk );
}

Expand Down Expand Up @@ -1734,7 +1733,7 @@ fd_runtime_finalize_txns_tpool( fd_exec_slot_ctx_t * slot_ctx,
results = fd_scratch_alloc( alignof(uchar), txn_cnt * sizeof(uchar) );
}

fd_borrowed_account_t * * accounts_to_save = fd_scratch_alloc( 8UL, 128UL * txn_cnt * sizeof(fd_borrowed_account_t *) );
fd_borrowed_account_t * * accounts_to_save = fd_scratch_alloc( 8UL, MAX_TX_ACCOUNT_LOCKS * txn_cnt * sizeof(fd_borrowed_account_t *) );
ulong acc_idx = 0UL;
ulong nonvote_txn_count = 0;
ulong failed_txn_count = 0;
Expand Down Expand Up @@ -1837,13 +1836,13 @@ fd_runtime_finalize_txns_tpool( fd_exec_slot_ctx_t * slot_ctx,
fd_borrowed_account_t * acc_rec = &txn_ctx->borrowed_accounts[i];

if( dirty_vote_acc && !memcmp( acc_rec->const_meta->info.owner, &fd_solana_vote_program_id, sizeof(fd_pubkey_t) ) ) {
fd_vote_store_account( slot_ctx, acc_rec );
FD_SCRATCH_SCOPE_BEGIN {
fd_vote_store_account( slot_ctx, acc_rec, txn_ctx->spad );
FD_SPAD_FRAME_BEGIN( txn_ctx->spad ) {
fd_vote_state_versioned_t vsv[1];
fd_bincode_decode_ctx_t decode_vsv =
{ .data = acc_rec->const_data,
.dataend = acc_rec->const_data + acc_rec->const_meta->dlen,
.valloc = fd_scratch_virtual() };
.valloc = fd_spad_virtual( txn_ctx->spad ) };

int err = fd_vote_state_versioned_decode( vsv, &decode_vsv );
if( err ) break; /* out of scratch scope */
Expand All @@ -1863,9 +1862,9 @@ fd_runtime_finalize_txns_tpool( fd_exec_slot_ctx_t * slot_ctx,
__builtin_unreachable();
}

fd_vote_record_timestamp_vote_with_slot( slot_ctx, acc_rec->pubkey, ts->timestamp, ts->slot );
}
FD_SCRATCH_SCOPE_END;
fd_valloc_t valloc = fd_spad_virtual( txn_ctx->spad );
fd_vote_record_timestamp_vote_with_slot( slot_ctx, acc_rec->pubkey, ts->timestamp, ts->slot, valloc );
} FD_SPAD_FRAME_END;
}

if( dirty_stake_acc && !memcmp( acc_rec->const_meta->info.owner, &fd_solana_stake_program_id, sizeof(fd_pubkey_t) ) ) {
Expand Down Expand Up @@ -2120,18 +2119,26 @@ fd_runtime_execute_txns_in_waves_tpool( fd_exec_slot_ctx_t * slot_ctx,
next_incomplete_txn_idxs = temp_incomplete_txn_idxs;
incomplete_txn_idxs_cnt = next_incomplete_txn_idxs_cnt;

// Dump txns in waves
if( dump_txn ) {
for( ulong i = 0; i < wave_task_infos_cnt; ++i ) {
fd_dump_txn_to_protobuf( wave_task_infos[i].txn_ctx, spads[0] );
}
for( ulong i=0UL; i<spad_cnt; i++ ) {
/* Borrowed accounts are allocated during prep and need to
persist till the end of finalize. This initial frame will
be holding that. */
fd_spad_push( spads[ i ] );
}

/* Assign out spads to the transaction contexts */
for( ulong i=0UL; i<wave_task_infos_cnt; i++ ) {
wave_task_infos[i].spads = spads;
}

// Dump txns in waves
if( dump_txn ) {
for( ulong i = 0; i < wave_task_infos_cnt; ++i ) {
/* Manual push/pop on the spad within the callee. */
fd_dump_txn_to_protobuf( wave_task_infos[i].txn_ctx, spads[0] );
}
}

res |= fd_runtime_verify_txn_signatures_tpool( wave_task_infos, wave_task_infos_cnt, tpool );
if( res != 0 ) {
FD_LOG_WARNING(("Fail signature verification"));
Expand All @@ -2147,9 +2154,19 @@ fd_runtime_execute_txns_in_waves_tpool( fd_exec_slot_ctx_t * slot_ctx,
FD_LOG_ERR(("Fail finalize"));
}

/* Resetting the spad is a O(1) operation */
for( ulong i=0UL; i<spad_cnt; i++ ) {
fd_spad_reset( spads[i] );
/* The first frame, which holds borrowed accounts, can be
pretty big, and there are additional dynamic allocations
during finalize. */
if( FD_UNLIKELY( fd_spad_verify( spads[ i ] ) ) ) {
FD_LOG_ERR(( "spad corrupted or overflown" ));
}
/* We indiscriminately pushed a frame to every spad.
So it should be safe to indiscriminately pop here. */
fd_spad_pop( spads[ i ] );
if( FD_UNLIKELY( fd_spad_frame_used( spads[ i ] )!=0 ) ) {
FD_LOG_ERR(( "stray spad frame frame_used=%lu", fd_spad_frame_used( spads[ i ] ) ));
}
}

// wave_time += fd_log_wallclock();
Expand Down Expand Up @@ -2957,10 +2974,12 @@ fd_runtime_txn_lamports_per_signature( fd_exec_txn_ctx_t * txn_ctx,
// why is asan not detecting access to uninitialized memory here?!
fd_nonce_state_versions_t state;
int err;
if ((NULL != txn_descriptor) && fd_load_nonce_account(txn_ctx, &state, txn_ctx->valloc, &err)) {
FD_SPAD_FRAME_BEGIN( txn_ctx->spad ) {
if ((NULL != txn_descriptor) && fd_load_nonce_account(txn_ctx, &state, fd_spad_virtual( txn_ctx->spad ), &err)) {
if (state.inner.current.discriminant == fd_nonce_state_enum_initialized)
return state.inner.current.inner.initialized.fee_calculator.lamports_per_signature;
}
} FD_SPAD_FRAME_END;

// lamports_per_signature = (transaction has a DurableNonce, use the lamports_per_signature from that nonce instead of looking up the recent_block_hash and using the lamports_per_signature associated with that hash
// let TransactionExecutionDetails {
Expand Down
Loading

0 comments on commit 2a82177

Please sign in to comment.