Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

flamenco: back input regions and bincode valloc with spad #3423

Merged
merged 1 commit into from
Dec 19, 2024
Merged

flamenco: back input regions and bincode valloc with spad #3423

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions src/app/fdctl/run/tiles/fd_replay.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,8 @@ scratch_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) {
l = FD_LAYOUT_APPEND( l, FD_BMTREE_COMMIT_ALIGN, FD_BMTREE_COMMIT_FOOTPRINT(0) );
}
l = FD_LAYOUT_APPEND( l, FD_SCRATCH_ALIGN_DEFAULT, tile->replay.tpool_thread_count * TPOOL_WORKER_MEM_SZ );
l = FD_LAYOUT_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * fd_spad_footprint( MAX_TX_ACCOUNT_LOCKS * fd_ulong_align_up( FD_ACC_TOT_SZ_MAX, FD_ACCOUNT_REC_ALIGN ) ) );
ulong thread_spad_size = fd_spad_footprint( FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_DEFAULT );
l = FD_LAYOUT_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * fd_ulong_align_up( thread_spad_size, fd_spad_align() ) );
l = FD_LAYOUT_APPEND( l, fd_scratch_smem_align(), fd_scratch_smem_footprint( SCRATCH_MAX ) );
l = FD_LAYOUT_APPEND( l, fd_scratch_fmem_align(), fd_scratch_fmem_footprint( SCRATCH_DEPTH ) );
l = FD_LAYOUT_FINI ( l, scratch_align() );
Expand Down Expand Up @@ -1723,8 +1724,8 @@ unprivileged_init( fd_topo_t * topo,
ctx->bmtree[i] = FD_SCRATCH_ALLOC_APPEND( l, FD_BMTREE_COMMIT_ALIGN, FD_BMTREE_COMMIT_FOOTPRINT(0) );
}
void * tpool_worker_mem = FD_SCRATCH_ALLOC_APPEND( l, FD_SCRATCH_ALIGN_DEFAULT, tile->replay.tpool_thread_count * TPOOL_WORKER_MEM_SZ );
ulong thread_spad_size = fd_spad_footprint( MAX_TX_ACCOUNT_LOCKS * fd_ulong_align_up( FD_ACC_TOT_SZ_MAX, FD_ACCOUNT_REC_ALIGN ) );
void * spad_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * thread_spad_size );
ulong thread_spad_size = fd_spad_footprint( FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_DEFAULT );
void * spad_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * fd_ulong_align_up( thread_spad_size, fd_spad_align() ) );
void * scratch_smem = FD_SCRATCH_ALLOC_APPEND( l, fd_scratch_smem_align(), fd_scratch_smem_footprint( SCRATCH_MAX ) );
void * scratch_fmem = FD_SCRATCH_ALLOC_APPEND( l, fd_scratch_fmem_align(), fd_scratch_fmem_footprint( SCRATCH_DEPTH ) );
ulong scratch_alloc_mem = FD_SCRATCH_ALLOC_FINI ( l, scratch_align() );
Expand Down Expand Up @@ -1950,7 +1951,7 @@ unprivileged_init( fd_topo_t * topo,
for( ulong i=0UL; i<tile->replay.tpool_thread_count; i++ ) {
fd_spad_t * spad = fd_spad_join( fd_spad_new( spad_mem_cur, thread_spad_size ) );
ctx->spads[ ctx->spad_cnt++ ] = spad;
spad_mem_cur += thread_spad_size;
spad_mem_cur += fd_ulong_align_up( thread_spad_size, fd_spad_align() );
}

/**********************************************************************/
Expand Down
2 changes: 1 addition & 1 deletion src/app/ledger/main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -467,7 +467,7 @@ fd_ledger_main_setup( fd_ledger_args_t * args ) {

args->spad_cnt = fd_tpool_worker_cnt( args->tpool );
for( ulong i=0UL; i<args->spad_cnt; i++ ) {
ulong total_mem_sz = fd_spad_footprint( MAX_TX_ACCOUNT_LOCKS * fd_ulong_align_up( FD_ACC_TOT_SZ_MAX, FD_ACCOUNT_REC_ALIGN ) );
ulong total_mem_sz = FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_FUZZ;
uchar * mem = fd_wksp_alloc_laddr( args->wksp, FD_SPAD_ALIGN, total_mem_sz, 999UL );
fd_spad_t * spad = fd_spad_join( fd_spad_new( mem, total_mem_sz ) );
if( FD_UNLIKELY( !spad ) ) {
Expand Down
1 change: 0 additions & 1 deletion src/flamenco/runtime/context/fd_exec_instr_ctx.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ struct __attribute__((aligned(8UL))) fd_exec_instr_ctx {

fd_funk_txn_t * funk_txn;
fd_acc_mgr_t * acc_mgr;
fd_valloc_t valloc;

/* Most instructions log the base58 program id multiple times, so it's
convenient to compute it once and reuse it. */
Expand Down
2 changes: 1 addition & 1 deletion src/flamenco/runtime/context/fd_exec_slot_ctx.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ recover_clock( fd_exec_slot_ctx_t * slot_ctx ) {

/* Record timestamp */
if( vote_state_timestamp.slot != 0 || n->elem.stake != 0 ) {
fd_vote_record_timestamp_vote_with_slot(slot_ctx, &n->elem.key, vote_state_timestamp.timestamp, vote_state_timestamp.slot);
fd_vote_record_timestamp_vote_with_slot( slot_ctx, &n->elem.key, vote_state_timestamp.timestamp, vote_state_timestamp.slot, slot_ctx->valloc );
}
}

Expand Down
1 change: 0 additions & 1 deletion src/flamenco/runtime/context/fd_exec_txn_ctx.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,6 @@ fd_exec_txn_ctx_from_exec_slot_ctx( fd_exec_slot_ctx_t * slot_ctx,
fd_exec_txn_ctx_t * txn_ctx ) {
txn_ctx->slot_ctx = slot_ctx;
txn_ctx->epoch_ctx = slot_ctx->epoch_ctx;
txn_ctx->valloc = slot_ctx->valloc;
txn_ctx->funk_txn = NULL;
txn_ctx->acc_mgr = slot_ctx->acc_mgr;
}
Expand Down
4 changes: 1 addition & 3 deletions src/flamenco/runtime/context/fd_exec_txn_ctx.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ struct __attribute__((aligned(8UL))) fd_exec_txn_ctx {

fd_funk_txn_t * funk_txn;
fd_acc_mgr_t * acc_mgr;
fd_valloc_t valloc;
fd_spad_t * spad; /* Sized out to handle the worst case footprint of single transaction execution. */

ulong paid_fees;
ulong compute_unit_limit; /* Compute unit limit for this transaction. */
Expand Down Expand Up @@ -126,8 +126,6 @@ struct __attribute__((aligned(8UL))) fd_exec_txn_ctx {
int exec_err;
int exec_err_kind;

fd_spad_t * spad;

/* The has_program_id flag is used to indicate if the current transaction has valid program indices or not.
It will be set in fd_executor_load_transaction_accounts similar to how program_indices is used in
load_transaction_accounts on the agave side */
Expand Down
46 changes: 3 additions & 43 deletions src/flamenco/runtime/fd_executor.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1185,6 +1185,7 @@ fd_instr_stack_pop( fd_exec_txn_ctx_t * txn_ctx,
int
fd_execute_instr( fd_exec_txn_ctx_t * txn_ctx,
fd_instr_info_t * instr ) {
FD_RUNTIME_TXN_SPAD_FRAME_BEGIN( txn_ctx->spad, txn_ctx ) {
FD_SCRATCH_SCOPE_BEGIN {
fd_exec_instr_ctx_t * parent = NULL;
if( txn_ctx->instr_stack_sz ) {
Expand All @@ -1203,7 +1204,6 @@ fd_execute_instr( fd_exec_txn_ctx_t * txn_ctx,
.txn_ctx = txn_ctx,
.epoch_ctx = txn_ctx->epoch_ctx,
.slot_ctx = txn_ctx->slot_ctx,
.valloc = fd_scratch_virtual(),
.acc_mgr = txn_ctx->acc_mgr,
.funk_txn = txn_ctx->funk_txn,
.parent = parent,
Expand Down Expand Up @@ -1267,6 +1267,7 @@ fd_execute_instr( fd_exec_txn_ctx_t * txn_ctx,

return exec_result;
} FD_SCRATCH_SCOPE_END;
} FD_RUNTIME_TXN_SPAD_FRAME_END;
}

void
Expand Down Expand Up @@ -1477,45 +1478,6 @@ fd_execute_txn_prepare_phase3( fd_exec_slot_ctx_t * slot_ctx,
return 0;
}

/* Stuff to be done after multithreading ends */
int
fd_execute_txn_finalize( fd_exec_txn_ctx_t * txn_ctx,
int exec_txn_err ) {
if( exec_txn_err != 0 ) {
for( ulong i = 0; i < txn_ctx->accounts_cnt; i++ ) {
fd_borrowed_account_t * acc_rec = &txn_ctx->borrowed_accounts[i];
void * acc_rec_data = fd_borrowed_account_destroy( acc_rec );
if( acc_rec_data != NULL ) {
fd_valloc_free( txn_ctx->valloc, acc_rec_data );
}
}

// fd_funk_txn_cancel( slot_ctx->acc_mgr->funk, txn_ctx->funk_txn, 0 );
return 0;
}

for( ulong i = 0; i < txn_ctx->accounts_cnt; i++ ) {
if( !fd_txn_account_is_writable_idx( txn_ctx, (int)i ) ) {
continue;
}

fd_borrowed_account_t * acc_rec = &txn_ctx->borrowed_accounts[i];

int ret = fd_acc_mgr_save_non_tpool( txn_ctx->acc_mgr, txn_ctx->funk_txn, acc_rec );
if( ret != FD_ACC_MGR_SUCCESS ) {
FD_LOG_ERR(( "failed to save edits to accounts" ));
return -1;
}

void * borrow_account_data = fd_borrowed_account_destroy( acc_rec );
if( borrow_account_data != NULL ) {
fd_valloc_free( txn_ctx->valloc, borrow_account_data );
}
}

return 0;
}

/* Creates a TxnContext Protobuf message from a provided txn_ctx.
- The transaction is assumed to have just finished phase 1 of preparation
- Caller of this function should have a scratch frame ready
Expand Down Expand Up @@ -1851,10 +1813,8 @@ fd_dump_txn_to_protobuf( fd_exec_txn_ctx_t *txn_ctx, fd_spad_t * spad ) {
fd_base58_encode_64( signature, &out_size, encoded_signature );

if( txn_ctx->capture_ctx->dump_proto_sig_filter ) {
ulong filter_strlen = (ulong) strlen(txn_ctx->capture_ctx->dump_proto_sig_filter);

// Terminate early if the signature does not match
if( memcmp( txn_ctx->capture_ctx->dump_proto_sig_filter, encoded_signature, filter_strlen < out_size ? filter_strlen : out_size ) ) {
if( strcmp( txn_ctx->capture_ctx->dump_proto_sig_filter, encoded_signature ) ) {
return;
}
}
Expand Down
4 changes: 0 additions & 4 deletions src/flamenco/runtime/fd_executor.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,6 @@ fd_execute_txn_prepare_phase3( fd_exec_slot_ctx_t * slot_ctx,
fd_exec_txn_ctx_t * txn_ctx,
fd_txn_p_t * txn );

int
fd_execute_txn_finalize( fd_exec_txn_ctx_t * txn_ctx,
int exec_txn_err );

/*
Execute the given transaction.
Expand Down
71 changes: 46 additions & 25 deletions src/flamenco/runtime/fd_runtime.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,9 @@ fd_runtime_init_bank_from_genesis( fd_exec_slot_ctx_t * slot_ctx,
.unused = 0,
.vote_accounts = (fd_vote_accounts_t){
.vote_accounts_pool = vacc_pool,
.vote_accounts_root = vacc_root}};
.vote_accounts_root = vacc_root},
.stake_history = {0}
};

slot_ctx->slot_bank.capitalization = capitalization;

Expand Down Expand Up @@ -1100,7 +1102,7 @@ fd_runtime_prepare_execute_finalize_txn( fd_exec_slot_ctx_t * slot_ctx,
fd_capture_ctx_t * capture_ctx,
fd_txn_p_t * txn,
fd_execute_txn_task_info_t * task_info ) {

FD_SPAD_FRAME_BEGIN( spad ) {
FD_SCRATCH_SCOPE_BEGIN {

int res = 0;
Expand All @@ -1121,8 +1123,6 @@ fd_runtime_prepare_execute_finalize_txn( fd_exec_slot_ctx_t * slot_ctx,
return -1;
}

txn_ctx->valloc = fd_scratch_virtual();

/* NOTE: This intentionally does not have sigverify */

fd_runtime_pre_execute_check( task_info );
Expand All @@ -1148,6 +1148,7 @@ fd_runtime_prepare_execute_finalize_txn( fd_exec_slot_ctx_t * slot_ctx,
return res;

} FD_SCRATCH_SCOPE_END;
} FD_SPAD_FRAME_END;
}


Expand Down Expand Up @@ -1441,13 +1442,13 @@ fd_runtime_finalize_txn( fd_exec_slot_ctx_t * slot_ctx,
if( dirty_vote_acc && 0==memcmp( acc_rec->const_meta->info.owner, &fd_solana_vote_program_id, sizeof(fd_pubkey_t) ) ) {
/* lock for inserting/modifying vote accounts in slot ctx. */
fd_funk_start_write( slot_ctx->acc_mgr->funk );
fd_vote_store_account( slot_ctx, acc_rec );
FD_SCRATCH_SCOPE_BEGIN {
fd_vote_store_account( slot_ctx, acc_rec, txn_ctx->spad );
FD_SPAD_FRAME_BEGIN( txn_ctx->spad ) {
fd_vote_state_versioned_t vsv[1];
fd_bincode_decode_ctx_t decode_vsv =
{ .data = acc_rec->const_data,
.dataend = acc_rec->const_data + acc_rec->const_meta->dlen,
.valloc = fd_scratch_virtual() };
.valloc = fd_spad_virtual( txn_ctx->spad ) };

int err = fd_vote_state_versioned_decode( vsv, &decode_vsv );
if( err ) break; /* out of scratch scope */
Expand All @@ -1467,9 +1468,9 @@ fd_runtime_finalize_txn( fd_exec_slot_ctx_t * slot_ctx,
__builtin_unreachable();
}

fd_vote_record_timestamp_vote_with_slot( slot_ctx, acc_rec->pubkey, ts->timestamp, ts->slot );
}
FD_SCRATCH_SCOPE_END;
fd_valloc_t valloc = fd_spad_virtual( txn_ctx->spad );
fd_vote_record_timestamp_vote_with_slot( slot_ctx, acc_rec->pubkey, ts->timestamp, ts->slot, valloc );
} FD_SPAD_FRAME_END;
fd_funk_end_write( slot_ctx->acc_mgr->funk );
}

Expand Down Expand Up @@ -1730,7 +1731,7 @@ fd_runtime_finalize_txns_tpool( fd_exec_slot_ctx_t * slot_ctx,
results = fd_scratch_alloc( alignof(uchar), txn_cnt * sizeof(uchar) );
}

fd_borrowed_account_t * * accounts_to_save = fd_scratch_alloc( 8UL, 128UL * txn_cnt * sizeof(fd_borrowed_account_t *) );
fd_borrowed_account_t * * accounts_to_save = fd_scratch_alloc( 8UL, MAX_TX_ACCOUNT_LOCKS * txn_cnt * sizeof(fd_borrowed_account_t *) );
ulong acc_idx = 0UL;
ulong nonvote_txn_count = 0;
ulong failed_txn_count = 0;
Expand Down Expand Up @@ -1833,13 +1834,13 @@ fd_runtime_finalize_txns_tpool( fd_exec_slot_ctx_t * slot_ctx,
fd_borrowed_account_t * acc_rec = &txn_ctx->borrowed_accounts[i];

if( dirty_vote_acc && !memcmp( acc_rec->const_meta->info.owner, &fd_solana_vote_program_id, sizeof(fd_pubkey_t) ) ) {
fd_vote_store_account( slot_ctx, acc_rec );
FD_SCRATCH_SCOPE_BEGIN {
fd_vote_store_account( slot_ctx, acc_rec, txn_ctx->spad );
FD_SPAD_FRAME_BEGIN( txn_ctx->spad ) {
fd_vote_state_versioned_t vsv[1];
fd_bincode_decode_ctx_t decode_vsv =
{ .data = acc_rec->const_data,
.dataend = acc_rec->const_data + acc_rec->const_meta->dlen,
.valloc = fd_scratch_virtual() };
.valloc = fd_spad_virtual( txn_ctx->spad ) };

int err = fd_vote_state_versioned_decode( vsv, &decode_vsv );
if( err ) break; /* out of scratch scope */
Expand All @@ -1859,9 +1860,9 @@ fd_runtime_finalize_txns_tpool( fd_exec_slot_ctx_t * slot_ctx,
__builtin_unreachable();
}

fd_vote_record_timestamp_vote_with_slot( slot_ctx, acc_rec->pubkey, ts->timestamp, ts->slot );
}
FD_SCRATCH_SCOPE_END;
fd_valloc_t valloc = fd_spad_virtual( txn_ctx->spad );
fd_vote_record_timestamp_vote_with_slot( slot_ctx, acc_rec->pubkey, ts->timestamp, ts->slot, valloc );
} FD_SPAD_FRAME_END;
}

if( dirty_stake_acc && !memcmp( acc_rec->const_meta->info.owner, &fd_solana_stake_program_id, sizeof(fd_pubkey_t) ) ) {
Expand Down Expand Up @@ -2116,18 +2117,26 @@ fd_runtime_execute_txns_in_waves_tpool( fd_exec_slot_ctx_t * slot_ctx,
next_incomplete_txn_idxs = temp_incomplete_txn_idxs;
incomplete_txn_idxs_cnt = next_incomplete_txn_idxs_cnt;

// Dump txns in waves
if( dump_txn ) {
for( ulong i = 0; i < wave_task_infos_cnt; ++i ) {
fd_dump_txn_to_protobuf( wave_task_infos[i].txn_ctx, spads[0] );
}
for( ulong i=0UL; i<spad_cnt; i++ ) {
/* Borrowed accounts are allocated during prep and need to
persist till the end of finalize. This initial frame will
be holding that. */
fd_spad_push( spads[ i ] );
}

/* Assign out spads to the transaction contexts */
for( ulong i=0UL; i<wave_task_infos_cnt; i++ ) {
wave_task_infos[i].spads = spads;
}

// Dump txns in waves
if( dump_txn ) {
for( ulong i = 0; i < wave_task_infos_cnt; ++i ) {
/* Manual push/pop on the spad within the callee. */
fd_dump_txn_to_protobuf( wave_task_infos[i].txn_ctx, spads[0] );
}
}

res |= fd_runtime_verify_txn_signatures_tpool( wave_task_infos, wave_task_infos_cnt, tpool );
if( res != 0 ) {
FD_LOG_WARNING(("Fail signature verification"));
Expand All @@ -2143,9 +2152,19 @@ fd_runtime_execute_txns_in_waves_tpool( fd_exec_slot_ctx_t * slot_ctx,
FD_LOG_ERR(("Fail finalize"));
}

/* Resetting the spad is a O(1) operation */
for( ulong i=0UL; i<spad_cnt; i++ ) {
fd_spad_reset( spads[i] );
/* The first frame, which holds borrowed accounts, can be
pretty big, and there are additional dynamic allocations
during finalize. */
if( FD_UNLIKELY( fd_spad_verify( spads[ i ] ) ) ) {
FD_LOG_ERR(( "spad corrupted or overflown" ));
}
/* We indiscriminately pushed a frame to every spad.
So it should be safe to indiscriminately pop here. */
fd_spad_pop( spads[ i ] );
if( FD_UNLIKELY( fd_spad_frame_used( spads[ i ] )!=0 ) ) {
FD_LOG_ERR(( "stray spad frame frame_used=%lu", fd_spad_frame_used( spads[ i ] ) ));
}
}

// wave_time += fd_log_wallclock();
Expand Down Expand Up @@ -2953,10 +2972,12 @@ fd_runtime_txn_lamports_per_signature( fd_exec_txn_ctx_t * txn_ctx,
// why is asan not detecting access to uninitialized memory here?!
fd_nonce_state_versions_t state;
int err;
if ((NULL != txn_descriptor) && fd_load_nonce_account(txn_ctx, &state, txn_ctx->valloc, &err)) {
FD_SPAD_FRAME_BEGIN( txn_ctx->spad ) {
if ((NULL != txn_descriptor) && fd_load_nonce_account(txn_ctx, &state, fd_spad_virtual( txn_ctx->spad ), &err)) {
if (state.inner.current.discriminant == fd_nonce_state_enum_initialized)
return state.inner.current.inner.initialized.fee_calculator.lamports_per_signature;
}
} FD_SPAD_FRAME_END;

// lamports_per_signature = (transaction has a DurableNonce, use the lamports_per_signature from that nonce instead of looking up the recent_block_hash and using the lamports_per_signature associated with that hash
// let TransactionExecutionDetails {
Expand Down
Loading
Loading