From 5aec642e063178ad6f39ded0e7c892aae8f6119a Mon Sep 17 00:00:00 2001 From: ibhatt-jumptrading Date: Thu, 31 Oct 2024 15:47:41 +0000 Subject: [PATCH] um? --- src/app/ledger/main.c | 3 +- src/ballet/txn/fd_txn.h | 4 + .../runtime/context/fd_exec_slot_ctx.c | 8 +- src/flamenco/runtime/fd_hashes.c | 5 - src/flamenco/runtime/fd_runtime.c | 16 ++- src/flamenco/runtime/fd_txncache.c | 35 +++++ src/flamenco/runtime/fd_txncache.h | 3 + .../runtime/program/fd_vote_program.c | 37 +---- src/flamenco/snapshot/fd_snapshot_create.h | 126 ++++-------------- 9 files changed, 92 insertions(+), 145 deletions(-) diff --git a/src/app/ledger/main.c b/src/app/ledger/main.c index 291ee6cb24..c11057add0 100644 --- a/src/app/ledger/main.c +++ b/src/app/ledger/main.c @@ -226,7 +226,8 @@ runtime_replay( fd_ledger_args_t * ledger_args ) { fd_blockstore_end_read( blockstore ); /* TODO:FIXME: This is where we want to do all of the snapshot related testing */ - if( false && ledger_args->slot_ctx->slot_bank.slot==254462501 ) { + //if( false && ledger_args->slot_ctx->slot_bank.slot==254462501 ) { + if( false && !first_create ) { fd_snapshot_create_manifest( ledger_args->slot_ctx ); first_create = 1; } diff --git a/src/ballet/txn/fd_txn.h b/src/ballet/txn/fd_txn.h index 2144531142..1148fb53a6 100644 --- a/src/ballet/txn/fd_txn.h +++ b/src/ballet/txn/fd_txn.h @@ -86,7 +86,11 @@ /* FD_TXN_INSTR_MAX: The (inclusive) maximum number of instructions a transaction can have. As of Solana 1.15.0, this is limited to 64. */ +#ifdef OFFLINE_REPLAY +#define FD_TXN_INSTR_MAX (128UL) +#else #define FD_TXN_INSTR_MAX (64UL) +#endif /* FD_TXN_MAX_SZ: The maximum amount of memory (in bytes) that a fd_txn can diff --git a/src/flamenco/runtime/context/fd_exec_slot_ctx.c b/src/flamenco/runtime/context/fd_exec_slot_ctx.c index d94fa95d0d..490fbacc0d 100644 --- a/src/flamenco/runtime/context/fd_exec_slot_ctx.c +++ b/src/flamenco/runtime/context/fd_exec_slot_ctx.c @@ -458,14 +458,16 @@ fd_exec_slot_ctx_recover_status_cache( fd_exec_slot_ctx_t * ctx, FD_SCRATCH_SCOPE_BEGIN { ulong num_entries = 0; + FD_LOG_WARNING(("SLOT DELTAS %lu", slot_deltas->slot_deltas_len)); for( ulong i = 0; i < slot_deltas->slot_deltas_len; i++ ) { fd_slot_delta_t * slot_delta = &slot_deltas->slot_deltas[i]; //FD_LOG_WARNING(("SLOT DELTA %lu %lu %lu", slot_delta->slot, (ulong)slot_delta->is_root, slot_delta->slot_delta_vec_len)); for( ulong j = 0; j < slot_delta->slot_delta_vec_len; j++ ) { - //FD_LOG_WARNING(("VALUE", slot_delta->slot_delta_vec[j].value.statuses/) + //FD_LOG_WARNING(("VALUE %lu", slot_delta->slot_delta_vec[j].value.statuses_len)); num_entries += slot_delta->slot_delta_vec[j].value.statuses_len; } } + FD_LOG_WARNING(("NUM ENTRIES %lu", num_entries)); fd_txncache_insert_t * insert_vals = fd_scratch_alloc( alignof(fd_txncache_insert_t), num_entries * sizeof(fd_txncache_insert_t) ); /* Dumb sort for 300 slot entries to insert in order. */ @@ -523,6 +525,10 @@ fd_exec_slot_ctx_recover_status_cache( fd_exec_slot_ctx_t * ctx, } } } FD_SCRATCH_SCOPE_END; + + //fd_txncache_get_entries( ctx->status_cache ); + + return ctx; } diff --git a/src/flamenco/runtime/fd_hashes.c b/src/flamenco/runtime/fd_hashes.c index 50c4cb325d..9b01439622 100644 --- a/src/flamenco/runtime/fd_hashes.c +++ b/src/flamenco/runtime/fd_hashes.c @@ -549,11 +549,6 @@ fd_update_hash_bank_tpool( fd_exec_slot_ctx_t * slot_ctx, continue; } - //FD_LOG_WARNING(("REMOVING %s", FD_BASE58_ENC_32_ALLOCA(task_info->rec->pair.key)));; - (void)funk; - fd_account_meta_t * metadata = fd_funk_val( task_info->rec, fd_funk_wksp( funk ) ); - FD_LOG_WARNING(("REMOVING %s %u %lu", FD_BASE58_ENC_32_ALLOCA(task_info->rec->pair.key), task_info->rec->val_sz, metadata->dlen)); - if( slot_ctx->slot_bank.slot < 254462500 ) { fd_funk_rec_remove(funk, fd_funk_rec_modify(funk, task_info->rec), 1); } diff --git a/src/flamenco/runtime/fd_runtime.c b/src/flamenco/runtime/fd_runtime.c index e35ab6975a..51c94c20f4 100644 --- a/src/flamenco/runtime/fd_runtime.c +++ b/src/flamenco/runtime/fd_runtime.c @@ -2437,8 +2437,8 @@ fd_runtime_block_execute_tpool_v2( fd_exec_slot_ctx_t * slot_ctx, fd_runtime_block_collect_txns( slot_ctx, block_info, txn_ptrs ); /* TODO: We need to call verify_ticks here. */ - slot_ctx->tick_height += 64UL; /* TODO: We should be using hte computed quantity of ticks here.*/ - slot_ctx->slot_bank.max_tick_height += 64; + slot_ctx->tick_height += 64UL; /* TODO: We should be using hte computed quantity of ticks here.*/ + slot_ctx->slot_bank.max_tick_height += 64UL; res = fd_runtime_execute_txns_in_waves_tpool( slot_ctx, capture_ctx, txn_ptrs, txn_cnt, tpool, spads, spad_cnt ); if( res != FD_RUNTIME_EXECUTE_SUCCESS ) { @@ -2870,6 +2870,12 @@ fd_runtime_publish_old_txns( fd_exec_slot_ctx_t * slot_ctx, fd_funk_txn_t * txnmap = fd_funk_txn_map(funk, fd_funk_wksp(funk)); uint depth = 0; for( fd_funk_txn_t * txn = slot_ctx->funk_txn; txn; txn = fd_funk_txn_parent(txn, txnmap) ) { + if( slot_ctx->status_cache ) { + fd_txncache_register_root_slot( slot_ctx->status_cache, txn->xid.ul[0] ); + } else { + FD_LOG_ERR(("OOPS")); + } + /* TODO: tmp change */ if (++depth == (FD_RUNTIME_NUM_ROOT_BLOCKS - 1) ) { FD_LOG_DEBUG(("publishing %s (slot %ld)", FD_BASE58_ENC_32_ALLOCA( &txn->xid ), txn->xid.ul[0])); @@ -2880,9 +2886,9 @@ fd_runtime_publish_old_txns( fd_exec_slot_ctx_t * slot_ctx, FD_LOG_ERR(("publish err")); return -1; } - if( slot_ctx->status_cache ) { - fd_txncache_register_root_slot( slot_ctx->status_cache, txn->xid.ul[0] ); - } + // if( slot_ctx->status_cache ) { + // fd_txncache_register_root_slot( slot_ctx->status_cache, txn->xid.ul[0] ); + // } if( FD_UNLIKELY( FD_FEATURE_ACTIVE(slot_ctx, epoch_accounts_hash) ) ) { fd_epoch_bank_t * epoch_bank = fd_exec_epoch_ctx_epoch_bank( slot_ctx->epoch_ctx ); diff --git a/src/flamenco/runtime/fd_txncache.c b/src/flamenco/runtime/fd_txncache.c index e24f057570..b688e3bc1c 100644 --- a/src/flamenco/runtime/fd_txncache.c +++ b/src/flamenco/runtime/fd_txncache.c @@ -996,3 +996,38 @@ fd_txncache_is_rooted_slot( fd_txncache_t * tc, fd_rwlock_unread( tc->lock ); return 0; } + +int +fd_txncache_get_entries( fd_txncache_t * tc ) { + + ulong num_entries = 0UL; + fd_rwlock_read( tc->lock ); + + fd_txncache_private_txnpage_t * txnpages = fd_txncache_get_txnpages( tc ); + ulong * root_slots = fd_txncache_get_root_slots( tc ); + for( ulong i=0UL; iroot_slots_cnt; i++ ) { + ulong slot = root_slots[ i ]; + fd_txncache_private_slotcache_t * slotcache; + if( FD_UNLIKELY( FD_TXNCACHE_FIND_FOUND!=fd_txncache_find_slot( tc, slot, 0, &slotcache ) ) ) continue; + + for( ulong j=0UL; j<300UL; j++ ) { + fd_txncache_private_slotblockcache_t * slotblockcache = &slotcache->blockcache[ j ]; + if( FD_UNLIKELY( slotblockcache->txnhash_offset>=ULONG_MAX-1UL ) ) { + continue; + } + + for( ulong k=0UL; kheads[ k ]; + for( ; head!=UINT_MAX; head=txnpages[ head/FD_TXNCACHE_TXNS_PER_PAGE ].txns[ head%FD_TXNCACHE_TXNS_PER_PAGE ]->slotblockcache_next ) { + fd_txncache_private_txn_t * FD_FN_UNUSED txn = txnpages[ head/FD_TXNCACHE_TXNS_PER_PAGE ].txns[ head%FD_TXNCACHE_TXNS_PER_PAGE ]; + num_entries++; + } + } + } + } + + fd_rwlock_unread( tc->lock ); + FD_LOG_WARNING(("GET THIS MANY ENTRIES %lu", num_entries)); + return 0; + +} diff --git a/src/flamenco/runtime/fd_txncache.h b/src/flamenco/runtime/fd_txncache.h index 2ed2aa4b60..847dcf38a2 100644 --- a/src/flamenco/runtime/fd_txncache.h +++ b/src/flamenco/runtime/fd_txncache.h @@ -394,6 +394,9 @@ int fd_txncache_is_rooted_slot( fd_txncache_t * tc, ulong slot ); +int +fd_txncache_get_entries( fd_txncache_t * tc ); + FD_PROTOTYPES_END #endif /* HEADER_fd_src_flamenco_runtime_txncache_h */ diff --git a/src/flamenco/runtime/program/fd_vote_program.c b/src/flamenco/runtime/program/fd_vote_program.c index 702eb233f2..8ae2326b05 100644 --- a/src/flamenco/runtime/program/fd_vote_program.c +++ b/src/flamenco/runtime/program/fd_vote_program.c @@ -2679,42 +2679,7 @@ fd_vote_program_execute( fd_exec_instr_ctx_t * ctx ) { */ case fd_vote_instruction_enum_compact_update_vote_state_switch: { /* https://github.com/anza-xyz/agave/blob/dc4b9dcbbf859ff48f40d00db824bde063fdafcc/programs/vote/src/vote_processor.rs#L183-L191 */ - if( FD_FEATURE_ACTIVE( ctx->slot_ctx, deprecate_legacy_vote_ixs ) && - FD_FEATURE_ACTIVE( ctx->slot_ctx, enable_tower_sync_ix ) ) { - return FD_EXECUTOR_INSTR_ERR_INVALID_INSTR_DATA; - } - - fd_compact_vote_state_update_t * vote_state_update = NULL; - if( instruction.discriminant == fd_vote_instruction_enum_compact_update_vote_state ) { - vote_state_update = &instruction.inner.compact_update_vote_state; - } else if( instruction.discriminant == - fd_vote_instruction_enum_compact_update_vote_state_switch ) { - vote_state_update = - &instruction.inner.compact_update_vote_state_switch.compact_vote_state_update; - } - - fd_vote_state_update_t vote_update; - fd_vote_state_update_new( &vote_update ); - if( FD_UNLIKELY( !fd_vote_decode_compact_update( vote_state_update, &vote_update ) ) ) - return FD_EXECUTOR_INSTR_ERR_INVALID_INSTR_DATA; - - if( FD_LIKELY( FD_FEATURE_ACTIVE( ctx->slot_ctx, allow_votes_to_directly_update_vote_state ) && - FD_FEATURE_ACTIVE( ctx->slot_ctx, compact_vote_state_updates ) ) ) { - // https://github.com/anza-xyz/agave/blob/v2.0.1/programs/vote/src/vote_processor.rs#L185 - fd_slot_hashes_t const * slot_hashes = fd_sysvar_cache_slot_hashes( ctx->slot_ctx->sysvar_cache ); - if( FD_UNLIKELY( !slot_hashes ) ) - return FD_EXECUTOR_INSTR_ERR_UNSUPPORTED_SYSVAR; - - fd_sol_sysvar_clock_t const * clock = fd_sysvar_cache_clock( ctx->slot_ctx->sysvar_cache ); - if( FD_UNLIKELY( !clock ) ) - return FD_EXECUTOR_INSTR_ERR_UNSUPPORTED_SYSVAR; - - // https://github.com/anza-xyz/agave/blob/v2.0.1/programs/vote/src/vote_processor.rs#L187 - rc = process_vote_state_update( 0, me, slot_hashes, clock, &vote_update, signers, ctx ); - } else { - // agave has removed the feature check.. we have not - rc = FD_EXECUTOR_INSTR_ERR_INVALID_INSTR_DATA; - } + return FD_EXECUTOR_INSTR_SUCCESS; break; } diff --git a/src/flamenco/snapshot/fd_snapshot_create.h b/src/flamenco/snapshot/fd_snapshot_create.h index c87dae04d5..4edf62634c 100644 --- a/src/flamenco/snapshot/fd_snapshot_create.h +++ b/src/flamenco/snapshot/fd_snapshot_create.h @@ -45,71 +45,29 @@ typedef struct fd_snapshot_create_private fd_snapshot_create_t; FD_PROTOTYPES_BEGIN -/* fd_snapshot_create_{align,footprint} return required memory region - parameters for the fd_snapshot_create_t object. - - worker_cnt is the number of workers for parallel snapshot create - (treated as 1UL parallel mode not available). compress_lvl is the - Zstandard compression level. compress_bufsz is the in-memory buffer - for writes (larger buffers results in less frequent but larger write - ops). funk_rec_cnt is the number of slots in the funk rec hashmap. - batch_acc_cnt is the max number of accounts per account vec. - - Resulting footprint approximates - - O( funk_rec_cnt + (worker_cnt * (compress_lvl + compress_bufsz + batch_acc_cnt)) ) */ - -FD_FN_CONST ulong -fd_snapshot_create_align( void ); - -ulong -fd_snapshot_create_footprint( ulong worker_cnt, - int compress_lvl, - ulong compress_bufsz, - ulong funk_rec_cnt, - ulong batch_acc_cnt ); - -/* fd_snapshot_create_new creates a new snapshot create object in the - given mem region, which adheres to above alignment/footprint - requirements. Returns qualified handle to object given create object - on success. Serializes data from given slot context. snap_path is - the final snapshot path. May create temporary files adject to - snap_path. {worker_cnt,compress_lvl,compress_bufsz,funk_rec_cnt, - batch_acc_cnt} must match arguments to footprint when mem was - created. On failure, returns NULL. Reasons for failure include - invalid memory region or invalid file descriptor. Logs reasons for - failure. */ - -fd_snapshot_create_t * -fd_snapshot_create_new( void * mem, - fd_exec_slot_ctx_t * slot_ctx, - const char * snap_path, - ulong worker_cnt, - int compress_lvl, - ulong compress_bufsz, - ulong funk_rec_cnt, - ulong batch_acc_cnt, - ulong max_accv_sz, - fd_rng_t * rng ); - -/* fd_snapshot_create_delete destroys the given snapshot create object - and frees any resources. Returns memory region and fd back to caller. */ - -void * -fd_snapshot_create_delete( fd_snapshot_create_t * create ); - -/* fd_snapshot_create exports the 'snapshot manifest' and a copy of all - accounts from the slot ctx that the create object is attached to. - Writes a .tar.zst stream out to the fd. Returns 1 on success, and - 0 on failure. Reason for failure is logged. */ - int -fd_snapshot_create( fd_snapshot_create_t * create, - fd_exec_slot_ctx_t * slot_ctx ); +fd_snapshot_create_populate_status_cache( fd_exec_slot_ctx_t * FD_FN_UNUSED slot_ctx, + fd_bank_slot_deltas_t * FD_FN_UNUSED bank_slot_deltas ) { + fd_txncache_t * status_cache = slot_ctx->status_cache; + + bank_slot_deltas->slot_deltas_len = 300UL; + bank_slot_deltas->slot_deltas = fd_scratch_alloc( 8UL, sizeof(fd_slot_delta_t) * bank_slot_deltas->slot_deltas_len ); + + + ulong * out_slots = fd_scratch_alloc( 8UL, sizeof(ulong) * bank_slot_deltas->slot_deltas_len ); + fd_txncache_root_slots( status_cache, out_slots ); + for( ulong i=0UL; i<300UL; i++ ) { + bank_slot_deltas->slot_deltas[ i ].slot = out_slots[ i ]; + bank_slot_deltas->slot_deltas[ i ].is_root = 1; + + FD_LOG_WARNING(("OUT SLOTS %lu", out_slots[ i ])); + } + return 0; +} int -fd_snapshot_create_populate_acc_vec_idx( fd_exec_slot_ctx_t * FD_FN_UNUSED slot_ctx, - fd_solana_manifest_serializable_t * FD_FN_UNUSED manifest ) { +fd_snapshot_create_populate_acc_vec_idx( fd_exec_slot_ctx_t * slot_ctx, + fd_solana_manifest_serializable_t * manifest ) { fd_funk_t * funk = slot_ctx->acc_mgr->funk; @@ -123,13 +81,6 @@ fd_snapshot_create_populate_acc_vec_idx( fd_exec_slot_ctx_t * FD_ ulong num_accs = 0UL; for (fd_funk_rec_t const *rec = fd_funk_txn_first_rec( funk, NULL ); NULL != rec; rec = fd_funk_txn_next_rec( funk, rec )) { - - uchar key[32]; - fd_base58_decode_32("JC3jTygxN7A3fEGPK9rMtN26f5HobTCydrkrQfG2B2M7", key); - if( !memcmp( key, rec->pair.key, sizeof(fd_pubkey_t))) { - FD_LOG_WARNING(("ASDF ASDF ASDF ASDF HERE")); - } - if( num_accs % 10000 == 0 ) { FD_LOG_WARNING(("ID %lu", id)); } @@ -378,7 +329,14 @@ fd_snapshot_create_serialiable_stakes( fd_exec_slot_ctx_t * slot_ctx, int FD_FN_UNUSED fd_snapshot_create_manifest( fd_exec_slot_ctx_t * slot_ctx ) { - /* Do a funk */ + FD_LOG_WARNING(("CREATING MANIFEST")); + + /* Populate the status cache */ + fd_bank_slot_deltas_t bank_slot_deltas = {0}; + fd_snapshot_create_populate_status_cache( slot_ctx, &bank_slot_deltas ); + FD_LOG_ERR(("ASDF ASDF")); + + /* Do a funk publish and a txncache publish */ fd_funk_t * funk = slot_ctx->acc_mgr->funk; fd_funk_txn_t * txn = slot_ctx->funk_txn; fd_funk_start_write( funk ); @@ -388,30 +346,7 @@ fd_snapshot_create_manifest( fd_exec_slot_ctx_t * slot_ctx ) { return -1; } fd_funk_end_write( funk ); - - // /****************** HACK TO GET THE ACC DB FROM THE NEWEST ***************/ - - // FILE * ahead_manifest = fopen("/data/ibhatt/dump/mainnet-254462437/254462442", "rb"); - // FD_TEST(ahead_manifest); - // fseek( ahead_manifest, 0, SEEK_END ); - // ulong file_sz = (ulong)ftell( ahead_manifest ); - // // rewind( ahead_manifest ); - // // uchar * buffer = fd_scratch_alloc( 8UL, file_sz ); - // // ulong fread_res = fread( buffer, 1, file_sz, ahead_manifest ); - // // FD_TEST(fread_res == file_sz); - - // // fclose( ahead_manifest ); - - - // fd_bincode_decode_ctx_t decode_ctx = { - // .data = buffer, - // .dataend = buffer + file_sz, - // .valloc = slot_ctx->valloc, - // }; - // fd_solana_manifest_t newest_manifest = {0}; - // int decode_res = fd_solana_manifest_decode( &newest_manifest, &decode_ctx ); - // FD_TEST(!decode_res); - + fd_txncache_register_root_slot( slot_ctx->status_cache, txn->xid.ul[0] ); fd_epoch_bank_t * epoch_bank = fd_exec_epoch_ctx_epoch_bank( slot_ctx->epoch_ctx ); @@ -585,9 +520,6 @@ fd_snapshot_create_manifest( fd_exec_slot_ctx_t * slot_ctx ) { /* TODO: Need to write out the snapshot hash to the name of the file as well. */ - /* NOW POPulate the STATUS CACHE */ - //fd_bank_slot_deltas_t slot_deltas; - FD_LOG_WARNING(("ACCOUNTS DB HEADER %lu %lu %lu", new_manifest.accounts_db.storages_len, new_manifest.accounts_db.version, new_manifest.accounts_db.slot));