From 0ce5ef9c5c55e81bf196c29e843a69f0f2a14d5c Mon Sep 17 00:00:00 2001 From: Tom Pointon Date: Thu, 5 Dec 2024 21:05:06 +0000 Subject: [PATCH] replay, ledger: use same bank hash calculation code everywhere --- src/app/fdctl/run/tiles/fd_replay.c | 2 +- src/app/ledger/main.c | 4 +- src/flamenco/runtime/fd_hashes.c | 162 ---------------------------- src/flamenco/runtime/fd_hashes.h | 4 - src/flamenco/runtime/fd_runtime.c | 9 +- src/flamenco/runtime/fd_runtime.h | 7 +- 6 files changed, 12 insertions(+), 176 deletions(-) diff --git a/src/app/fdctl/run/tiles/fd_replay.c b/src/app/fdctl/run/tiles/fd_replay.c index 78091f8074..92b1d1caf8 100644 --- a/src/app/fdctl/run/tiles/fd_replay.c +++ b/src/app/fdctl/run/tiles/fd_replay.c @@ -1394,7 +1394,7 @@ init_snapshot( fd_replay_tile_ctx_t * ctx, read_snapshot( ctx, ctx->snapshot, ctx->incremental ); } - fd_runtime_read_genesis( ctx->slot_ctx, ctx->genesis, is_snapshot, ctx->capture_ctx ); + fd_runtime_read_genesis( ctx->slot_ctx, ctx->genesis, is_snapshot, ctx->capture_ctx, ctx->tpool ); ctx->epoch_ctx->bank_hash_cmp = ctx->bank_hash_cmp; init_after_snapshot( ctx ); diff --git a/src/app/ledger/main.c b/src/app/ledger/main.c index 2871e9d968..fda1eafcff 100644 --- a/src/app/ledger/main.c +++ b/src/app/ledger/main.c @@ -870,7 +870,7 @@ ingest( fd_ledger_args_t * args ) { } if( args->genesis ) { - fd_runtime_read_genesis( slot_ctx, args->genesis, args->snapshot != NULL, NULL ); + fd_runtime_read_genesis( slot_ctx, args->genesis, args->snapshot != NULL, NULL, args->tpool ); } if( !args->snapshot && (args->restore_funk != NULL || args->restore != NULL) ) { @@ -1006,7 +1006,7 @@ replay( fd_ledger_args_t * args ) { FD_LOG_NOTICE(( "imported %lu records from snapshot", fd_funk_rec_cnt( fd_funk_rec_map( funk, fd_funk_wksp( funk ) ) ) )); } if( args->genesis ) { - fd_runtime_read_genesis( args->slot_ctx, args->genesis, args->snapshot != NULL, NULL ); + fd_runtime_read_genesis( args->slot_ctx, args->genesis, args->snapshot != NULL, NULL, args->tpool ); } } else { FD_LOG_NOTICE(( "found funk with %lu records", rec_cnt )); diff --git a/src/flamenco/runtime/fd_hashes.c b/src/flamenco/runtime/fd_hashes.c index f62bd7c7ea..a2bcdaa279 100644 --- a/src/flamenco/runtime/fd_hashes.c +++ b/src/flamenco/runtime/fd_hashes.c @@ -654,168 +654,6 @@ fd_print_account_hashes( fd_exec_slot_ctx_t * slot_ctx, return 0; } -int -fd_update_hash_bank( fd_exec_slot_ctx_t * slot_ctx, - fd_capture_ctx_t * capture_ctx, - fd_hash_t * hash, - ulong signature_cnt ) { - - fd_acc_mgr_t * acc_mgr = slot_ctx->acc_mgr; - fd_funk_t * funk = acc_mgr->funk; - fd_funk_txn_t * txn = slot_ctx->funk_txn; - - /* Collect list of changed accounts to be added to bank hash */ - - - ulong rec_cnt = 0; - for( fd_funk_rec_t const * rec = fd_funk_txn_first_rec( funk, txn ); - NULL != rec; - rec = fd_funk_txn_next_rec( funk, rec ) ) { - - if( !fd_funk_key_is_acc( rec->pair.key ) ) continue; - if( !fd_funk_rec_is_modified( funk, rec ) ) continue; - - rec_cnt++; - } - /* Iterate over accounts that have been changed in the current - database transaction. */ - fd_pubkey_hash_pair_t * dirty_keys = fd_valloc_malloc( slot_ctx->valloc, FD_PUBKEY_HASH_PAIR_ALIGN, rec_cnt * FD_PUBKEY_HASH_PAIR_FOOTPRINT ); - fd_funk_rec_t const * * erase_recs = fd_valloc_malloc( slot_ctx->valloc, 8UL, rec_cnt * sizeof(fd_funk_rec_t *) ); - - ulong dirty_key_cnt = 0; - ulong erase_rec_cnt = 0; - - for( fd_funk_rec_t const * rec = fd_funk_txn_first_rec( funk, txn ); - NULL != rec; - rec = fd_funk_txn_next_rec( funk, rec ) ) { - - fd_pubkey_t const * acc_key = fd_type_pun_const( rec->pair.key[0].uc ); - - if( !fd_funk_key_is_acc( rec->pair.key ) ) continue; - if( !fd_funk_rec_is_modified( funk, rec ) ) continue; - - /* Get dirty account */ - - fd_funk_rec_t const * rec = NULL; - - int err = 0; - fd_account_meta_t const * acc_meta = fd_acc_mgr_view_raw( acc_mgr, txn, acc_key, &rec, &err, NULL); - if( FD_UNLIKELY( err!=FD_ACC_MGR_SUCCESS ) ) { - FD_LOG_ERR(( "failed to view account during bank hash" )); - } - uchar const * acc_data = (uchar *)acc_meta + acc_meta->hlen; - - /* Hash account */ - - fd_hash_t acc_hash[1]; - // TODO: talk to jsiegel about this - if (FD_UNLIKELY(acc_meta->info.lamports == 0)) { //!fd_acc_exists(_raw))) { - fd_memset( acc_hash->hash, 0, FD_HASH_FOOTPRINT ); - - /* If we erase records instantly, this causes problems with the - iterator. Instead, we will store away the record and erase - it later where appropriate. */ - erase_recs[erase_rec_cnt++] = rec; - } else { - // Maybe instead of going through the whole hash mechanism, we - // can find the parent funky record and just compare the data? - fd_hash_account_current( acc_hash->hash, NULL, acc_meta, acc_key->key, acc_data ); - } - - /* If hash didn't change, nothing to do */ - if( 0==memcmp( acc_hash->hash, acc_meta->hash, sizeof(fd_hash_t) ) ) { - if( acc_meta->slot == slot_ctx->slot_bank.slot ) { - /* no-op */ - } else { - continue; - } - } - - /* Upgrade to writable record */ - - // How the heck do we deal with new accounts? test that - FD_BORROWED_ACCOUNT_DECL(acc_rec); - acc_rec->const_rec = rec; - - err = fd_acc_mgr_modify( acc_mgr, txn, acc_key, 0, 0UL, acc_rec); - if( FD_UNLIKELY( err!=FD_ACC_MGR_SUCCESS ) ) { - FD_LOG_ERR(( "failed to modify account during bank hash" )); - } - - /* Update hash */ - - memcpy( acc_rec->meta->hash, acc_hash->hash, sizeof(fd_hash_t) ); - acc_rec->meta->slot = slot_ctx->slot_bank.slot; - - /* Logging ... */ - FD_LOG_DEBUG(( "fd_acc_mgr_update_hash: %s " - "slot: %lu " - "lamports: %lu " - "owner: %s " - "executable: %s, " - "rent_epoch: %lu, " - "data_len: %lu", - FD_BASE58_ENC_32_ALLOCA( acc_key ), - slot_ctx->slot_bank.slot, - acc_rec->meta->info.lamports, - FD_BASE58_ENC_32_ALLOCA( acc_rec->meta->info.owner ), - acc_rec->meta->info.executable ? "true" : "false", - acc_rec->meta->info.rent_epoch, - acc_rec->meta->dlen )); - - - /* Add account to "dirty keys" list, which will be added to the - bank hash. */ - - fd_pubkey_hash_pair_t * dirty_entry = &dirty_keys[dirty_key_cnt++]; - dirty_entry->rec = rec; - dirty_entry->hash = (fd_hash_t const *)acc_rec->meta->hash; - - /* Add to capture */ - if( capture_ctx != NULL && capture_ctx->capture != NULL ) { - err = fd_solcap_write_account( - capture_ctx->capture, - acc_key->uc, - &acc_rec->meta->info, - acc_data, - acc_rec->meta->dlen, - acc_hash->hash ); - } - FD_TEST( err==0 ); - } - - /* Sort and hash "dirty keys" to the accounts delta hash. */ - - // FD_LOG_DEBUG(("slot %ld, dirty %ld", slot_ctx->slot_bank.slot, dirty_key_cnt)); - - slot_ctx->signature_cnt = signature_cnt; - fd_hash_bank( slot_ctx, capture_ctx, hash, dirty_keys, dirty_key_cnt ); - -// if( FD_FEATURE_ACTIVE( slot_ctx, lattice_account_hash ) ) { -// // Sanity-check LT Hash -// fd_accounts_check_lthash( slot_ctx ); - - // Check that the old account_delta_hash is the same as the lthash -// FD_TEST( 0==memcmp( slot_ctx->slot_bank.lthash.lthash, slot_ctx->account_delta_hash.hash, sizeof(fd_hash_t) ) ); -// } - - fd_epoch_bank_t * epoch_bank = fd_exec_epoch_ctx_epoch_bank( slot_ctx->epoch_ctx ); - if (slot_ctx->slot_bank.slot >= epoch_bank->eah_start_slot) { - fd_accounts_hash( slot_ctx, NULL, &slot_ctx->slot_bank.epoch_account_hash ); - epoch_bank->eah_start_slot = ULONG_MAX; - } - - for (ulong i = 0; i < erase_rec_cnt; i++) { - fd_funk_rec_t const * erase_rec = erase_recs[i]; - fd_funk_rec_remove(funk, fd_funk_rec_modify(funk, erase_rec), 1); - } - - fd_valloc_free( slot_ctx->valloc, dirty_keys ); - fd_valloc_free( slot_ctx->valloc, erase_recs ); - - return FD_EXECUTOR_INSTR_SUCCESS; -} - void const * fd_hash_account( uchar hash[ static 32 ], fd_lthash_value_t * lthash, diff --git a/src/flamenco/runtime/fd_hashes.h b/src/flamenco/runtime/fd_hashes.h index 40e5d5e034..2ebecb531e 100644 --- a/src/flamenco/runtime/fd_hashes.h +++ b/src/flamenco/runtime/fd_hashes.h @@ -16,10 +16,6 @@ typedef struct fd_pubkey_hash_pair fd_pubkey_hash_pair_t; FD_PROTOTYPES_BEGIN -int fd_update_hash_bank( fd_exec_slot_ctx_t * slot_ctx, - fd_capture_ctx_t * capture_ctx, - fd_hash_t * hash, - ulong signature_cnt ); int fd_update_hash_bank_tpool( fd_exec_slot_ctx_t * slot_ctx, fd_capture_ctx_t * capture_ctx, diff --git a/src/flamenco/runtime/fd_runtime.c b/src/flamenco/runtime/fd_runtime.c index 280b7e271a..abf1856a5a 100644 --- a/src/flamenco/runtime/fd_runtime.c +++ b/src/flamenco/runtime/fd_runtime.c @@ -4544,7 +4544,7 @@ int fd_runtime_sysvar_cache_load( fd_exec_slot_ctx_t * slot_ctx ) { } int -fd_runtime_process_genesis_block( fd_exec_slot_ctx_t * slot_ctx, fd_capture_ctx_t * capture_ctx ) { +fd_runtime_process_genesis_block( fd_exec_slot_ctx_t * slot_ctx, fd_capture_ctx_t * capture_ctx, fd_tpool_t * tpool ) { ulong hashcnt_per_slot = slot_ctx->epoch_ctx->epoch_bank.hashes_per_tick * slot_ctx->epoch_ctx->epoch_bank.ticks_per_slot; while(hashcnt_per_slot--) { fd_sha256_hash( slot_ctx->slot_bank.poh.uc, 32UL, slot_ctx->slot_bank.poh.uc ); @@ -4560,7 +4560,7 @@ fd_runtime_process_genesis_block( fd_exec_slot_ctx_t * slot_ctx, fd_capture_ctx_ fd_runtime_freeze( slot_ctx ); /* sort and update bank hash */ - int result = fd_update_hash_bank( slot_ctx, capture_ctx, &slot_ctx->slot_bank.banks_hash, slot_ctx->signature_cnt ); + int result = fd_update_hash_bank_tpool( slot_ctx, capture_ctx, &slot_ctx->slot_bank.banks_hash, slot_ctx->signature_cnt, tpool ); if (result != FD_EXECUTOR_INSTR_SUCCESS) { FD_LOG_ERR(("Failed to update bank hash with error=%d", result)); } @@ -4576,7 +4576,8 @@ void fd_runtime_read_genesis( fd_exec_slot_ctx_t * slot_ctx, char const * genesis_filepath, uchar is_snapshot, - fd_capture_ctx_t * capture_ctx + fd_capture_ctx_t * capture_ctx, + fd_tpool_t * tpool ) { if ( strlen( genesis_filepath ) == 0 ) return; @@ -4663,7 +4664,7 @@ fd_runtime_read_genesis( fd_exec_slot_ctx_t * slot_ctx, slot_ctx->slot_bank.slot = 0UL; - int err = fd_runtime_process_genesis_block( slot_ctx, capture_ctx ); + int err = fd_runtime_process_genesis_block( slot_ctx, capture_ctx, tpool ); if( FD_UNLIKELY( err ) ) { FD_LOG_ERR(( "Genesis slot 0 execute failed with error %d", err )); } diff --git a/src/flamenco/runtime/fd_runtime.h b/src/flamenco/runtime/fd_runtime.h index 3952447c37..b70dfb8bfe 100644 --- a/src/flamenco/runtime/fd_runtime.h +++ b/src/flamenco/runtime/fd_runtime.h @@ -277,9 +277,10 @@ fd_runtime_collect_rent_accounts_prune( ulong slot, void fd_runtime_read_genesis( fd_exec_slot_ctx_t * slot_ctx, - char const * genesis_filepath, - uchar is_snapshot, - fd_capture_ctx_t * capture_ctx ); + char const * genesis_filepath, + uchar is_snapshot, + fd_capture_ctx_t * capture_ctx, + fd_tpool_t * tpool ); void fd_runtime_checkpt( fd_capture_ctx_t * capture_ctx,