Skip to content

Commit

Permalink
working snapshot in offline
Browse files Browse the repository at this point in the history
  • Loading branch information
@ibhatt-jumptrading
ibhatt-jumptrading committed Oct 31, 2024
1 parent 4c06baf commit 0237e61
Show file tree
Hide file tree
Showing 6 changed files with 166 additions and 8 deletions.
39 changes: 38 additions & 1 deletion src/flamenco/runtime/context/fd_exec_slot_ctx.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,7 @@ fd_exec_slot_ctx_recover_status_cache( fd_exec_slot_ctx_t * ctx,
ulong num_entries = 0;
for( ulong i = 0; i < slot_deltas->slot_deltas_len; i++ ) {
fd_slot_delta_t * slot_delta = &slot_deltas->slot_deltas[i];
//FD_LOG_WARNING(("SLOT DELTA %lu %lu %lu", slot_delta->slot, (ulong)slot_delta->is_root, slot_delta->slot_delta_vec_len));
FD_LOG_WARNING(("SLOT DELTA %lu %lu %lu", slot_delta->slot, (ulong)slot_delta->is_root, slot_delta->slot_delta_vec_len));
for( ulong j = 0; j < slot_delta->slot_delta_vec_len; j++ ) {
//FD_LOG_WARNING(("VALUE", slot_delta->slot_delta_vec[j].value.statuses/)
num_entries += slot_delta->slot_delta_vec[j].value.statuses_len;
Expand Down Expand Up @@ -522,6 +522,43 @@ fd_exec_slot_ctx_recover_status_cache( fd_exec_slot_ctx_t * ctx,
fd_txncache_set_txnhash_offset( ctx->status_cache, slot, blockhash->uc, pair->value.txn_idx );
}
}


// fd_bank_slot_deltas_t slot_deltas_new = {0};
// fd_txncache_get_entries( ctx->status_cache,
// &slot_deltas_new,
// ctx->valloc );

// ulong num_entries_two = 0;
// for( ulong i = 0; i < slot_deltas_new.slot_deltas_len; i++ ) {
// fd_slot_delta_t * slot_delta = &slot_deltas_new.slot_deltas[i];
// FD_LOG_WARNING(("SLOT DELTA %lu %lu %lu", slot_delta->slot, (ulong)slot_delta->is_root, slot_delta->slot_delta_vec_len));
// for( ulong j = 0; j < slot_delta->slot_delta_vec_len; j++ ) {
// //FD_LOG_WARNING(("VALUE", slot_delta->slot_delta_vec[j].value.statuses/)
// num_entries_two += slot_delta->slot_delta_vec[j].value.statuses_len;
// }
// }
// FD_LOG_WARNING(("NUM ENTRIES %lu %lu", num_entries, num_entries_two));

// ulong sz = fd_bank_slot_deltas_size( &slot_deltas_new );
// uchar * data = fd_valloc_malloc( ctx->valloc, 8UL, sz );
// fd_bincode_encode_ctx_t encode_ctx = {
// .data = data,
// .dataend = data + sz,
// };
// FD_TEST( 0==fd_bank_slot_deltas_encode( &slot_deltas_new, &encode_ctx ) );
// FD_LOG_WARNING(("ENCODED"));

// fd_bincode_decode_ctx_t decode_ctx = {
// .data = data,
// .dataend = data + sz,
// .valloc = ctx->valloc,
// };
// FD_TEST( 0==fd_bank_slot_deltas_decode( &slot_deltas_new, &decode_ctx ) );
// FD_LOG_WARNING(("DECODED"));



} FD_SCRATCH_SCOPE_END;
return ctx;
}
Expand Down
8 changes: 4 additions & 4 deletions src/flamenco/runtime/fd_hashes.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -549,10 +549,10 @@ fd_update_hash_bank_tpool( fd_exec_slot_ctx_t * slot_ctx,
continue;
}

//FD_LOG_WARNING(("REMOVING %s", FD_BASE58_ENC_32_ALLOCA(task_info->rec->pair.key)));;
(void)funk;
fd_account_meta_t * metadata = fd_funk_val( task_info->rec, fd_funk_wksp( funk ) );
FD_LOG_WARNING(("REMOVING %s %u %lu", FD_BASE58_ENC_32_ALLOCA(task_info->rec->pair.key), task_info->rec->val_sz, metadata->dlen));
// FD_LOG_WARNING(("REMOVING %s", FD_BASE58_ENC_32_ALLOCA(task_info->rec->pair.key)));;
// (void)funk;
// fd_account_meta_t * metadata = fd_funk_val( task_info->rec, fd_funk_wksp( funk ) );
// FD_LOG_WARNING(("REMOVING %s %u %lu", FD_BASE58_ENC_32_ALLOCA(task_info->rec->pair.key), task_info->rec->val_sz, metadata->dlen));

if( slot_ctx->slot_bank.slot < 254462500 ) {
fd_funk_rec_remove(funk, fd_funk_rec_modify(funk, task_info->rec), 1);
Expand Down
5 changes: 5 additions & 0 deletions src/flamenco/runtime/fd_runtime.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2917,6 +2917,11 @@ fd_runtime_block_eval_tpool( fd_exec_slot_ctx_t * slot_ctx,
ulong spad_cnt ) {
(void)scheduler;

if( slot_ctx->status_cache ) {
FD_LOG_WARNING(("REGISTERING SLOT %lu", slot_ctx->slot_bank.slot));
fd_txncache_register_root_slot( slot_ctx->status_cache, slot_ctx->slot_bank.slot );
}

// int err = fd_runtime_publish_old_txns( slot_ctx, capture_ctx, tpool );
// if( err != 0 ) {
// return err;
Expand Down
77 changes: 77 additions & 0 deletions src/flamenco/runtime/fd_txncache.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -996,3 +996,80 @@ fd_txncache_is_rooted_slot( fd_txncache_t * tc,
fd_rwlock_unread( tc->lock );
return 0;
}

int
fd_txncache_get_entries( fd_txncache_t * tc,
fd_bank_slot_deltas_t * slot_deltas,
fd_valloc_t valloc ) {

ulong num_entries = 0UL;
fd_rwlock_read( tc->lock );

FD_TEST( tc->root_slots_cnt == 300UL );

slot_deltas->slot_deltas_len = tc->root_slots_cnt;
slot_deltas->slot_deltas = fd_valloc_malloc( valloc, 8UL, tc->root_slots_cnt * sizeof(fd_slot_delta_t) );


fd_txncache_private_txnpage_t * txnpages = fd_txncache_get_txnpages( tc );
ulong * root_slots = fd_txncache_get_root_slots( tc );
for( ulong i=0UL; i<tc->root_slots_cnt; i++ ) {
ulong slot = root_slots[ i ];

slot_deltas->slot_deltas[ i ].slot = slot;
slot_deltas->slot_deltas[ i ].is_root = 1;
slot_deltas->slot_deltas[ i ].slot_delta_vec = fd_valloc_malloc( valloc, 8UL, 300UL * sizeof(fd_status_pair_t) );
slot_deltas->slot_deltas[ i ].slot_delta_vec_len = 0UL;
ulong slot_delta_vec_len = 0UL;

fd_txncache_private_slotcache_t * slotcache;
if( FD_UNLIKELY( FD_TXNCACHE_FIND_FOUND!=fd_txncache_find_slot( tc, slot, 0, &slotcache ) ) ) continue;

for( ulong j=0UL; j<300UL; j++ ) {
fd_txncache_private_slotblockcache_t * slotblockcache = &slotcache->blockcache[ j ];
if( FD_UNLIKELY( slotblockcache->txnhash_offset>=ULONG_MAX-1UL ) ) {
continue;
}
fd_status_pair_t * status_pair = &slot_deltas->slot_deltas[ i ].slot_delta_vec[ slot_delta_vec_len++ ];
fd_memcpy( &status_pair->hash, slotblockcache->blockhash, sizeof(fd_hash_t) );
status_pair->value.txn_idx = slotblockcache->txnhash_offset;


ulong num_statuses = 0UL;
for( ulong k=0UL; k<FD_TXNCACHE_SLOTCACHE_MAP_CNT; k++ ) {
uint head = slotblockcache->heads[ k ];
for( ; head!=UINT_MAX; head=txnpages[ head/FD_TXNCACHE_TXNS_PER_PAGE ].txns[ head%FD_TXNCACHE_TXNS_PER_PAGE ]->slotblockcache_next ) {
num_statuses++;
}
}
//ulong num_status_fp = num_statuses;

// status_pair->value.statuses_len = 0UL;
// status_pair->value.statuses = NULL;
fd_txn_result_t txn_result_default = {0};
fd_cache_status_t status_pair_default = {0};
status_pair->value.statuses_len = num_statuses;
status_pair->value.statuses = fd_valloc_malloc( valloc, 8UL, num_statuses * sizeof(fd_cache_status_t) );
for( ulong i=0UL; i<status_pair->value.statuses_len; i++ ) {
status_pair->value.statuses[ i ] = status_pair_default;
}


num_statuses = 0UL;
for( ulong k=0UL; k<FD_TXNCACHE_SLOTCACHE_MAP_CNT; k++ ) {
uint head = slotblockcache->heads[ k ];
for( ; head!=UINT_MAX; head=txnpages[ head/FD_TXNCACHE_TXNS_PER_PAGE ].txns[ head%FD_TXNCACHE_TXNS_PER_PAGE ]->slotblockcache_next ) {
fd_txncache_private_txn_t * txn = txnpages[ head/FD_TXNCACHE_TXNS_PER_PAGE ].txns[ head%FD_TXNCACHE_TXNS_PER_PAGE ];
fd_memcpy( status_pair->value.statuses[ num_statuses ].key_slice, txn->txnhash, 20 );
status_pair->value.statuses[ num_statuses++ ].result = txn_result_default;
}
}
}
slot_deltas->slot_deltas[ i ].slot_delta_vec_len = slot_delta_vec_len;
}

fd_rwlock_unread( tc->lock );
FD_LOG_WARNING(("GET THIS MANY ENTRIES %lu", num_entries));
return 0;

}
7 changes: 7 additions & 0 deletions src/flamenco/runtime/fd_txncache.h
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
#ifndef HEADER_fd_src_flamenco_runtime_txncache_h
#define HEADER_fd_src_flamenco_runtime_txncache_h

#include "../types/fd_types.h"

/* A txn cache is a concurrent map for saving the result (status) of
transactions that have executed. In addition to supporting fast
concurrent insertion and query of transaction results, the txn
Expand Down Expand Up @@ -394,6 +396,11 @@ int
fd_txncache_is_rooted_slot( fd_txncache_t * tc,
ulong slot );

int
fd_txncache_get_entries( fd_txncache_t * tc,
fd_bank_slot_deltas_t * bank_slot_deltas,
fd_valloc_t valloc );

FD_PROTOTYPES_END

#endif /* HEADER_fd_src_flamenco_runtime_txncache_h */
38 changes: 35 additions & 3 deletions src/flamenco/snapshot/fd_snapshot_create.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -586,7 +586,39 @@ fd_snapshot_create_manifest( fd_exec_slot_ctx_t * slot_ctx ) {
/* TODO: Need to write out the snapshot hash to the name of the file as well. */

/* NOW POPulate the STATUS CACHE */
//fd_bank_slot_deltas_t slot_deltas;
fd_bank_slot_deltas_t slot_deltas_new = {0};
fd_txncache_get_entries( slot_ctx->status_cache,
&slot_deltas_new,
slot_ctx->valloc );
ulong bank_slot_deltas_sz = fd_bank_slot_deltas_size( &slot_deltas_new );
uchar * out_status_cache = fd_valloc_malloc( slot_ctx->valloc, 8UL, bank_slot_deltas_sz );
fd_bincode_encode_ctx_t encode_status_cache = {
.data = out_status_cache,
.dataend = out_status_cache + bank_slot_deltas_sz,
};
FD_TEST( 0==fd_bank_slot_deltas_encode( &slot_deltas_new, &encode_status_cache ) );


fd_bincode_decode_ctx_t decode_asdf = {
.data = out_status_cache,
.dataend = out_status_cache + bank_slot_deltas_sz,
.valloc = slot_ctx->valloc,
};

fd_bank_slot_deltas_t two = {0};
FD_TEST( 0==fd_bank_slot_deltas_decode( &two, &decode_asdf ) );
FD_LOG_WARNING(("BANK SLOT DELTAS _SZ %lu", bank_slot_deltas_sz));



FILE * sc_file = fopen( "/data/ibhatt/dump/mainnet-254462437/own_snapshot/snapshots/status_cache", "wb" );
ulong bytes_written= fwrite( out_status_cache, 1, bank_slot_deltas_sz, sc_file );
if( bytes_written != bank_slot_deltas_sz ) {
FD_LOG_ERR(("FAILED TO WRITE OUT"));
}
fclose(sc_file);




FD_LOG_WARNING(("ACCOUNTS DB HEADER %lu %lu %lu", new_manifest.accounts_db.storages_len, new_manifest.accounts_db.version, new_manifest.accounts_db.slot));
Expand All @@ -602,8 +634,8 @@ fd_snapshot_create_manifest( fd_exec_slot_ctx_t * slot_ctx ) {
.dataend = out_manifest + new_manifest_sz + 1 };
FD_TEST( 0==fd_solana_manifest_serializable_encode( &new_manifest, &encode ) );

FILE * file = fopen( "/data/ibhatt/dump/mainnet-254462437/own_snapshot/snapshots/254462437/254462437", "wb" );
ulong bytes_written= fwrite( out_manifest, 1, new_manifest_sz, file );
FILE * file = fopen( "/data/ibhatt/dump/mainnet-254462437/own_snapshot/snapshots/254462437/254462437", "wb" );
bytes_written = fwrite( out_manifest, 1, new_manifest_sz, file );
if( bytes_written != new_manifest_sz ) {
FD_LOG_ERR(("FAILED TO WRITE OUT"));
}
Expand Down

0 comments on commit 0237e61

Please sign in to comment.