Skip to content

Commit

Permalink
fd_pack: fix (once again) extra transaction storage
Browse files Browse the repository at this point in the history
  • Loading branch information
@ptaffet-jump @mmcgee-jump
ptaffet-jump authored and mmcgee-jump committed Apr 11, 2024
1 parent 4c0f1db commit 5c308f2
Show file tree
Hide file tree
Showing 4 changed files with 47 additions and 18 deletions.
13 changes: 10 additions & 3 deletions src/app/fdctl/run/tiles/fd_pack.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,9 @@ after_credit( void * _ctx,
ctx->max_pending_transactions-fd_pack_avail_txn_cnt( ctx->pack ) );
for( ulong i=0UL; i<qty_to_insert; i++ ) {
fd_txn_p_t * spot = fd_pack_insert_txn_init( ctx->pack );
*spot = *extra_txn_deq_remove_head( ctx->extra_txn_deq );
*spot = *extra_txn_deq_peek_head( ctx->extra_txn_deq );
extra_txn_deq_remove_head( ctx->extra_txn_deq );


long insert_duration = -fd_tickcount();
int result = fd_pack_insert_txn_fini( ctx->pack, spot, (ulong)now+TIME_OFFSET );
Expand All @@ -361,6 +363,7 @@ after_credit( void * _ctx,
fd_histf_sample( ctx->insert_duration, (ulong)insert_duration );
if( FD_LIKELY( result>=0 ) ) ctx->last_successful_insert = now;
}
FD_MCNT_INC( PACK, TRANSACTION_INSERTED_FROM_EXTRA, qty_to_insert );
}

/* Did we send the maximum allowed microblocks? Then end the slot. */
Expand Down Expand Up @@ -430,9 +433,13 @@ during_frag( void * _ctx,
ctx->cur_spot = fd_pack_insert_txn_init( ctx->pack );
ctx->insert_to_extra = 0;
} else {
if( FD_UNLIKELY( extra_txn_deq_full( ctx->extra_txn_deq ) ) ) extra_txn_deq_remove_head( ctx->extra_txn_deq );
ctx->cur_spot = extra_txn_deq_insert_tail( ctx->extra_txn_deq );
if( FD_UNLIKELY( extra_txn_deq_full( ctx->extra_txn_deq ) ) ) {
extra_txn_deq_remove_head( ctx->extra_txn_deq );
FD_MCNT_INC( PACK, TRANSACTION_DROPPED_FROM_EXTRA, 1UL );
}
ctx->cur_spot = extra_txn_deq_peek_tail( extra_txn_deq_insert_tail( ctx->extra_txn_deq ) );
ctx->insert_to_extra = 1;
FD_MCNT_INC( PACK, TRANSACTION_INSERTED_TO_EXTRA, 1UL );
}

ulong payload_sz;
Expand Down
3 changes: 3 additions & 0 deletions src/disco/metrics/generated/fd_metrics_pack.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ const fd_metrics_meta_t FD_METRICS_PACK[FD_METRICS_PACK_TOTAL] = {
DECLARE_METRIC_COUNTER( PACK, METRIC_TIMING_TXN_NO_BANK_LEADER_MICROBLOCK ),
DECLARE_METRIC_COUNTER( PACK, METRIC_TIMING_NO_TXN_BANK_LEADER_MICROBLOCK ),
DECLARE_METRIC_COUNTER( PACK, METRIC_TIMING_TXN_BANK_LEADER_MICROBLOCK ),
DECLARE_METRIC_COUNTER( PACK, TRANSACTION_DROPPED_FROM_EXTRA ),
DECLARE_METRIC_COUNTER( PACK, TRANSACTION_INSERTED_TO_EXTRA ),
DECLARE_METRIC_COUNTER( PACK, TRANSACTION_INSERTED_FROM_EXTRA ),
DECLARE_METRIC_GAUGE( PACK, AVAILABLE_TRANSACTIONS ),
DECLARE_METRIC_GAUGE( PACK, AVAILABLE_VOTE_TRANSACTIONS ),
DECLARE_METRIC_GAUGE( PACK, PENDING_TRANSACTIONS_HEAP_SIZE ),
Expand Down
45 changes: 30 additions & 15 deletions src/disco/metrics/generated/fd_metrics_pack.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,74 +195,89 @@
#define FD_METRICS_COUNTER_PACK_METRIC_TIMING_TXN_BANK_LEADER_MICROBLOCK_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_METRIC_TIMING_TXN_BANK_LEADER_MICROBLOCK_DESC "Time in nanos spent in each state (Pack scheduled a non-empty microblock)"

#define FD_METRICS_GAUGE_PACK_AVAILABLE_TRANSACTIONS_OFF (273UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_DROPPED_FROM_EXTRA_OFF (273UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_DROPPED_FROM_EXTRA_NAME "pack_transaction_dropped_from_extra"
#define FD_METRICS_COUNTER_PACK_TRANSACTION_DROPPED_FROM_EXTRA_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_DROPPED_FROM_EXTRA_DESC "Transactions dropped from the extra transaction storage because it was full"

#define FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_TO_EXTRA_OFF (274UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_TO_EXTRA_NAME "pack_transaction_inserted_to_extra"
#define FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_TO_EXTRA_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_TO_EXTRA_DESC "Transactions inserted into the extra transaction storage because pack's primary storage was full"

#define FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_FROM_EXTRA_OFF (275UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_FROM_EXTRA_NAME "pack_transaction_inserted_from_extra"
#define FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_FROM_EXTRA_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_INSERTED_FROM_EXTRA_DESC "Transactions pulled from the extra transaction storage and inserted into pack's primary storage"

#define FD_METRICS_GAUGE_PACK_AVAILABLE_TRANSACTIONS_OFF (276UL)
#define FD_METRICS_GAUGE_PACK_AVAILABLE_TRANSACTIONS_NAME "pack_available_transactions"
#define FD_METRICS_GAUGE_PACK_AVAILABLE_TRANSACTIONS_TYPE (FD_METRICS_TYPE_GAUGE)
#define FD_METRICS_GAUGE_PACK_AVAILABLE_TRANSACTIONS_DESC "The total number of pending transactions in pack's pool that are available to be scheduled"

#define FD_METRICS_GAUGE_PACK_AVAILABLE_VOTE_TRANSACTIONS_OFF (274UL)
#define FD_METRICS_GAUGE_PACK_AVAILABLE_VOTE_TRANSACTIONS_OFF (277UL)
#define FD_METRICS_GAUGE_PACK_AVAILABLE_VOTE_TRANSACTIONS_NAME "pack_available_vote_transactions"
#define FD_METRICS_GAUGE_PACK_AVAILABLE_VOTE_TRANSACTIONS_TYPE (FD_METRICS_TYPE_GAUGE)
#define FD_METRICS_GAUGE_PACK_AVAILABLE_VOTE_TRANSACTIONS_DESC "The number of pending simple vote transactions in pack's pool that are available to be scheduled"

#define FD_METRICS_GAUGE_PACK_PENDING_TRANSACTIONS_HEAP_SIZE_OFF (275UL)
#define FD_METRICS_GAUGE_PACK_PENDING_TRANSACTIONS_HEAP_SIZE_OFF (278UL)
#define FD_METRICS_GAUGE_PACK_PENDING_TRANSACTIONS_HEAP_SIZE_NAME "pack_pending_transactions_heap_size"
#define FD_METRICS_GAUGE_PACK_PENDING_TRANSACTIONS_HEAP_SIZE_TYPE (FD_METRICS_TYPE_GAUGE)
#define FD_METRICS_GAUGE_PACK_PENDING_TRANSACTIONS_HEAP_SIZE_DESC "The maximum number of pending transactions that pack can consider. This value is fixed at Firedancer startup but is a useful reference for AvailableTransactions and AvailableVoteTransactions."

#define FD_METRICS_COUNTER_PACK_MICROBLOCK_PER_BLOCK_LIMIT_OFF (276UL)
#define FD_METRICS_COUNTER_PACK_MICROBLOCK_PER_BLOCK_LIMIT_OFF (279UL)
#define FD_METRICS_COUNTER_PACK_MICROBLOCK_PER_BLOCK_LIMIT_NAME "pack_microblock_per_block_limit"
#define FD_METRICS_COUNTER_PACK_MICROBLOCK_PER_BLOCK_LIMIT_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_MICROBLOCK_PER_BLOCK_LIMIT_DESC "The number of times pack did not pack a microblock because the limit on microblocks/block had been reached"

#define FD_METRICS_COUNTER_PACK_DATA_PER_BLOCK_LIMIT_OFF (277UL)
#define FD_METRICS_COUNTER_PACK_DATA_PER_BLOCK_LIMIT_OFF (280UL)
#define FD_METRICS_COUNTER_PACK_DATA_PER_BLOCK_LIMIT_NAME "pack_data_per_block_limit"
#define FD_METRICS_COUNTER_PACK_DATA_PER_BLOCK_LIMIT_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_DATA_PER_BLOCK_LIMIT_DESC "The number of times pack did not pack a microblock because it reached reached the data per block limit at the start of trying to schedule a microblock"

#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_OFF (278UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_OFF (281UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_CNT (6UL)

#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_TAKEN_OFF (278UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_TAKEN_OFF (281UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_TAKEN_NAME "pack_transaction_schedule_taken"
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_TAKEN_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_TAKEN_DESC "Result of trying to consider a transaction for scheduling (Pack included the transaction in the microblock)"

#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_CU_LIMIT_OFF (279UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_CU_LIMIT_OFF (282UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_CU_LIMIT_NAME "pack_transaction_schedule_cu_limit"
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_CU_LIMIT_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_CU_LIMIT_DESC "Result of trying to consider a transaction for scheduling (Pack skipped the transaction because it would have exceeded the block CU limit)"

#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_FAST_PATH_OFF (280UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_FAST_PATH_OFF (283UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_FAST_PATH_NAME "pack_transaction_schedule_fast_path"
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_FAST_PATH_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_FAST_PATH_DESC "Result of trying to consider a transaction for scheduling (Pack skipped the transaction because of account conflicts using the fast bitvector check)"

#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_BYTE_LIMIT_OFF (281UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_BYTE_LIMIT_OFF (284UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_BYTE_LIMIT_NAME "pack_transaction_schedule_byte_limit"
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_BYTE_LIMIT_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_BYTE_LIMIT_DESC "Result of trying to consider a transaction for scheduling (Pack skipped the transaction because it would have exceeded the block data size limit)"

#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_WRITE_COST_OFF (282UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_WRITE_COST_OFF (285UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_WRITE_COST_NAME "pack_transaction_schedule_write_cost"
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_WRITE_COST_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_WRITE_COST_DESC "Result of trying to consider a transaction for scheduling (Pack skipped the transaction because it would have caused a writable account to exceed the per-account block write cost limit)"

#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_SLOW_PATH_OFF (283UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_SLOW_PATH_OFF (286UL)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_SLOW_PATH_NAME "pack_transaction_schedule_slow_path"
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_SLOW_PATH_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_TRANSACTION_SCHEDULE_SLOW_PATH_DESC "Result of trying to consider a transaction for scheduling (Pack skipped the transaction because of account conflicts using the full slow check)"

#define FD_METRICS_COUNTER_PACK_DELETE_MISSED_OFF (284UL)
#define FD_METRICS_COUNTER_PACK_DELETE_MISSED_OFF (287UL)
#define FD_METRICS_COUNTER_PACK_DELETE_MISSED_NAME "pack_delete_missed"
#define FD_METRICS_COUNTER_PACK_DELETE_MISSED_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_DELETE_MISSED_DESC "Count of attempts to delete a transaction that wasn't found"

#define FD_METRICS_COUNTER_PACK_DELETE_HIT_OFF (285UL)
#define FD_METRICS_COUNTER_PACK_DELETE_HIT_OFF (288UL)
#define FD_METRICS_COUNTER_PACK_DELETE_HIT_NAME "pack_delete_hit"
#define FD_METRICS_COUNTER_PACK_DELETE_HIT_TYPE (FD_METRICS_TYPE_COUNTER)
#define FD_METRICS_COUNTER_PACK_DELETE_HIT_DESC "Count of attempts to delete a transaction that was found and deleted"


#define FD_METRICS_PACK_TOTAL (48UL)
#define FD_METRICS_PACK_TOTAL (51UL)
extern const fd_metrics_meta_t FD_METRICS_PACK[FD_METRICS_PACK_TOTAL];
4 changes: 4 additions & 0 deletions src/disco/metrics/metrics.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,10 @@ metric introduced.
<counter name="TransactionInserted" enum="PackTxnInsertReturn" summary="Result of inserting a transaction into the pack object" />
<counter name="MetricTiming" enum="PackTimingState" summary="Time in nanos spent in each state" />

<counter name="TransactionDroppedFromExtra" summary="Transactions dropped from the extra transaction storage because it was full" />
<counter name="TransactionInsertedToExtra" summary="Transactions inserted into the extra transaction storage because pack's primary storage was full" />
<counter name="TransactionInsertedFromExtra" summary="Transactions pulled from the extra transaction storage and inserted into pack's primary storage" />

<gauge name="AvailableTransactions" summary="The total number of pending transactions in pack's pool that are available to be scheduled" />
<gauge name="AvailableVoteTransactions" summary="The number of pending simple vote transactions in pack's pool that are available to be scheduled" />
<gauge name="PendingTransactionsHeapSize" summary="The maximum number of pending transactions that pack can consider. This value is fixed at Firedancer startup but is a useful reference for AvailableTransactions and AvailableVoteTransactions." />
Expand Down

0 comments on commit 5c308f2

Please sign in to comment.