flamenco: size out input regions and migrate them to spad

src/app/fdctl/run/tiles/fd_replay.c

@@ -280,7 +280,7 @@ scratch_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) {
     l = FD_LAYOUT_APPEND( l, FD_BMTREE_COMMIT_ALIGN, FD_BMTREE_COMMIT_FOOTPRINT(0) );
   }
   l = FD_LAYOUT_APPEND( l, FD_SCRATCH_ALIGN_DEFAULT, tile->replay.tpool_thread_count * TPOOL_WORKER_MEM_SZ );
-  l = FD_LAYOUT_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * fd_spad_footprint( MAX_TX_ACCOUNT_LOCKS * fd_ulong_align_up( FD_ACC_TOT_SZ_MAX, FD_ACCOUNT_REC_ALIGN ) ) );
+  l = FD_LAYOUT_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_DEFAULT );
   l = FD_LAYOUT_APPEND( l, fd_scratch_smem_align(), fd_scratch_smem_footprint( SCRATCH_MAX   ) );
   l = FD_LAYOUT_APPEND( l, fd_scratch_fmem_align(), fd_scratch_fmem_footprint( SCRATCH_DEPTH ) );
   l = FD_LAYOUT_FINI  ( l, scratch_align() );
@@ -1595,7 +1595,7 @@ unprivileged_init( fd_topo_t *      topo,
     ctx->bmtree[i]           = FD_SCRATCH_ALLOC_APPEND( l, FD_BMTREE_COMMIT_ALIGN, FD_BMTREE_COMMIT_FOOTPRINT(0) );
   }
   void * tpool_worker_mem    = FD_SCRATCH_ALLOC_APPEND( l, FD_SCRATCH_ALIGN_DEFAULT, tile->replay.tpool_thread_count * TPOOL_WORKER_MEM_SZ );
-  ulong  thread_spad_size    = fd_spad_footprint( MAX_TX_ACCOUNT_LOCKS * fd_ulong_align_up( FD_ACC_TOT_SZ_MAX, FD_ACCOUNT_REC_ALIGN ) );
+  ulong  thread_spad_size    = FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_DEFAULT;
   void * spad_mem            = FD_SCRATCH_ALLOC_APPEND( l, fd_spad_align(), tile->replay.tpool_thread_count * thread_spad_size );
   void * scratch_smem        = FD_SCRATCH_ALLOC_APPEND( l, fd_scratch_smem_align(), fd_scratch_smem_footprint( SCRATCH_MAX   ) );
   void * scratch_fmem        = FD_SCRATCH_ALLOC_APPEND( l, fd_scratch_fmem_align(), fd_scratch_fmem_footprint( SCRATCH_DEPTH ) );

src/app/ledger/main.c

@@ -442,7 +442,7 @@ fd_ledger_main_setup( fd_ledger_args_t * args ) {
 
   args->spad_cnt = fd_tpool_worker_cnt( args->tpool );
   for( ulong i=0UL; i<args->spad_cnt; i++ ) {
-    ulong       total_mem_sz = fd_spad_footprint( MAX_TX_ACCOUNT_LOCKS * fd_ulong_align_up( FD_ACC_TOT_SZ_MAX, FD_ACCOUNT_REC_ALIGN ) );
+    ulong       total_mem_sz = FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_FUZZ;
     uchar *     mem          = fd_wksp_alloc_laddr( args->wksp, FD_SPAD_ALIGN, total_mem_sz, 999UL );
     fd_spad_t * spad         = fd_spad_join( fd_spad_new( mem, total_mem_sz ) );
     if( FD_UNLIKELY( !spad ) ) {

src/flamenco/runtime/context/fd_exec_instr_ctx.h

@@ -25,7 +25,7 @@ struct __attribute__((aligned(8UL))) fd_exec_instr_ctx {
 
   fd_funk_txn_t * funk_txn;
   fd_acc_mgr_t *  acc_mgr;
-  fd_valloc_t     valloc;
+  fd_valloc_t     valloc; /* Scratch-backed valloc TODO: migrate to transaction spad */
 
   /* Most instructions log the base58 program id multiple times, so it's
      convenient to compute it once and reuse it. */

src/flamenco/runtime/context/fd_exec_txn_ctx.h

@@ -64,7 +64,8 @@ struct __attribute__((aligned(8UL))) fd_exec_txn_ctx {
 
   fd_funk_txn_t *       funk_txn;
   fd_acc_mgr_t *        acc_mgr;
-  fd_valloc_t           valloc;
+  fd_valloc_t           valloc;                                      /* Workspace-backed valloc TODO: migrate to spad */
+  fd_spad_t *           spad;                                        /* Sized out to handle the worst case footprint of single transaction execution. */
 
   ulong                 paid_fees;
   ulong                 compute_unit_limit;                          /* Compute unit limit for this transaction. */
@@ -126,8 +127,6 @@ struct __attribute__((aligned(8UL))) fd_exec_txn_ctx {
   int exec_err;
   int exec_err_kind;
 
-  fd_spad_t * spad;
-
   /* The has_program_id flag is used to indicate if the current transaction has valid program indices or not.
      It will be set in fd_executor_load_transaction_accounts similar to how program_indices is used in
      load_transaction_accounts on the agave side */

src/flamenco/runtime/fd_executor.c

@@ -1129,6 +1129,7 @@ fd_instr_stack_pop( fd_exec_txn_ctx_t *       txn_ctx,
 int
 fd_execute_instr( fd_exec_txn_ctx_t * txn_ctx,
                   fd_instr_info_t *   instr ) {
+  FD_RUNTIME_SPAD_FRAME_BEGIN( txn_ctx->spad, txn_ctx ) {
   FD_SCRATCH_SCOPE_BEGIN {
     fd_exec_instr_ctx_t * parent = NULL;
     if( txn_ctx->instr_stack_sz ) {
@@ -1211,6 +1212,7 @@ fd_execute_instr( fd_exec_txn_ctx_t * txn_ctx,
 
     return exec_result;
   } FD_SCRATCH_SCOPE_END;
+  } FD_RUNTIME_SPAD_FRAME_END;
 }
 
 void

src/flamenco/runtime/fd_runtime.c

@@ -1123,7 +1123,7 @@ fd_runtime_prepare_execute_finalize_txn( fd_exec_slot_ctx_t *         slot_ctx,
                                          fd_capture_ctx_t *           capture_ctx,
                                          fd_txn_p_t *                 txn,
                                          fd_execute_txn_task_info_t * task_info ) {
-
+  FD_SPAD_FRAME_BEGIN( spad ) {
   FD_SCRATCH_SCOPE_BEGIN {
 
   int res = 0;
@@ -1191,6 +1191,7 @@ fd_runtime_prepare_execute_finalize_txn( fd_exec_slot_ctx_t *         slot_ctx,
   return res;
 
   } FD_SCRATCH_SCOPE_END;
+  } FD_SPAD_FRAME_END;
 }
 
 
@@ -1776,7 +1777,7 @@ fd_runtime_finalize_txns_tpool( fd_exec_slot_ctx_t *         slot_ctx,
       results       = fd_scratch_alloc( alignof(uchar), txn_cnt * sizeof(uchar) );
     }
 
-    fd_borrowed_account_t * * accounts_to_save = fd_scratch_alloc( 8UL, 128UL * txn_cnt * sizeof(fd_borrowed_account_t *) );
+    fd_borrowed_account_t * * accounts_to_save = fd_scratch_alloc( 8UL, MAX_TX_ACCOUNT_LOCKS * txn_cnt * sizeof(fd_borrowed_account_t *) );
     ulong acc_idx = 0UL;
     for( ulong txn_idx=0UL; txn_idx<txn_cnt; txn_idx++ ) {
       /* Transaction was skipped due to preparation failure. */
@@ -2138,6 +2139,13 @@ fd_runtime_execute_txns_in_waves_tpool( fd_exec_slot_ctx_t * slot_ctx,
         next_incomplete_txn_idxs = temp_incomplete_txn_idxs;
         incomplete_txn_idxs_cnt  = next_incomplete_txn_idxs_cnt;
 
+        for( ulong i=0UL; i<spad_cnt; i++ ) {
+          /* Borrowed accounts are allocated during prep and need to
+             persist till the end of finalize. This initial frame will
+             be holding that. */
+          fd_spad_push( spads[ i ] );
+        }
+
         // Dump txns in waves
         if( dump_txn ) {
           for( ulong i = 0; i < wave_task_infos_cnt; ++i ) {
@@ -2165,9 +2173,17 @@ fd_runtime_execute_txns_in_waves_tpool( fd_exec_slot_ctx_t * slot_ctx,
           FD_LOG_ERR(("Fail finalize"));
         }
 
-        /* Resetting the spad is a O(1) operation */
         for( ulong i=0UL; i<spad_cnt; i++ ) {
-          fd_spad_reset( spads[i] );
+          /* The first frame which holds borrowed accounts can be
+             pretty big, and there could be additional dynamic
+             allocations during finalize. */
+          if( FD_UNLIKELY( fd_spad_verify( spads[ i ] ) ) ) {
+            FD_LOG_ERR(( "spad corrupted or overflown" ));
+          }
+          /* We indiscriminately pushed a frame to every spad.
+             So it should be safe to indiscriminately pop here. */
+          fd_spad_pop( spads[ i ] );
+          FD_TEST( fd_spad_frame_used( spads[ i ] )==0 );
         }
 
         // wave_time += fd_log_wallclock();

src/flamenco/runtime/fd_runtime.h

@@ -13,6 +13,7 @@
 #include "context/fd_exec_epoch_ctx.h"
 #include "context/fd_exec_slot_ctx.h"
 #include "context/fd_capture_ctx.h"
+#include "context/fd_exec_txn_ctx.h"
 #include "info/fd_block_info.h"
 #include "info/fd_instr_info.h"
 #include "../gossip/fd_gossip.h"
@@ -95,6 +96,103 @@ FD_STATIC_ASSERT( FD_ACCOUNT_REC_ALIGN>=FD_ACCOUNT_REC_DATA_ALIGN, account_rec_d
 FD_STATIC_ASSERT( (offsetof(fd_account_rec_t, meta)%FD_ACCOUNT_META_ALIGN)==0,     account_rec_meta_offset );
 FD_STATIC_ASSERT( (offsetof(fd_account_rec_t, data)%FD_ACCOUNT_REC_DATA_ALIGN)==0, account_rec_data_offset );
 
+#define MAX_PERMITTED_DATA_INCREASE (10240UL) // 10KB
+#define FD_BPF_ALIGN_OF_U128        (8UL    )
+FD_STATIC_ASSERT( FD_BPF_ALIGN_OF_U128==FD_ACCOUNT_REC_DATA_ALIGN, input_data_align );
+#define FD_RUNTIME_INPUT_REGION_ALLOC_ALIGN_UP (16UL)
+
+/******** These macros try to bound out memory footprint ********/
+
+/* The tight upper bound on borrowed account footprint over the
+   execution of a single transaction. */
+#define FD_RUNTIME_BORROWED_ACCOUNT_FOOTPRINT (fd_spad_footprint( MAX_TX_ACCOUNT_LOCKS * fd_ulong_align_up( FD_ACC_TOT_SZ_MAX, FD_ACCOUNT_REC_ALIGN ) ))
+/* The tight-ish upper bound on input region footprint over the
+   execution of a single transaction. See input serialization code for
+   reference: fd_bpf_loader_serialization.c
+
+   This bound is based off of the transaction MTU. We consider the
+   question of what kind of transaction one would construct to
+   maximally bloat the input region.
+   The worst case scenario is when every nested instruction references
+   all unique accounts in the transaction. A transaction can lock a max
+   of MAX_TX_ACCOUNT_LOCKS accounts. Then all remaining input account
+   references are going to be duplicates, which cost 1 byte to specify
+   offset in payload, and which cost 8 bytes during serialization. Then
+   there would be 0 bytes of instruction data, because they exist byte
+   for byte in the raw payload, which is not a worthwhile bloat factor.
+   */
+#define FD_RUNTIME_INPUT_REGION_UNIQUE_ACCOUNT_FOOTPRINT(direct_mapping)                                                                                      \
+                                                        (1UL                         /* dup byte          */                                                + \
+                                                         sizeof(uchar)               /* is_signer         */                                                + \
+                                                         sizeof(uchar)               /* is_writable       */                                                + \
+                                                         sizeof(uchar)               /* executable        */                                                + \
+                                                         sizeof(uint)                /* original_data_len */                                                + \
+                                                         sizeof(fd_pubkey_t)         /* key               */                                                + \
+                                                         sizeof(fd_pubkey_t)         /* owner             */                                                + \
+                                                         sizeof(ulong)               /* lamports          */                                                + \
+                                                         sizeof(ulong)               /* data len          */                                                + \
+                                                         (direct_mapping ? FD_BPF_ALIGN_OF_U128 : fd_ulong_align_up( FD_ACC_SZ_MAX, FD_BPF_ALIGN_OF_U128 )) + \
+                                                         MAX_PERMITTED_DATA_INCREASE                                                                        + \
+                                                         sizeof(ulong))              /* rent_epoch        */
+#define FD_RUNTIME_INPUT_REGION_INSN_FOOTPRINT(account_lock_limit, direct_mapping)                                                                       \
+                                              (fd_ulong_align_up( (sizeof(ulong)         /* acct_cnt       */                                          + \
+                                                                   account_lock_limit*FD_RUNTIME_INPUT_REGION_UNIQUE_ACCOUNT_FOOTPRINT(direct_mapping) + \
+                                                                   sizeof(ulong)         /* instr data len */                                          + \
+                                                                                         /* No instr data  */                                            \
+                                                                   sizeof(fd_pubkey_t)), /* program id     */                                            \
+                                                                   FD_RUNTIME_INPUT_REGION_ALLOC_ALIGN_UP ) + FD_BPF_ALIGN_OF_U128)
+#define FD_RUNTIME_INPUT_REGION_TXN_FOOTPRINT(account_lock_limit, direct_mapping)                                                                           \
+                                              (FD_MAX_INSTRUCTION_STACK_DEPTH*FD_RUNTIME_INPUT_REGION_INSN_FOOTPRINT(account_lock_limit, direct_mapping)) + \
+                                              ((FD_TXN_MTU-FD_TXN_MIN_SERIALIZED_SZ-account_lock_limit)*8UL) /* We can have roughly this much duplicate offsets */
+/* Now finally, we bound out the footprint of transaction execution. */
+#define FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT(account_lock_limit, direct_mapping)                                         \
+                                                  (FD_RUNTIME_BORROWED_ACCOUNT_FOOTPRINT                                     + \
+                                                   FD_RUNTIME_INPUT_REGION_TXN_FOOTPRINT(account_lock_limit, direct_mapping))
+/* Convenience macros for common use cases. */
+#define FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_FUZZ    FD_RUNTIME_BORROWED_ACCOUNT_FOOTPRINT
+#define FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT_DEFAULT FD_RUNTIME_TRANSACTION_EXECUTION_FOOTPRINT(64UL, 0)
+
+/* Helpers for runtime spad frame management. */
+struct fd_runtime_spad_handle {
+  fd_spad_t *         spad;
+  fd_exec_txn_ctx_t * txn_ctx;
+};
+typedef struct fd_runtime_spad_handle fd_runtime_spad_handle_t;
+
+static inline void
+fd_runtime_spad_private_frame_end( fd_runtime_spad_handle_t * _spad_handle ) {
+  /* fd_spad_verify() returns 0 if everything looks good, and non-zero
+     otherwise.
+     Since the fast spad alloc API doesn't check for or indicate an OOM
+     situation and is going to happily permit an OOB alloc, we need
+     some way of detecting that. Moreover, we would also like to detect
+     unbalanced frame push/pop or usage of more frames than allowed.
+     While surrounding the spad with guard regions will help detect the
+     former, it won't necessarily catch the latter.
+     On compliant transactions, fd_spad_verify() isn't all that
+     expensive.  Nonetheless, We invoke fd_spad_verify() only at the
+     peak of memory usage, and not gratuitously everywhere. One peak
+     would be right before we do the most deeply nested spad frame pop.
+     However, we do pops through compiler-inserted cleanup functions
+     that take only a single pointer, so we define this helper function
+     to access the needed context info.  The end result is that we do
+     super fast spad calls everywhere in the runtime, and every now and
+     then we invoke verify to check things. */
+  /* -1UL because spad pop is called after instr stack pop. */
+  if( FD_UNLIKELY( _spad_handle->txn_ctx->instr_stack_sz>=FD_MAX_INSTRUCTION_STACK_DEPTH-1UL && fd_spad_verify( _spad_handle->txn_ctx->spad ) ) ) {
+    FD_LOG_ERR(( "spad corrupted or overflown" ));
+  }
+  fd_spad_pop( _spad_handle->spad );
+}
+
+#define FD_RUNTIME_SPAD_FRAME_BEGIN(_spad, _txn_ctx) do {                                             \
+  fd_runtime_spad_handle_t _spad_handle __attribute__((cleanup(fd_runtime_spad_private_frame_end))) = \
+    (fd_runtime_spad_handle_t) { .spad = _spad, .txn_ctx = _txn_ctx };                                \
+  fd_spad_push( _spad_handle.spad );                                                                  \
+  do
+
+#define FD_RUNTIME_SPAD_FRAME_END while(0); } while(0)
+
 FD_PROTOTYPES_BEGIN
 
 ulong

src/flamenco/runtime/info/fd_instr_info.h

@@ -24,6 +24,7 @@ struct fd_instr_info {
   fd_pubkey_t             acct_pubkeys[FD_INSTR_ACCT_MAX];
   uchar                   is_duplicate[FD_INSTR_ACCT_MAX];
 
+  /* Indexed by index in instruction, not by index in transaction. */
   fd_borrowed_account_t * borrowed_accounts[FD_INSTR_ACCT_MAX];
 
   /* fd_uwide representation of uint_128 */

src/flamenco/runtime/program/fd_bpf_loader_program.c

@@ -453,7 +453,7 @@ common_close_account( fd_pubkey_t * authority_address,
 
 
 /* https://github.com/anza-xyz/agave/blob/574bae8fefc0ed256b55340b9d87b7689bcdf222/programs/bpf_loader/src/lib.rs#L1332-L1501 */
-int
+static int
 execute( fd_exec_instr_ctx_t * instr_ctx, fd_sbpf_validated_program_t * prog, uchar is_deprecated ) {
 
   fd_sbpf_syscalls_t * syscalls = fd_sbpf_syscalls_new( fd_valloc_malloc( instr_ctx->valloc,
@@ -574,8 +574,6 @@ execute( fd_exec_instr_ctx_t * instr_ctx, fd_sbpf_validated_program_t * prog, uc
 
   /* Handles instr + EBPF errors */
   if( FD_UNLIKELY( exec_err!=FD_VM_SUCCESS ) ) {
-    fd_valloc_free( instr_ctx->valloc, input );
-
     /* Instr error case */
     if( instr_ctx->txn_ctx->exec_err_kind==FD_EXECUTOR_ERR_KIND_INSTR ) {
       return instr_ctx->txn_ctx->exec_err;
@@ -623,8 +621,6 @@ execute( fd_exec_instr_ctx_t * instr_ctx, fd_sbpf_validated_program_t * prog, uc
      TODO: vm should report */
   ulong syscall_err = vm->reg[0];
   if( FD_UNLIKELY( syscall_err ) ) {
-    fd_valloc_free( instr_ctx->valloc, input );
-
     /* https://github.com/anza-xyz/agave/blob/v2.0.9/programs/bpf_loader/src/lib.rs#L1431-L1434 */
     instr_ctx->txn_ctx->exec_err_kind = FD_EXECUTOR_ERR_KIND_INSTR;
     return program_error_to_instr_error( syscall_err, &instr_ctx->txn_ctx->custom_err );
@@ -634,16 +630,13 @@ execute( fd_exec_instr_ctx_t * instr_ctx, fd_sbpf_validated_program_t * prog, uc
   if( FD_UNLIKELY( is_deprecated ) ) {
     err = fd_bpf_loader_input_deserialize_unaligned( *instr_ctx, pre_lens, input, input_sz, !direct_mapping );
     if( FD_UNLIKELY( err!=0 ) ) {
-      fd_valloc_free( instr_ctx->valloc, input );
       return err;
     }
   } else {
     err = fd_bpf_loader_input_deserialize_aligned( *instr_ctx, pre_lens, input, input_sz, !direct_mapping );
     if( FD_UNLIKELY( err!=0 ) ) {
-      fd_valloc_free( instr_ctx->valloc, input );
       return err;
     }
-
   }
 
   return FD_EXECUTOR_INSTR_SUCCESS;

src/flamenco/runtime/program/fd_bpf_loader_serialization.c

@@ -157,7 +157,7 @@ fd_bpf_loader_input_serialize_aligned( fd_exec_instr_ctx_t       ctx,
 
   /* https://github.com/anza-xyz/agave/blob/b5f5c3cdd3f9a5859c49ebc27221dc27e143d760/programs/bpf_loader/src/serialization.rs#L429-L459 */
   ulong serialized_size = 0UL;
-  serialized_size += sizeof(ulong);
+  serialized_size += sizeof(ulong); // acct_cnt
   /* First pass is to calculate size of buffer to allocate */
   for( ushort i=0; i<ctx.instr->acct_cnt; i++ ) {
     uchar acc_idx = instr_acc_idxs[i];
@@ -204,7 +204,7 @@ fd_bpf_loader_input_serialize_aligned( fd_exec_instr_ctx_t       ctx,
                   +  ctx.instr->data_sz
                   +  sizeof(fd_pubkey_t); // program id
 
-  uchar * serialized_params            = fd_valloc_malloc( ctx.valloc, FD_BPF_ALIGN_OF_U128, fd_ulong_align_up( serialized_size, 16UL ) );
+  uchar * serialized_params            = fd_spad_alloc( ctx.txn_ctx->spad, FD_BPF_ALIGN_OF_U128, fd_ulong_align_up( serialized_size, FD_RUNTIME_INPUT_REGION_ALLOC_ALIGN_UP ) );
   uchar * serialized_params_start      = serialized_params;
   uchar * curr_serialized_params_start = serialized_params;
 
@@ -473,8 +473,6 @@ fd_bpf_loader_input_deserialize_aligned( fd_exec_instr_ctx_t ctx,
     }
   }
 
-  fd_valloc_free( ctx.valloc, buffer );
-
   return FD_EXECUTOR_INSTR_SUCCESS;
 }
 
@@ -538,8 +536,8 @@ fd_bpf_loader_input_serialize_unaligned( fd_exec_instr_ctx_t       ctx,
                    + ctx.instr->data_sz   // instruction data
                    + sizeof(fd_pubkey_t); // program id
 
-  uchar * serialized_params = fd_valloc_malloc( ctx.valloc, 1UL, serialized_size );
-  uchar * serialized_params_start = serialized_params;
+  uchar * serialized_params            = fd_spad_alloc( ctx.txn_ctx->spad, 1UL, serialized_size );
+  uchar * serialized_params_start      = serialized_params;
   uchar * curr_serialized_params_start = serialized_params;
 
   FD_STORE( ulong, serialized_params, ctx.instr->acct_cnt );
@@ -724,7 +722,5 @@ fd_bpf_loader_input_deserialize_unaligned( fd_exec_instr_ctx_t ctx,
     return FD_EXECUTOR_INSTR_ERR_INVALID_ARG;
   }
 
-  fd_valloc_free( ctx.valloc, input );
-
   return 0;
 }

src/flamenco/runtime/program/fd_bpf_loader_serialization.h

@@ -3,9 +3,8 @@
 
 #include "../../fd_flamenco_base.h"
 #include "../../vm/fd_vm.h"
+#include "../fd_runtime.h"
 
-#define MAX_PERMITTED_DATA_INCREASE (10240UL)
-#define FD_BPF_ALIGN_OF_U128        (8UL    )
 #define FD_NON_DUP_MARKER           (0xFF   )
 
 FD_PROTOTYPES_BEGIN