feat(consensus): stub out tower module, tower_sync vote ixs

This PR stubs out the fd_tower module which will implement various safe
guards in the consensus protocol (threshold check, switch proof). It
also stubs out the new tower_sync instruction in the vote program.

Implementation in PR to follow.

.gitignore

@@ -10,8 +10,10 @@ target/
 *.lcov
 *.profraw
 
-# Clang
+# Clangd
+.cache
 .clangd
+compile_commands.json
 
 # Rust
 **/Cargo.lock
@@ -30,6 +32,9 @@ deps-bundle.tar.zst
 *.swp
 *.swo
 
+# VSCode
+.vscode
+
 # Packet Captures
 *.pcap
 *.pcapng

doc/organization.txt

@@ -34,10 +34,11 @@ Firedancer source tree
   │   │                needed for interoperability with the Solana ecosystem
   │   │                (hash functions, cryptographic algorithms)
   │   │
+  |   ├── choreo/      Consensus components (fork choice, voting)
+  │   │
   │   ├── disco/       Tiles running on the tango messaging layer
   │   │
-  │   ├── flamenco/    Major Solana runtime and consensus components
-  │   │                (bank, leader schedule, virtual machine, ...)
+  │   ├── flamenco/    Major Solana runtime components
   │   │
   │   ├── funk/        Database optimized for storing Solana ledger and
   │   │                accounts

src/choreo/bft/fd_bft.c

@@ -6,6 +6,7 @@
 
 #include "../fd_choreo_base.h"
 #include "../ghost/fd_ghost.h"
+#include "../tower/fd_tower.h"
 #include "fd_bft.h"
 
 #pragma GCC diagnostic ignored "-Wformat"
@@ -77,29 +78,45 @@ fd_bft_delete( void * bft ) {
 
 static void
 count_votes( fd_bft_t * bft, fd_fork_t * fork ) {
-  for( fd_tower_deque_iter_t iter = fd_tower_deque_iter_init( fork->slot_ctx.towers );
-       !fd_tower_deque_iter_done( fork->slot_ctx.towers, iter );
-       iter = fd_tower_deque_iter_next( fork->slot_ctx.towers, iter ) ) {
-    fd_tower_t * tower = fd_tower_deque_iter_ele( fork->slot_ctx.towers, iter );
+  for( fd_latest_vote_deque_iter_t iter =
+           fd_latest_vote_deque_iter_init( fork->slot_ctx.latest_votes );
+       !fd_latest_vote_deque_iter_done( fork->slot_ctx.latest_votes, iter );
+       iter = fd_latest_vote_deque_iter_next( fork->slot_ctx.latest_votes, iter ) ) {
+    fd_latest_vote_t * latest_vote =
+        fd_latest_vote_deque_iter_ele( fork->slot_ctx.latest_votes, iter );
 
-    ulong vote_slot = tower->slots[tower->cnt - 1];
+    ulong latest_vote_slot = latest_vote->slot_hash.slot;
 
     /* Ignore votes for slots < snapshot_slot. */
 
-    if( FD_UNLIKELY( vote_slot < bft->snapshot_slot ) ) continue;
+    if( FD_UNLIKELY( latest_vote_slot < bft->snapshot_slot ) ) continue;
 
     /* Look up _our_ bank hash for this vote slot. */
 
-    fd_hash_t const * vote_hash = fd_blockstore_bank_hash_query( bft->blockstore, vote_slot );
-    if( FD_UNLIKELY( !vote_hash ) ) {
-      FD_LOG_WARNING( ( "couldn't find bank hash for slot %lu", vote_slot ) );
+    fd_hash_t const * bank_hash =
+        fd_blockstore_bank_hash_query( bft->blockstore, latest_vote_slot );
+
+    /* TODO we need to implement repair logic here */
+
+    if( FD_UNLIKELY( !bank_hash ) ) {
+      FD_LOG_WARNING( ( "couldn't find bank hash for slot %lu", latest_vote_slot ) );
+      continue;
+    }
+
+    /* TODO we need to implement eqv block logic here */
+
+    if( FD_UNLIKELY( memcmp( bank_hash, &latest_vote->slot_hash.hash, sizeof( fd_hash_t ) ) ) ) {
+      FD_LOG_WARNING( ( "possible equivocating block for %lu. ours: %32J vs. theirs: %32J",
+                        latest_vote_slot,
+                        bank_hash,
+                        &latest_vote->slot_hash.hash ) );
       continue;
     }
-    fd_slot_hash_t vote_key = { .slot = vote_slot, .hash = *vote_hash };
 
-    /* Look up latest vote for this node (pubkey) and stake. */
+    /* Look up the stake for this pubkey. */
 
-    fd_pubkey_t * pubkey = &tower->vote_acc_addr;
+    fd_slot_hash_t slot_hash = { .slot = latest_vote_slot, .hash = *bank_hash };
+    fd_pubkey_t *  pubkey    = &latest_vote->node_pubkey;
     fd_epoch_bank_t * epoch_bank = fd_exec_epoch_ctx_epoch_bank( fork->slot_ctx.epoch_ctx );
     fd_vote_accounts_pair_t_mapnode_t * root =
         epoch_bank->stakes.vote_accounts.vote_accounts_root;
@@ -112,7 +129,7 @@ count_votes( fd_bft_t * bft, fd_fork_t * fork ) {
 
     /* If it's not in the epoch stakes map, look in the incremental-epoch stakes map.  */
 
-    if( !vote_node ) {
+    if( FD_UNLIKELY( !vote_node ) ) {
       pool = fork->slot_ctx.slot_bank.vote_account_keys.vote_accounts_pool;
       root = fork->slot_ctx.slot_bank.vote_account_keys.vote_accounts_root;
       fd_vote_accounts_pair_t_mapnode_t * vote_node =
@@ -125,30 +142,32 @@ count_votes( fd_bft_t * bft, fd_fork_t * fork ) {
 
     /* Set the stake. */
 
-    ulong             stake      = vote_node->elem.stake;
-    fd_ghost_node_t * node = fd_ghost_node_query( bft->ghost, &vote_key );
+    ulong             stake = vote_node->elem.stake;
+    fd_ghost_node_t * ghost_node  = fd_ghost_node_query( bft->ghost, &slot_hash );
 
     /* This slot hash must have been inserted, because ghost only processes replay votes. */
 
-    if( FD_UNLIKELY( !node ) ) {
-      FD_LOG_ERR( ( "missing ghost key %lu %32J", vote_key.slot, vote_key.hash.hash ) );
+    if( FD_UNLIKELY( !ghost_node ) ) {
+      FD_LOG_ERR( ( "missing ghost key %lu %32J", slot_hash.slot, slot_hash.hash.hash ) );
     };
 
-    fd_ghost_latest_vote_upsert( bft->ghost, &vote_key, pubkey, stake );
+    fd_ghost_latest_vote_upsert( bft->ghost, &slot_hash, pubkey, stake );
 
-    double pct = (double)node->stake / (double)bft->epoch_stake;
-    if( FD_UNLIKELY( !fork->eqv_safe && pct > FD_BFT_EQV_SAFE ) ) {
-      fork->eqv_safe = 1;
+    double pct = (double)ghost_node->stake / (double)bft->epoch_stake;
+
+    if( FD_UNLIKELY( !ghost_node->eqv_safe && pct > FD_BFT_EQV_SAFE ) ) {
+      ghost_node->eqv_safe = 1;
 #if FD_BFT_USE_HANDHOLDING
       FD_LOG_NOTICE(
-          ( "[bft] eqv safe (%lf): (%lu, %32J)", pct, vote_key.slot, vote_key.hash.hash ) );
+          ( "[bft] eqv safe (%lf): (%lu, %32J)", pct, slot_hash.slot, slot_hash.hash.hash ) );
 #endif
     }
-    if( FD_UNLIKELY( !fork->opt_conf && pct > FD_BFT_OPT_CONF ) ) {
-      fork->opt_conf = 1;
+
+    if( FD_UNLIKELY( !ghost_node->opt_conf && pct > FD_BFT_OPT_CONF ) ) {
+      ghost_node->opt_conf = 1;
 #if FD_BFT_USE_HANDHOLDING
       FD_LOG_NOTICE(
-          ( "[bft] opt conf (%lf): (%lu, %32J)", pct, vote_key.slot, vote_key.hash.hash ) );
+          ( "[bft] opt conf (%lf): (%lu, %32J)", pct, slot_hash.slot, slot_hash.hash.hash ) );
 #endif
     }
   }
@@ -167,7 +186,7 @@ fd_bft_fork_update( fd_bft_t * bft, fd_fork_t * fork ) {
 
   /* Get the parent key. Every block must have a parent (except genesis or snapshot block). */
 
-  ulong parent_slot = fd_blockstore_parent_slot_query(bft->blockstore, fork->slot);
+  ulong parent_slot = fd_blockstore_parent_slot_query( bft->blockstore, fork->slot );
 
   /* Insert this fork into bft. */
 
@@ -185,7 +204,7 @@ fd_bft_fork_update( fd_bft_t * bft, fd_fork_t * fork ) {
     };
     FD_LOG_NOTICE( ( "[ghost] insert slot: %lu hash: %32J parent: %lu parent_hash: %32J",
                      curr_key.slot,
-                     curr_key.hash.uc,
+                     curr_key.hash.hash,
                      parent_slot,
                      parent_bank_hash->hash ) );
     fd_ghost_leaf_insert( bft->ghost, &curr_key, &parent_key );
@@ -243,9 +262,13 @@ fd_bft_fork_choice( fd_bft_t * bft ) {
     fd_slot_hash_t    key  = { .slot = fork->slot, .hash = fork->slot_ctx.slot_bank.banks_hash };
     fd_ghost_node_t * node = fd_ghost_node_query( ghost, &key );
 
-    #if FD_BFT_USE_HANDHOLDING
+    /* do not pick forks with equivocating blocks. */
+
+    if( FD_UNLIKELY( node->eqv && !node->eqv_safe ) ) continue;
+
+#if FD_BFT_USE_HANDHOLDING
 
-    /* invariant: node must have been inserted by now */
+    /* invariant: node must have been inserted by now. */
 
     if( !node ) FD_LOG_ERR( ( "missing ghost node %lu", fork->slot ) );
 #endif
@@ -258,11 +281,13 @@ fd_bft_fork_choice( fd_bft_t * bft ) {
 
   if( heaviest_fork_key ) {
     double pct = (double)heaviest_fork_weight / (double)bft->epoch_stake;
-    FD_LOG_NOTICE( ( "[bft] voting for heaviest fork %lu %lu (%lf)", heaviest_fork_key->slot, heaviest_fork_weight, pct ) );
-    // fd_ghost_print( ghost );
+    FD_LOG_NOTICE( ( "[bft] voting for heaviest fork %lu %lu (%lf)",
+                     heaviest_fork_key->slot,
+                     heaviest_fork_weight,
+                     pct ) );
   }
 
-  return heaviest_fork_key; /* lifetime as long as it remains in forks->bft */
+  return heaviest_fork_key; /* lifetime as long as it remains in the frontier */
 }
 
 void

src/choreo/bft/fd_bft.h

@@ -8,6 +8,7 @@
 #include "../fd_choreo_base.h"
 #include "../forks/fd_forks.h"
 #include "../ghost/fd_ghost.h"
+#include "../tower/fd_tower.h"
 
 /* FD_BFT_USE_HANDHOLDING:  Define this to non-zero at compile time
    to turn on additional runtime checks and logging. */
@@ -22,8 +23,9 @@
 /* fd_bft implements Solana's Proof-of-Stake consensus protocol. */
 
 struct fd_bft {
-  ulong snapshot_slot;
-  ulong epoch_stake; /* total amount of stake in the current epoch */
+  ulong      snapshot_slot;
+  ulong      epoch_stake; /* total amount of stake in the current epoch */
+  fd_tower_t tower;       /* our local vote tower */
 
   /* external joins */
 
@@ -91,4 +93,7 @@ fd_bft_fork_choice( fd_bft_t * bft );
 void
 fd_bft_epoch_stake_update( fd_bft_t * bft, fd_exec_epoch_ctx_t * epoch_ctx );
 
+void
+fd_bft_tower_threshold_check( fd_bft_t * bft );
+
 #endif /* HEADER_fd_src_choreo_bft_fd_bft_h */

src/choreo/fd_choreo_base.h

@@ -1,10 +1,29 @@
 #ifndef HEADER_fd_src_choreo_fd_choreo_base_h
 #define HEADER_fd_src_choreo_fd_choreo_base_h
 
+/* Choreo is the consensus library.
+
+  - bft: wires it all together.
+  - eqv: equivocation (also called "duplicate block") handling.
+  - forks: data structures and associated functions to manage "forks", ie. competing views in the
+  canonical state of the blockchain.
+  - ghost: fork choice rule, ie. which fork is the best one that I should pick.
+  - tower: TowerBFT rules for reaching consensus by "finalizing" blocks, ie. you
+    can no longer rollback a block or switch to a different fork.
+
+    Includes threshold check and switch proof.
+
+    Note this is the TowerBFT implementation on the local ("self") side. The Vote Program is the
+    TowerBFT implementation for the cluster ("others") side. In other words, the local validator
+    runs through these rules before submitting the vote. The other validators in the cluster then
+    process the submitted vote using the Vote Program. */
+
 #include "../flamenco/fd_flamenco_base.h"
 #include "../flamenco/types/fd_types.h"
 
 /* clang-format off */
+#define FD_LG_NODE_MAX (16UL) /* the maximum number of nodes (unique pubkeys) consensus data structures will support */
+
 #define FD_SLOT_HASH_CMP(a,b)  (fd_int_if(((a)->slot)<((b)->slot),-1,fd_int_if(((a)->slot)>((b)->slot),1),memcmp((a),(b),sizeof(fd_slot_hash_t))))
 #define FD_SLOT_HASH_EQ(a,b)   ((((a)->slot)==((b)->slot)) & !(memcmp(((a)->hash.uc),((b)->hash.uc),sizeof(fd_hash_t))))
 /* clang-format on */

src/choreo/forks/fd_forks.h

@@ -6,25 +6,11 @@
 #include "../../flamenco/runtime/fd_blockstore.h"
 #include "../fd_choreo_base.h"
 
-/* FD_FORKS_USE_HANDHOLDING:  Define this to non-zero at compile time
-   to turn on additional runtime checks and logging. */
-
-#ifndef FD_FORKS_USE_HANDHOLDING
-#define FD_FORKS_USE_HANDHOLDING 1
-#endif
-
-#define FD_FORKS_EQV_SAFE ( 0.52 )
-#define FD_FORKS_OPT_CONF ( 2.0 / 3.0 )
-
 struct fd_fork {
-  ulong slot; /* head of the fork, frontier key */
-  ulong next; /* reserved for use by fd_pool and fd_map_chain */
-
-  int   eqv_safe;    /* equivocation safe */
-  int   opt_conf;    /* optimistically confirmed */
-
-  fd_exec_slot_ctx_t slot_ctx;
-  fd_block_t *       head; /* the block representing the head of the fork */
+  ulong              slot;     /* head of the fork, frontier key */
+  ulong              next;     /* reserved for use by fd_pool and fd_map_chain */
+  fd_block_t *       head;     /* the block representing the head of the fork */
+  fd_exec_slot_ctx_t slot_ctx; /* the bank representing the head of the fork */
 };
 typedef struct fd_fork fd_fork_t;
 

src/choreo/ghost/fd_ghost.h

@@ -32,18 +32,22 @@
 /* clang-format off */
 typedef struct fd_ghost_node fd_ghost_node_t;
 struct __attribute__((aligned(128UL))) fd_ghost_node {
-  fd_slot_hash_t    key;     /* (slot, bank_hash) to index ghost */
-  ulong             next;    /* reserved for internal use by fd_pool and fd_map_chain */
-  ulong             weight;  /* sum of stake for the subtree rooted at this slot hash */
-  ulong             stake;   /* stake amount for only this slot hash */
-  fd_ghost_node_t * head;    /* the head of the fork i.e. leaf of the highest-weight subtree */
-  fd_ghost_node_t * parent;  /* parent slot hash */
-  fd_ghost_node_t * child;   /* pointer to the left-most child */
-  fd_ghost_node_t * sibling; /* pointer to next sibling */
+  fd_slot_hash_t    key;      /* (slot, bank_hash) to index ghost */
+  ulong             next;     /* reserved for internal use by fd_pool and fd_map_chain */
+  ulong             weight;   /* sum of stake for the subtree rooted at this slot hash */
+  ulong             stake;    /* stake amount for only this slot hash */
+  int               eqv;      /* flag for equivocation (multiple blocks) in this slot */
+  int               eqv_safe; /* flag for equivocation safety (ie. 52% of stake has voted for this slot hash) */
+  int               opt_conf; /* flag for optimistic confirmation (ie. 2/3 of stake has voted for this slot hash ) */
+  fd_ghost_node_t * head;     /* the head of the fork i.e. leaf of the highest-weight subtree */
+  fd_ghost_node_t * parent;   /* parent slot hash */
+  fd_ghost_node_t * child;    /* pointer to the left-most child */
+  fd_ghost_node_t * sibling;  /* pointer to next sibling */
 };
-/* clang-format on */
 
-/* clang-format off */
+#define FD_GHOST_EQV_SAFE ( 0.52 )
+#define FD_GHOST_OPT_CONF ( 2.0 / 3.0 )
+
 /* fork a's weight > fork b's weight, with lower slot # as tie-break */
 #define FD_GHOST_NODE_MAX(a,b) (fd_ptr_if(fd_int_if(a->weight==b->weight, a->key.slot<b->key.slot, a->weight>b->weight),a,b))
 #define FD_GHOST_NODE_EQ(a,b)  (FD_SLOT_HASH_EQ(&a->key,&b->key))

src/choreo/tower/fd_tower.c

@@ -0,0 +1,33 @@
+#include "fd_tower.h"
+
+int
+fd_tower_threshold_check( fd_tower_t * tower,
+                          ulong        total_stake,
+                          ulong        threshold_depth,
+                          float        threshold_pct ) {
+  if( FD_UNLIKELY( tower->slots_cnt < threshold_depth ) ) return 1;
+  ulong             slot      = tower->slots[tower->slots_cnt - threshold_depth];
+  fd_hash_t const * hash      = fd_blockstore_bank_hash_query( tower->blockstore, slot );
+  fd_slot_hash_t    slot_hash = { .slot = slot, .hash = *hash };
+
+  fd_ghost_node_t const * ghost_node = fd_ghost_node_query( tower->ghost, &slot_hash );
+
+#if FD_TOWER_USE_HANDHOLDING
+
+  /* This shouldn't happen because slot hashes are inserted into the ghost upon execution. Indicates
+   * a likely programming error. */
+
+  if( FD_UNLIKELY( ghost_node == NULL ) ) {
+    FD_LOG_ERR( ( "invariant violation: slot %lu, hash: %32J not found in ghost", slot, hash->hash ) );
+  }
+
+#endif
+
+  float pct = (float)ghost_node->weight / (float)total_stake;
+  return pct > threshold_pct;
+}
+
+int
+fd_tower_switch_proof_construct( fd_tower_t * tower ) {
+  FD_LOG_ERR(("unimplemented"));
+}