add randomized overlap committee generator

crates/example-types/src/node_types.rs

@@ -4,9 +4,14 @@
 // You should have received a copy of the MIT License
 // along with the HotShot repository. If not, see <https://mit-license.org/>.
 
+use std::marker::PhantomData;
+
+pub use hotshot::traits::election::helpers::{
+    RandomOverlapQuorumFilterConfig, StableQuorumFilterConfig,
+};
 use hotshot::traits::{
     election::{
-        randomized_committee::RandomizedCommittee,
+        helpers::QuorumFilterConfig, randomized_committee::RandomizedCommittee,
         randomized_committee_members::RandomizedCommitteeMembers,
         static_committee::StaticCommittee,
         static_committee_leader_two_views::StaticCommitteeLeaderForTwoViews,
@@ -104,10 +109,11 @@ impl NodeType for TestTypesRandomizedLeader {
 )]
 /// filler struct to implement node type and allow us
 /// to select our traits
-pub struct TestTypesRandomizedCommitteeMembers<const SEED: u64, const OVERLAP: u64>;
-impl<const SEED: u64, const OVERLAP: u64> NodeType
-    for TestTypesRandomizedCommitteeMembers<SEED, OVERLAP>
-{
+pub struct TestTypesRandomizedCommitteeMembers<CONFIG: QuorumFilterConfig> {
+    _pd: PhantomData<CONFIG>,
+}
+
+impl<CONFIG: QuorumFilterConfig> NodeType for TestTypesRandomizedCommitteeMembers<CONFIG> {
     type AuctionResult = TestAuctionResult;
     type View = ViewNumber;
     type Epoch = EpochNumber;
@@ -117,11 +123,8 @@ impl<const SEED: u64, const OVERLAP: u64> NodeType
     type Transaction = TestTransaction;
     type ValidatedState = TestValidatedState;
     type InstanceState = TestInstanceState;
-    type Membership = RandomizedCommitteeMembers<
-        TestTypesRandomizedCommitteeMembers<SEED, OVERLAP>,
-        SEED,
-        OVERLAP,
-    >;
+    type Membership =
+        RandomizedCommitteeMembers<TestTypesRandomizedCommitteeMembers<CONFIG>, CONFIG>;
     type BuilderSignatureKey = BuilderKey;
 }
 

crates/examples/push-cdn/all.rs

@@ -14,7 +14,6 @@ use cdn_broker::{
     reexports::{crypto::signature::KeyPair, def::hook::NoMessageHook},
     Broker,
 };
-
 use cdn_marshal::Marshal;
 use hotshot::{
     helpers::initialize_logging,

crates/hotshot/src/tasks/mod.rs

@@ -10,11 +10,6 @@
 pub mod task_state;
 use std::{collections::BTreeMap, fmt::Debug, num::NonZeroUsize, sync::Arc, time::Duration};
 
-use crate::{
-    tasks::task_state::CreateTaskState, types::SystemContextHandle, ConsensusApi,
-    ConsensusMetricsValue, ConsensusTaskRegistry, HotShotConfig, HotShotInitializer,
-    MarketplaceConfig, Memberships, NetworkTaskRegistry, SignatureKey, SystemContext, Versions,
-};
 use async_broadcast::{broadcast, RecvError};
 use async_lock::RwLock;
 use async_trait::async_trait;
@@ -48,6 +43,12 @@ use hotshot_types::{
 use tokio::{spawn, time::sleep};
 use vbs::version::StaticVersionType;
 
+use crate::{
+    tasks::task_state::CreateTaskState, types::SystemContextHandle, ConsensusApi,
+    ConsensusMetricsValue, ConsensusTaskRegistry, HotShotConfig, HotShotInitializer,
+    MarketplaceConfig, Memberships, NetworkTaskRegistry, SignatureKey, SystemContext, Versions,
+};
+
 /// event for global event stream
 #[derive(Clone, Debug)]
 pub enum GlobalEvent {

crates/hotshot/src/traits/election/helpers.rs

@@ -4,7 +4,7 @@
 // You should have received a copy of the MIT License
 // along with the HotShot repository. If not, see <https://mit-license.org/>.
 
-use std::collections::BTreeSet;
+use std::{collections::BTreeSet, hash::Hash};
 
 use rand::{rngs::StdRng, Rng, SeedableRng};
 
@@ -52,7 +52,11 @@ impl Iterator for NonRepeatValueIterator {
 }
 
 /// Create a single u64 seed by merging two u64s. Done this way to allow easy seeding of the number generator
-/// from both a stable SOUND as well as a moving value ROUND (typically, epoch).
+/// from both a stable SOUND as well as a moving value ROUND (typically, epoch). Shift left by 8 to avoid
+/// scenarios where someone manually stepping seeds would pass over the same space of random numbers across
+/// sequential rounds. Doesn't have to be 8, but has to be large enough that it is unlikely that a given
+/// test run will collide; using 8 means that 256 rounds (epochs) would have to happen inside of a test before
+/// the test starts repeating values from SEED+1.
 fn make_seed(seed: u64, round: u64) -> u64 {
     seed.wrapping_add(round.wrapping_shl(8))
 }
@@ -102,7 +106,12 @@ fn calc_num_slots(count: u64, odd: bool) -> u64 {
 }
 
 impl StableQuorumIterator {
+    #[must_use]
     /// Create a new StableQuorumIterator
+    ///
+    /// # Panics
+    ///
+    /// panics if overlap is greater than half of count
     pub fn new(seed: u64, round: u64, count: u64, overlap: u64) -> Self {
         assert!(
             count / 2 > overlap,
@@ -127,22 +136,33 @@ impl Iterator for StableQuorumIterator {
 
     fn next(&mut self) -> Option<Self::Item> {
         if self.index >= (self.count / 2) {
+            // Always return exactly half of the possible values. If we have OVERLAP>0 then
+            // we need to return (COUNT/2)-OVERLAP of the current set, even if there are additional
+            // even (or odd) numbers that we can return.
             None
         } else if self.index < self.overlap {
-            // Generate enough values for the previous round
+            // Generate enough values for the previous round. If the current round is odd, then
+            // we want to pick even values that were selected from the previous round to create OVERLAP
+            // even values.
             let v = self.prev_rng.next().unwrap();
             self.index += 1;
             Some(v * 2 + self.round % 2)
         } else {
-            // Generate new values
+            // Generate new values. If our current round is odd, we'll be creating (COUNT/2)-OVERLAP
+            // odd values here.
             let v = self.this_rng.next().unwrap();
             self.index += 1;
             Some(v * 2 + (1 - self.round % 2))
         }
     }
 }
 
+#[must_use]
 /// Helper function to convert the arguments to a StableQuorumIterator into an ordered set of values.
+///
+/// # Panics
+///
+/// panics if the arguments are invalid for StableQuorumIterator::new
 pub fn stable_quorum_filter(seed: u64, round: u64, count: usize, overlap: u64) -> BTreeSet<usize> {
     StableQuorumIterator::new(seed, round, count as u64, overlap)
         // We should never have more than u32_max members in a test
@@ -175,7 +195,12 @@ pub struct RandomOverlapQuorumIterator {
 }
 
 impl RandomOverlapQuorumIterator {
+    #[must_use]
     /// Create a new RandomOverlapQuorumIterator
+    ///
+    /// # Panics
+    ///
+    /// panics if overlap and members can produce invalid results or if ranges are invalid
     pub fn new(
         seed: u64,
         round: u64,
@@ -231,16 +256,109 @@ impl Iterator for RandomOverlapQuorumIterator {
             // Generate enough values for the previous round
             let v = self.prev_rng.next().unwrap();
             self.index += 1;
-            Some(v * 2 + self.round % 2)
+            Some(v * 2 + (1 - self.round % 2))
         } else {
             // Generate new values
             let v = self.this_rng.next().unwrap();
             self.index += 1;
-            Some(v * 2 + (1 - self.round % 2))
+            Some(v * 2 + self.round % 2)
         }
     }
 }
 
+#[must_use]
+/// Helper function to convert the arguments to a StableQuorumIterator into an ordered set of values.
+///
+/// # Panics
+///
+/// panics if the arguments are invalid for RandomOverlapQuorumIterator::new
+pub fn random_overlap_quorum_filter(
+    seed: u64,
+    round: u64,
+    count: usize,
+    members_min: u64,
+    members_max: u64,
+    overlap_min: u64,
+    overlap_max: u64,
+) -> BTreeSet<usize> {
+    RandomOverlapQuorumIterator::new(
+        seed,
+        round,
+        count as u64,
+        members_min,
+        members_max,
+        overlap_min,
+        overlap_max,
+    )
+    // We should never have more than u32_max members in a test
+    .map(|x| usize::try_from(x).unwrap())
+    .collect()
+}
+
+/// Trait wrapping a config for quorum filters. This allows selection between either the StableQuorumIterator or the
+/// RandomOverlapQuorumIterator functionality from above
+pub trait QuorumFilterConfig:
+    Copy
+    + Clone
+    + std::fmt::Debug
+    + Default
+    + Send
+    + Sync
+    + Ord
+    + PartialOrd
+    + Eq
+    + PartialEq
+    + Hash
+    + 'static
+{
+    /// Called to run the filter and return a set of indices
+    fn execute(epoch: u64, count: usize) -> BTreeSet<usize>;
+}
+
+#[derive(Debug, Copy, Clone, Default, Eq, PartialEq, Hash, Ord, PartialOrd)]
+/// Provides parameters to use the StableQuorumIterator
+pub struct StableQuorumFilterConfig<const SEED: u64, const OVERLAP: u64> {}
+
+impl<const SEED: u64, const OVERLAP: u64> QuorumFilterConfig
+    for StableQuorumFilterConfig<SEED, OVERLAP>
+{
+    fn execute(epoch: u64, count: usize) -> BTreeSet<usize> {
+        stable_quorum_filter(SEED, epoch, count, OVERLAP)
+    }
+}
+
+#[derive(Debug, Copy, Clone, Default, Eq, PartialEq, Hash, Ord, PartialOrd)]
+/// Provides parameters to use the RandomOverlapQuorumIterator
+pub struct RandomOverlapQuorumFilterConfig<
+    const SEED: u64,
+    const MEMBERS_MIN: u64,
+    const MEMBERS_MAX: u64,
+    const OVERLAP_MIN: u64,
+    const OVERLAP_MAX: u64,
+> {}
+
+impl<
+        const SEED: u64,
+        const MEMBERS_MIN: u64,
+        const MEMBERS_MAX: u64,
+        const OVERLAP_MIN: u64,
+        const OVERLAP_MAX: u64,
+    > QuorumFilterConfig
+    for RandomOverlapQuorumFilterConfig<SEED, MEMBERS_MIN, MEMBERS_MAX, OVERLAP_MIN, OVERLAP_MAX>
+{
+    fn execute(epoch: u64, count: usize) -> BTreeSet<usize> {
+        random_overlap_quorum_filter(
+            SEED,
+            epoch,
+            count,
+            MEMBERS_MIN,
+            MEMBERS_MAX,
+            OVERLAP_MIN,
+            OVERLAP_MAX,
+        )
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -279,4 +397,37 @@ mod tests {
             assert!(matched, "prev_set={prev_set:?}, this_set={this_set:?}");
         }
     }
+
+    #[test]
+    fn test_odd_even() {
+        for _ in 0..100 {
+            let seed = rand::random::<u64>();
+
+            let odd_set: Vec<u64> = StableQuorumIterator::new(seed, 1, 10, 2).collect();
+            let even_set: Vec<u64> = StableQuorumIterator::new(seed, 2, 10, 2).collect();
+
+            assert!(
+                odd_set[2] % 2 == 1,
+                "odd set non-overlap value should be odd (stable)"
+            );
+            assert!(
+                even_set[2] % 2 == 0,
+                "even set non-overlap value should be even (stable)"
+            );
+
+            let odd_set: Vec<u64> =
+                RandomOverlapQuorumIterator::new(seed, 1, 20, 5, 10, 2, 3).collect();
+            let even_set: Vec<u64> =
+                RandomOverlapQuorumIterator::new(seed, 2, 20, 5, 10, 2, 3).collect();
+
+            assert!(
+                odd_set[3] % 2 == 1,
+                "odd set non-overlap value should be odd (random overlap)"
+            );
+            assert!(
+                even_set[3] % 2 == 0,
+                "even set non-overlap value should be even (random overlap)"
+            );
+        }
+    }
 }

crates/hotshot/src/traits/election/mod.rs

@@ -19,4 +19,4 @@ pub mod static_committee;
 pub mod static_committee_leader_two_views;
 
 /// general helpers
-mod helpers;
+pub mod helpers;

crates/hotshot/src/traits/election/randomized_committee_members.rs

@@ -7,6 +7,7 @@
 use std::{
     cmp::max,
     collections::{BTreeMap, BTreeSet},
+    marker::PhantomData,
     num::NonZeroU64,
 };
 
@@ -23,13 +24,11 @@ use hotshot_types::{
 use rand::{rngs::StdRng, Rng};
 use utils::anytrace::Result;
 
-use super::helpers::stable_quorum_filter;
+use crate::traits::election::helpers::QuorumFilterConfig;
 
 #[derive(Clone, Debug, Eq, PartialEq, Hash)]
-
 /// The static committee election
-
-pub struct RandomizedCommitteeMembers<T: NodeType, const SEED: u64, const OVERLAP: u64> {
+pub struct RandomizedCommitteeMembers<T: NodeType, C: QuorumFilterConfig> {
     /// The nodes eligible for leadership.
     /// NOTE: This is currently a hack because the DA leader needs to be the quorum
     /// leader but without voting rights.
@@ -44,19 +43,20 @@ pub struct RandomizedCommitteeMembers<T: NodeType, const SEED: u64, const OVERLA
 
     /// The network topic of the committee
     committee_topic: Topic,
+
+    /// Phantom
+    _pd: PhantomData<C>,
 }
 
-impl<TYPES: NodeType, const SEED: u64, const OVERLAP: u64>
-    RandomizedCommitteeMembers<TYPES, SEED, OVERLAP>
-{
+impl<TYPES: NodeType, CONFIG: QuorumFilterConfig> RandomizedCommitteeMembers<TYPES, CONFIG> {
     /// Creates a set of indices into the stake_table which reference the nodes selected for this epoch's committee
     fn make_quorum_filter(&self, epoch: <TYPES as NodeType>::Epoch) -> BTreeSet<usize> {
-        stable_quorum_filter(SEED, epoch.u64(), self.stake_table.len(), OVERLAP)
+        CONFIG::execute(epoch.u64(), self.stake_table.len())
     }
 }
 
-impl<TYPES: NodeType, const SEED: u64, const OVERLAP: u64> Membership<TYPES>
-    for RandomizedCommitteeMembers<TYPES, SEED, OVERLAP>
+impl<TYPES: NodeType, CONFIG: QuorumFilterConfig> Membership<TYPES>
+    for RandomizedCommitteeMembers<TYPES, CONFIG>
 {
     type Error = utils::anytrace::Error;
 
@@ -102,6 +102,7 @@ impl<TYPES: NodeType, const SEED: u64, const OVERLAP: u64> Membership<TYPES>
             stake_table: members,
             indexed_stake_table,
             committee_topic,
+            _pd: PhantomData,
         }
     }