feat(nns): Highly scalable voting

rs/nns/common/src/lib.rs

@@ -1,7 +1,6 @@
-use crate::pb::v1::NeuronId;
+use crate::pb::v1::{NeuronId, ProposalId};
 use ic_crypto_sha2::Sha256;
-use ic_stable_structures::storable::Bound;
-use ic_stable_structures::Storable;
+use ic_stable_structures::{storable::Bound, Storable};
 use num_traits::bounds::{LowerBounded, UpperBounded};
 use std::{borrow::Cow, convert::TryInto};
 
@@ -44,7 +43,24 @@ impl Storable for NeuronId {
     }
 
     fn from_bytes(bytes: Cow<[u8]>) -> Self {
-        NeuronId {
+        Self {
+            id: u64::from_bytes(bytes),
+        }
+    }
+
+    const BOUND: Bound = Bound::Bounded {
+        max_size: std::mem::size_of::<u64>() as u32,
+        is_fixed_size: true,
+    };
+}
+
+impl Storable for ProposalId {
+    fn to_bytes(&self) -> Cow<[u8]> {
+        self.id.to_bytes()
+    }
+
+    fn from_bytes(bytes: Cow<[u8]>) -> Self {
+        Self {
             id: u64::from_bytes(bytes),
         }
     }

rs/nns/governance/BUILD.bazel

@@ -37,6 +37,7 @@ DEPENDENCIES = [
     "//rs/nervous_system/common",
     "//rs/nervous_system/governance",
     "//rs/nervous_system/linear_map",
+    "//rs/nervous_system/long_message",
     "//rs/nervous_system/neurons_fund",
     "//rs/nervous_system/proto",
     "//rs/nervous_system/root",

rs/nns/governance/Cargo.toml

@@ -51,6 +51,7 @@ ic-nervous-system-common = { path = "../../nervous_system/common" }
 ic-nervous-system-common-build-metadata = { path = "../../nervous_system/common/build_metadata" }
 ic-nervous-system-governance = { path = "../../nervous_system/governance" }
 ic-nervous-system-linear-map = { path = "../../nervous_system/linear_map" }
+ic-nervous-system-long-message = { path = "../../nervous_system/long_message" }
 ic-nervous-system-root = { path = "../../nervous_system/root" }
 ic-nervous-system-runtime = { path = "../../nervous_system/runtime" }
 ic-nervous-system-proto = { path = "../../nervous_system/proto" }

rs/nns/governance/canbench/canbench_results.yml

@@ -1,61 +1,61 @@
 benches:
   add_neuron_active_maximum:
     total:
-      instructions: 36108795
+      instructions: 36177882
       heap_increase: 1
       stable_memory_increase: 0
     scopes: {}
   add_neuron_active_typical:
     total:
-      instructions: 1832299
+      instructions: 1835342
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   add_neuron_inactive_maximum:
     total:
-      instructions: 96119480
+      instructions: 96157842
       heap_increase: 1
       stable_memory_increase: 0
     scopes: {}
   add_neuron_inactive_typical:
     total:
-      instructions: 7375058
+      instructions: 7371261
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   cascading_vote_all_heap:
     total:
-      instructions: 32012069
+      instructions: 32067972
       heap_increase: 0
-      stable_memory_increase: 0
+      stable_memory_increase: 128
     scopes: {}
   cascading_vote_heap_neurons_stable_index:
     total:
-      instructions: 54483561
+      instructions: 54201401
       heap_increase: 0
-      stable_memory_increase: 0
+      stable_memory_increase: 128
     scopes: {}
   cascading_vote_stable_everything:
     total:
-      instructions: 160682199
+      instructions: 149924763
       heap_increase: 0
-      stable_memory_increase: 0
+      stable_memory_increase: 128
     scopes: {}
   cascading_vote_stable_neurons_with_heap_index:
     total:
-      instructions: 138319942
+      instructions: 127727264
       heap_increase: 0
-      stable_memory_increase: 0
+      stable_memory_increase: 128
     scopes: {}
   centralized_following_all_stable:
     total:
-      instructions: 66145106
+      instructions: 33257511
       heap_increase: 0
-      stable_memory_increase: 0
+      stable_memory_increase: 128
     scopes: {}
   compute_ballots_for_new_proposal_with_stable_neurons:
     total:
-      instructions: 1808724
+      instructions: 1810795
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
@@ -67,7 +67,7 @@ benches:
     scopes: {}
   list_neurons_ready_to_unstake_maturity_stable:
     total:
-      instructions: 36291394
+      instructions: 36937433
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
@@ -79,7 +79,7 @@ benches:
     scopes: {}
   list_ready_to_spawn_neuron_ids_stable:
     total:
-      instructions: 36280095
+      instructions: 36926134
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
@@ -91,25 +91,25 @@ benches:
     scopes: {}
   neuron_metrics_calculation_stable:
     total:
-      instructions: 1842928
+      instructions: 1841732
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   range_neurons_performance:
     total:
-      instructions: 48528017
+      instructions: 47699842
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   single_vote_all_stable:
     total:
-      instructions: 364769
+      instructions: 148771
       heap_increase: 0
-      stable_memory_increase: 0
+      stable_memory_increase: 128
     scopes: {}
   update_recent_ballots_stable_memory:
     total:
-      instructions: 13455039
+      instructions: 236994
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}

rs/nns/governance/canister/canister.rs

@@ -161,6 +161,7 @@ fn set_governance(gov: Governance) {
 fn schedule_timers() {
     schedule_seeding(Duration::from_nanos(0));
     schedule_adjust_neurons_storage(Duration::from_nanos(0), NeuronIdProto { id: 0 });
+    schedule_vote_processing();
 }
 
 // Seeding interval seeks to find a balance between the need for rng secrecy, and
@@ -216,6 +217,15 @@ fn schedule_adjust_neurons_storage(delay: Duration, start_neuron_id: NeuronIdPro
     });
 }
 
+/// The interval at which the voting state machines are processed.
+const VOTE_PROCESSING_INTERVAL: Duration = Duration::from_secs(3);
+
+fn schedule_vote_processing() {
+    ic_cdk_timers::set_timer_interval(VOTE_PROCESSING_INTERVAL, || {
+        governance_mut().process_voting_state_machines();
+    });
+}
+
 struct CanisterEnv {
     rng: Option<ChaCha20Rng>,
     time_warp: GovTimeWarp,

rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto

@@ -2985,3 +2985,13 @@ message ArchivedMonthlyNodeProviderRewards {
     ic_nns_governance.pb.v1.MonthlyNodeProviderRewards rewards = 1;
   }
 }
+
+// Internal type to allow ProposalVotingStateMachine to be stored
+// in stable memory.
+message ProposalVotingStateMachine {
+  ic_nns_common.pb.v1.ProposalId proposal_id = 1;
+  Topic topic = 2;
+  repeated ic_nns_common.pb.v1.NeuronId neurons_to_check_followers = 3;
+  repeated ic_nns_common.pb.v1.NeuronId followers_to_check = 4;
+  map<uint64, Vote> recent_neuron_ballots_to_record = 5;
+}

rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs

@@ -4724,6 +4724,29 @@ pub mod archived_monthly_node_provider_rewards {
         Version1(V1),
     }
 }
+/// Internal type to allow ProposalVotingStateMachine to be stored
+/// in stable memory.
+#[derive(
+    candid::CandidType,
+    candid::Deserialize,
+    serde::Serialize,
+    comparable::Comparable,
+    Clone,
+    PartialEq,
+    ::prost::Message,
+)]
+pub struct ProposalVotingStateMachine {
+    #[prost(message, optional, tag = "1")]
+    pub proposal_id: ::core::option::Option<::ic_nns_common::pb::v1::ProposalId>,
+    #[prost(enumeration = "Topic", tag = "2")]
+    pub topic: i32,
+    #[prost(message, repeated, tag = "3")]
+    pub neurons_to_check_followers: ::prost::alloc::vec::Vec<::ic_nns_common::pb::v1::NeuronId>,
+    #[prost(message, repeated, tag = "4")]
+    pub followers_to_check: ::prost::alloc::vec::Vec<::ic_nns_common::pb::v1::NeuronId>,
+    #[prost(map = "uint64, enumeration(Vote)", tag = "5")]
+    pub recent_neuron_ballots_to_record: ::std::collections::HashMap<u64, i32>,
+}
 /// Proposal types are organized into topics. Neurons can automatically
 /// vote based on following other neurons, and these follow
 /// relationships are defined per topic.

rs/nns/governance/src/governance.rs

@@ -136,6 +136,7 @@ pub mod tla_macros;
 #[cfg(feature = "tla")]
 pub mod tla;
 
+use crate::storage::with_voting_state_machines_mut;
 #[cfg(feature = "tla")]
 pub use tla::{
     tla_update_method, InstrumentationState, ToTla, CLAIM_NEURON_DESC, MERGE_NEURONS_DESC,
@@ -1065,6 +1066,9 @@ impl ProposalData {
             0 => {
                 if self.accepts_vote(now_seconds, voting_period_seconds) {
                     ProposalRewardStatus::AcceptVotes
+                    // voting_is_finished means !accepts_vote && some votes are still being processed
+                } else if !self.voting_is_finished(now_seconds, voting_period_seconds) {
+                    ProposalRewardStatus::VotesProcessing
                 } else {
                     ProposalRewardStatus::ReadyToSettle
                 }
@@ -1105,6 +1109,19 @@ impl ProposalData {
         now_seconds < self.get_deadline_timestamp_seconds(voting_period_seconds)
     }
 
+    /// Returns if voting is closed along with whether or not any outstanding votes still
+    /// need to be cast (when processing following spans multiple messages).
+    pub fn voting_is_finished(&self, now_seconds: u64, voting_period_seconds: u64) -> bool {
+        let voting_closed = !self.accepts_vote(now_seconds, voting_period_seconds);
+        let votes_processed = with_voting_state_machines_mut(|vsm| {
+            vsm.with_machine(self.id.unwrap(), self.topic(), |machine| {
+                machine.is_voting_finished()
+            })
+        });
+
+        voting_closed && votes_processed
+    }
+
     pub fn evaluate_wait_for_quiet(
         &mut self,
         now_seconds: u64,
@@ -1271,7 +1288,7 @@ impl ProposalData {
             // equivalent to (2 * yes > total) || (2 * no >= total).
             let majority =
                 (tally.yes > tally.total - tally.yes) || (tally.no >= tally.total - tally.no);
-            let expired = !self.accepts_vote(now_seconds, voting_period_seconds);
+            let expired = self.voting_is_finished(now_seconds, voting_period_seconds);
             let decision_reason = match (majority, expired) {
                 (true, true) => Some("majority and expiration"),
                 (true, false) => Some("majority"),
@@ -4110,7 +4127,7 @@ impl Governance {
         // to have Open status while it does not accept votes anymore, since
         // the status change happens below this point.
         if proposal.status() == ProposalStatus::Open
-            || proposal.accepts_vote(now_seconds, voting_period_seconds)
+            || !proposal.voting_is_finished(now_seconds, voting_period_seconds)
         {
             proposal.recompute_tally(now_seconds, voting_period_seconds);
         }

rs/nns/governance/src/governance/benches.rs

@@ -467,21 +467,6 @@ fn centralized_following_all_stable() -> BenchResult {
     )
 }
 
-/// Benchmark the `cascading_vote` function with stable neurons and a heap index.
-/// Before we do the migration of the ballots function to be more efficient:
-/// Benchmark: compute_ballots_for_new_proposal_with_stable_neurons (new)
-//   total:
-//     instructions: 78.49 M (new)
-//     heap_increase: 0 pages (new)
-//     stable_memory_increase: 0 pages (new)
-//
-// After we migrate to be more efficient:
-// Benchmark: compute_ballots_for_new_proposal_with_stable_neurons (new)
-//   total:
-//     instructions: 1.50 M (new)
-//     heap_increase: 0 pages (new)
-//     stable_memory_increase: 0 pages (new)
-//
 #[bench(raw)]
 fn compute_ballots_for_new_proposal_with_stable_neurons() -> BenchResult {
     let now_seconds = 1732817584;

rs/nns/governance/src/governance/tests/mod.rs

@@ -844,6 +844,7 @@ mod convert_create_service_nervous_system_proposal_to_sns_init_payload_tests_wit
 
 mod metrics_tests {
 
+    use ic_nns_common::pb::v1::ProposalId;
     use maplit::btreemap;
 
     use crate::{
@@ -858,6 +859,7 @@ mod metrics_tests {
     #[test]
     fn test_metrics_total_voting_power() {
         let proposal_1 = ProposalData {
+            id: Some(ProposalId { id: 1 }),
             proposal: Some(Proposal {
                 title: Some("Foo Foo Bar".to_string()),
                 action: Some(proposal::Action::Motion(Motion {
@@ -875,6 +877,7 @@ mod metrics_tests {
         };
 
         let proposal_2 = ProposalData {
+            id: Some(ProposalId { id: 2 }),
             proposal: Some(Proposal {
                 title: Some("Foo Foo Bar".to_string()),
                 action: Some(proposal::Action::ManageNeuron(Box::default())),
@@ -923,6 +926,7 @@ mod metrics_tests {
         });
 
         let open_proposal = ProposalData {
+            id: Some(ProposalId { id: 1 }),
             proposal: Some(Proposal {
                 title: Some("open_proposal".to_string()),
                 action: Some(manage_neuron_action.clone()),
@@ -932,6 +936,7 @@ mod metrics_tests {
         };
 
         let rejected_proposal = ProposalData {
+            id: Some(ProposalId { id: 2 }),
             proposal: Some(Proposal {
                 title: Some("rejected_proposal".to_string()),
                 action: Some(manage_neuron_action.clone()),
@@ -942,6 +947,7 @@ mod metrics_tests {
         };
 
         let motion_proposal = ProposalData {
+            id: Some(ProposalId { id: 3 }),
             proposal: Some(Proposal {
                 title: Some("Foo Foo Bar".to_string()),
                 action: Some(motion_action.clone()),

rs/nns/governance/src/neuron_store/benches.rs

@@ -122,6 +122,7 @@ fn build_neuron(rng: &mut impl RngCore, location: NeuronLocation, size: NeuronSi
             vote: Vote::Yes as i32,
         })
         .collect();
+    neuron.recent_ballots_next_entry_index = Some(0);
 
     neuron
 }