VID share distribution algorithm (2)

crates/hotshot/examples/infra/mod.rs

@@ -153,9 +153,9 @@ async fn webserver_network_from_config<TYPES: NodeType>(
     let WebServerConfig {
         url,
         wait_between_polls,
-    }: WebServerConfig = config.clone().web_server_config.unwrap();
+    }: WebServerConfig = config.web_server_config.unwrap();
 
-    WebServerNetwork::create(url, wait_between_polls, pub_key.clone(), false)
+    WebServerNetwork::create(url, wait_between_polls, pub_key, false)
 }
 
 async fn libp2p_network_from_config<TYPES: NodeType>(

crates/hotshot/src/tasks/mod.rs

@@ -362,6 +362,7 @@ pub async fn add_da_task<TYPES: NodeType, I: NodeImplementation<TYPES>>(
         consensus: handle.hotshot.get_consensus(),
         da_membership: c_api.inner.memberships.da_membership.clone().into(),
         da_network: c_api.inner.networks.da_network.clone().into(),
+        quorum_membership: c_api.inner.memberships.quorum_membership.clone().into(),
         cur_view: TYPES::Time::new(0),
         vote_collector: None,
         event_stream: event_stream.clone(),

crates/hotshot/src/traits/storage/memory_storage.rs

@@ -117,7 +117,10 @@ mod test {
     use hotshot_types::{
         data::{fake_commitment, genesis_proposer_id, Leaf},
         simple_certificate::QuorumCertificate,
-        traits::{node_implementation::NodeType, state::ConsensusTime},
+        traits::{
+            block_contents::genesis_vid_commitment, node_implementation::NodeType,
+            state::ConsensusTime,
+        },
     };
     use std::marker::PhantomData;
     use tracing::instrument;
@@ -126,7 +129,7 @@ mod test {
         let payload = TestBlockPayload::genesis();
         let header = TestBlockHeader {
             block_number: 0,
-            payload_commitment: payload.payload_commitment,
+            payload_commitment: genesis_vid_commitment(),
         };
         let dummy_leaf_commit = fake_commitment::<Leaf<TestTypes>>();
         let data = hotshot_types::simple_vote::QuorumData {

crates/task-impls/src/consensus.rs

@@ -614,21 +614,12 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
                                 if let Some(encoded_txns) =
                                     consensus.saved_payloads.get(leaf.get_payload_commitment())
                                 {
-                                    let num_quorum_committee = self.quorum_membership.total_nodes();
-
                                     let payload = BlockPayload::from_bytes(
                                         encoded_txns.clone().into_iter(),
                                         leaf.get_block_header().metadata(),
-                                        num_quorum_committee,
                                     );
-                                    if let Err(e) =
-                                        leaf.fill_block_payload(payload, num_quorum_committee)
-                                    {
-                                        error!(
-                                            "Saved block payload and commitment don't match: {:?}",
-                                            e
-                                        );
-                                    }
+
+                                    leaf.fill_block_payload_unchecked(payload);
                                 }
 
                                 leaf_views.push(leaf.clone());

crates/task-impls/src/da.rs

@@ -57,6 +57,11 @@ pub struct DATaskState<
     /// Membership for the DA committee
     pub da_membership: Arc<TYPES::Membership>,
 
+    /// Membership for the quorum committee
+    /// We need this only for calculating the proper VID scheme
+    /// from the number of nodes in the quorum.
+    pub quorum_membership: Arc<TYPES::Membership>,
+
     /// Network for DA
     pub da_network: Arc<I::CommitteeNetwork>,
 
@@ -86,7 +91,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
         event: HotShotEvent<TYPES>,
     ) -> Option<HotShotTaskCompleted> {
         match event {
-            HotShotEvent::DAProposalRecv(proposal, sender, num_quorum_committee) => {
+            HotShotEvent::DAProposalRecv(proposal, sender) => {
                 debug!(
                     "DA proposal received for view: {:?}",
                     proposal.data.get_view_number()
@@ -113,8 +118,10 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
                     return None;
                 }
 
-                let payload_commitment =
-                    vid_commitment(&proposal.data.encoded_transactions, num_quorum_committee);
+                let payload_commitment = vid_commitment(
+                    &proposal.data.encoded_transactions,
+                    self.quorum_membership.total_nodes(),
+                );
                 let encoded_transactions_hash = Sha256::digest(&proposal.data.encoded_transactions);
 
                 // ED Is this the right leader?
@@ -252,12 +259,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
 
                 return None;
             }
-            HotShotEvent::TransactionsSequenced(
-                encoded_transactions,
-                metadata,
-                view,
-                num_quorum_committee,
-            ) => {
+            HotShotEvent::TransactionsSequenced(encoded_transactions, metadata, view) => {
                 self.da_network
                     .inject_consensus_info(ConsensusIntentEvent::CancelPollForTransactions(*view))
                     .await;
@@ -286,7 +288,6 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
                     .publish(HotShotEvent::DAProposalSend(
                         message.clone(),
                         self.public_key.clone(),
-                        num_quorum_committee,
                     ))
                     .await;
             }
@@ -312,10 +313,10 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
     pub fn filter(event: &HotShotEvent<TYPES>) -> bool {
         matches!(
             event,
-            HotShotEvent::DAProposalRecv(_, _, _)
+            HotShotEvent::DAProposalRecv(_, _)
                 | HotShotEvent::DAVoteRecv(_)
                 | HotShotEvent::Shutdown
-                | HotShotEvent::TransactionsSequenced(_, _, _, _)
+                | HotShotEvent::TransactionsSequenced(_, _, _)
                 | HotShotEvent::Timeout(_)
                 | HotShotEvent::ViewChange(_)
         )

crates/task-impls/src/events.rs

@@ -29,11 +29,7 @@ pub enum HotShotEvent<TYPES: NodeType> {
     /// Send a timeout vote to the network; emitted by consensus task replicas
     TimeoutVoteSend(TimeoutVote<TYPES>),
     /// A DA proposal has been received from the network; handled by the DA task
-    DAProposalRecv(
-        Proposal<TYPES, DAProposal<TYPES>>,
-        TYPES::SignatureKey,
-        usize,
-    ),
+    DAProposalRecv(Proposal<TYPES, DAProposal<TYPES>>, TYPES::SignatureKey),
     /// A DA vote has been received by the network; handled by the DA task
     DAVoteRecv(DAVote<TYPES>),
     /// A Data Availability Certificate (DAC) has been recieved by the network; handled by the consensus task
@@ -43,11 +39,7 @@ pub enum HotShotEvent<TYPES: NodeType> {
     /// Send a quorum vote to the next leader; emitted by a replica in the consensus task after seeing a valid quorum proposal
     QuorumVoteSend(QuorumVote<TYPES>),
     /// Send a DA proposal to the DA committee; emitted by the DA leader (which is the same node as the leader of view v + 1) in the DA task
-    DAProposalSend(
-        Proposal<TYPES, DAProposal<TYPES>>,
-        TYPES::SignatureKey,
-        usize,
-    ),
+    DAProposalSend(Proposal<TYPES, DAProposal<TYPES>>, TYPES::SignatureKey),
     /// Send a DA vote to the DA leader; emitted by DA committee members in the DA task after seeing a valid DA proposal
     DAVoteSend(DAVote<TYPES>),
     /// The next leader has collected enough votes to form a QC; emitted by the next leader in the consensus task; an internal event only
@@ -100,12 +92,11 @@ pub enum HotShotEvent<TYPES: NodeType> {
         VidCommitment,
         <TYPES::BlockPayload as BlockPayload>::Metadata,
     ),
-    /// Event when the transactions task has sequenced transactions. Contains the encoded transactions
+    /// Event when the transactions task has sequenced transactions. Contains the encoded transactions, the metadata, and the view number
     TransactionsSequenced(
         Vec<u8>,
         <TYPES::BlockPayload as BlockPayload>::Metadata,
         TYPES::Time,
-        usize,
     ),
     /// Event when the transactions task has a block formed
     BlockReady(VidDisperse<TYPES>, TYPES::Time),

crates/task-impls/src/network.rs

@@ -56,7 +56,7 @@ impl<TYPES: NodeType> NetworkMessageTaskState<TYPES> {
                     let event = match consensus_message.0 {
                         Either::Left(general_message) => match general_message {
                             GeneralConsensusMessage::Proposal(proposal) => {
-                                HotShotEvent::QuorumProposalRecv(proposal.clone(), sender)
+                                HotShotEvent::QuorumProposalRecv(proposal, sender)
                             }
                             GeneralConsensusMessage::Vote(vote) => {
                                 HotShotEvent::QuorumVoteRecv(vote.clone())
@@ -91,14 +91,9 @@ impl<TYPES: NodeType> NetworkMessageTaskState<TYPES> {
                             }
                         },
                         Either::Right(committee_message) => match committee_message {
-                            CommitteeConsensusMessage::DAProposal(
-                                proposal,
-                                num_quorum_committee,
-                            ) => HotShotEvent::DAProposalRecv(
-                                proposal.clone(),
-                                sender,
-                                num_quorum_committee,
-                            ),
+                            CommitteeConsensusMessage::DAProposal(proposal) => {
+                                HotShotEvent::DAProposalRecv(proposal, sender)
+                            }
                             CommitteeConsensusMessage::DAVote(vote) => {
                                 HotShotEvent::DAVoteRecv(vote.clone())
                             }
@@ -189,10 +184,10 @@ impl<TYPES: NodeType, COMMCHANNEL: CommunicationChannel<TYPES>>
                 TransmitType::Broadcast, // TODO not a broadcast https://github.com/EspressoSystems/HotShot/issues/1696
                 None,
             ),
-            HotShotEvent::DAProposalSend(proposal, sender, num_quorum_committee) => (
+            HotShotEvent::DAProposalSend(proposal, sender) => (
                 sender,
                 MessageKind::<TYPES>::from_consensus_message(SequencingMessage(Right(
-                    CommitteeConsensusMessage::DAProposal(proposal, num_quorum_committee),
+                    CommitteeConsensusMessage::DAProposal(proposal),
                 ))),
                 TransmitType::Broadcast,
                 None,
@@ -241,7 +236,7 @@ impl<TYPES: NodeType, COMMCHANNEL: CommunicationChannel<TYPES>>
             HotShotEvent::ViewSyncPreCommitCertificate2Send(certificate, sender) => (
                 sender,
                 MessageKind::<TYPES>::from_consensus_message(SequencingMessage(Left(
-                    GeneralConsensusMessage::ViewSyncPreCommitCertificate(certificate.clone()),
+                    GeneralConsensusMessage::ViewSyncPreCommitCertificate(certificate),
                 ))),
                 TransmitType::Broadcast,
                 None,
@@ -333,7 +328,7 @@ impl<TYPES: NodeType, COMMCHANNEL: CommunicationChannel<TYPES>>
     fn committee_filter(event: &HotShotEvent<TYPES>) -> bool {
         matches!(
             event,
-            HotShotEvent::DAProposalSend(_, _, _)
+            HotShotEvent::DAProposalSend(_, _)
                 | HotShotEvent::DAVoteSend(_)
                 | HotShotEvent::Shutdown
                 | HotShotEvent::ViewChange(_)

crates/task-impls/src/transactions.rs

@@ -206,17 +206,11 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
 
                 drop(consensus);
 
-                // get the number of storage nodes for VID
-                let num_quorum_committee = self.membership.total_nodes();
-
                 // TODO (Keyao) Determine whether to allow empty blocks.
                 // <https://github.com/EspressoSystems/HotShot/issues/1822>
                 let txns = self.wait_for_transactions(parent_leaf).await?;
                 let (payload, metadata) =
-                    match <TYPES::BlockPayload as BlockPayload>::from_transactions(
-                        txns,
-                        num_quorum_committee,
-                    ) {
+                    match <TYPES::BlockPayload as BlockPayload>::from_transactions(txns) {
                         Ok((payload, metadata)) => (payload, metadata),
                         Err(e) => {
                             error!("Failed to build the block payload: {:?}.", e);
@@ -239,7 +233,6 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
                         encoded_transactions,
                         metadata,
                         view + 1,
-                        num_quorum_committee,
                     ))
                     .await;
 

crates/task-impls/src/vid.rs

@@ -77,12 +77,10 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
         event: HotShotEvent<TYPES>,
     ) -> Option<HotShotTaskCompleted> {
         match event {
-            HotShotEvent::TransactionsSequenced(
-                encoded_transactions,
-                metadata,
-                view_number,
-                num_quorum_committee,
-            ) => {
+            HotShotEvent::TransactionsSequenced(encoded_transactions, metadata, view_number) => {
+                // get the number of quorum committee members to be used for VID calculation
+                let num_quorum_committee = self.membership.total_nodes();
+
                 // TODO <https://github.com/EspressoSystems/HotShot/issues/1686>
                 let srs = test_srs(num_quorum_committee);
 
@@ -170,7 +168,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
         matches!(
             event,
             HotShotEvent::Shutdown
-                | HotShotEvent::TransactionsSequenced(_, _, _, _)
+                | HotShotEvent::TransactionsSequenced(_, _, _)
                 | HotShotEvent::BlockReady(_, _)
                 | HotShotEvent::ViewChange(_)
         )

crates/testing/src/block_types.rs

@@ -5,9 +5,9 @@ use std::{
 
 use commit::{Commitment, Committable};
 use hotshot_types::{
-    data::{BlockError, VidCommitment, VidScheme, VidSchemeTrait},
+    data::{BlockError, VidCommitment},
     traits::{
-        block_contents::{vid_commitment, BlockHeader, Transaction, NUM_STORAGE_NODES},
+        block_contents::{genesis_vid_commitment, BlockHeader, Transaction},
         state::TestableBlock,
         BlockPayload,
     },
@@ -67,8 +67,6 @@ impl Transaction for TestTransaction {}
 pub struct TestBlockPayload {
     /// List of transactions.
     pub transactions: Vec<TestTransaction>,
-    /// VID commitment to the block payload.
-    pub payload_commitment: <VidScheme as VidSchemeTrait>::Commit,
 }
 
 impl TestBlockPayload {
@@ -80,10 +78,8 @@ impl TestBlockPayload {
     pub fn genesis() -> Self {
         let txns: Vec<u8> = vec![0];
         // It's impossible for `encode` to fail because the transaciton length is very small.
-        let encoded = TestTransaction::encode(vec![TestTransaction(txns.clone())]).unwrap();
         TestBlockPayload {
             transactions: vec![TestTransaction(txns)],
-            payload_commitment: vid_commitment(&encoded, NUM_STORAGE_NODES),
         }
     }
 }
@@ -112,24 +108,17 @@ impl BlockPayload for TestBlockPayload {
 
     fn from_transactions(
         transactions: impl IntoIterator<Item = Self::Transaction>,
-        num_storage_nodes: usize,
     ) -> Result<(Self, Self::Metadata), Self::Error> {
         let txns_vec: Vec<TestTransaction> = transactions.into_iter().collect();
-        let encoded = TestTransaction::encode(txns_vec.clone())?;
         Ok((
             Self {
                 transactions: txns_vec,
-                payload_commitment: vid_commitment(&encoded, num_storage_nodes),
             },
             (),
         ))
     }
 
-    fn from_bytes<E>(
-        encoded_transactions: E,
-        _metadata: Self::Metadata,
-        num_storage_nodes: usize,
-    ) -> Self
+    fn from_bytes<E>(encoded_transactions: E, _metadata: Self::Metadata) -> Self
     where
         E: Iterator<Item = u8>,
     {
@@ -151,10 +140,7 @@ impl BlockPayload for TestBlockPayload {
             current_index = next_index;
         }
 
-        Self {
-            transactions,
-            payload_commitment: vid_commitment(&encoded_vec, num_storage_nodes),
-        }
+        Self { transactions }
     }
 
     fn genesis() -> (Self, Self::Metadata) {
@@ -205,7 +191,7 @@ impl BlockHeader for TestBlockHeader {
         (
             Self {
                 block_number: 0,
-                payload_commitment: payload.payload_commitment,
+                payload_commitment: genesis_vid_commitment(),
             },
             payload,
             metadata,