Merge pull request #1814 from EspressoSystems/keyao/refactor-payload

Refactor block payload and VID disperse

crates/hotshot/Cargo.toml

@@ -108,7 +108,6 @@ libp2p-networking = { workspace = true }
 rand = { workspace = true }
 rand_chacha = { workspace = true }
 serde = { workspace = true, features = ["rc"] }
-sha3 = "^0.10"
 snafu = { workspace = true }
 surf-disco = { workspace = true }
 time = { workspace = true }

crates/hotshot/examples/infra/modDA.rs

@@ -20,8 +20,8 @@ use hotshot_orchestrator::{
     config::{NetworkConfig, WebServerConfig},
 };
 use hotshot_task::task::FilterEvent;
-use hotshot_types::HotShotConfig;
 use hotshot_types::{
+    block_impl::{VIDBlockPayload, VIDTransaction},
     certificate::ViewSyncCertificate,
     data::{QuorumProposal, SequencingLeaf, TestableLeaf},
     event::{Event, EventType},
@@ -37,6 +37,7 @@ use hotshot_types::{
         },
         state::{ConsensusTime, TestableBlock, TestableState},
     },
+    HotShotConfig,
 };
 use libp2p_identity::{
     ed25519::{self, SecretKey},
@@ -407,7 +408,7 @@ pub struct WebServerDARun<
 
 #[async_trait]
 impl<
-        TYPES: NodeType,
+        TYPES: NodeType<Transaction = VIDTransaction, BlockType = VIDBlockPayload>,
         MEMBERSHIP: Membership<TYPES> + Debug,
         NODE: NodeImplementation<
             TYPES,
@@ -551,7 +552,7 @@ pub struct Libp2pDARun<TYPES: NodeType, I: NodeImplementation<TYPES>, MEMBERSHIP
 
 #[async_trait]
 impl<
-        TYPES: NodeType,
+        TYPES: NodeType<Transaction = VIDTransaction, BlockType = VIDBlockPayload>,
         MEMBERSHIP: Membership<TYPES> + Debug,
         NODE: NodeImplementation<
             TYPES,
@@ -765,7 +766,7 @@ where
 
 /// Main entry point for validators
 pub async fn main_entry_point<
-    TYPES: NodeType,
+    TYPES: NodeType<Transaction = VIDTransaction, BlockType = VIDBlockPayload>,
     MEMBERSHIP: Membership<TYPES> + Debug,
     DANETWORK: CommunicationChannel<TYPES, Message<TYPES, NODE>, MEMBERSHIP> + Debug,
     QUORUMNETWORK: CommunicationChannel<TYPES, Message<TYPES, NODE>, MEMBERSHIP> + Debug,

crates/hotshot/src/demo.rs

@@ -5,15 +5,13 @@
 //!
 //! These implementations are useful in examples and integration testing, but are not suitable for
 //! production use.
-use crate::{
-    block_impl::{BlockPayloadError, VIDBlockPayload, VIDTransaction},
-    traits::election::static_committee::{StaticElectionConfig, StaticVoteToken},
-};
+use crate::traits::election::static_committee::{StaticElectionConfig, StaticVoteToken};
 use commit::{Commitment, Committable};
 use derivative::Derivative;
 use either::Either;
 use hotshot_signature_key::bn254::BLSPubKey;
 use hotshot_types::{
+    block_impl::{BlockPayloadError, VIDBlockPayload, VIDTransaction},
     certificate::{AssembledSignature, QuorumCertificate},
     data::{
         fake_commitment, genesis_proposer_id, random_commitment, LeafType, SequencingLeaf,
@@ -72,10 +70,6 @@ impl State for SDemoState {
 
     type Time = ViewNumber;
 
-    fn next_block(_state: Option<Self>) -> Self::BlockType {
-        VIDBlockPayload(Vec::new())
-    }
-
     fn validate_block(&self, _block: &Self::BlockType, view_number: &Self::Time) -> bool {
         if view_number == &ViewNumber::genesis() {
             &self.view_number == view_number
@@ -109,7 +103,9 @@ impl TestableState for SDemoState {
         _rng: &mut dyn rand::RngCore,
         padding: u64,
     ) -> <Self::BlockType as BlockPayload>::Transaction {
-        VIDTransaction(vec![0; padding as usize])
+        /// clippy appeasement for `RANDOM_TX_BASE_SIZE`
+        const RANDOM_TX_BASE_SIZE: usize = 8;
+        VIDTransaction(vec![0; RANDOM_TX_BASE_SIZE + (padding as usize)])
     }
 }
 /// Implementation of [`NodeType`] for [`VDemoNode`]

crates/hotshot/src/lib.rs

@@ -19,7 +19,6 @@
 #[cfg(feature = "docs")]
 pub mod documentation;
 
-pub mod block_impl;
 /// Contains structures and functions for committee election
 pub mod certificate;
 #[cfg(feature = "demo")]
@@ -56,6 +55,7 @@ use hotshot_task::{
 use hotshot_task_impls::{events::SequencingHotShotEvent, network::NetworkTaskKind};
 
 use hotshot_types::{
+    block_impl::{VIDBlockPayload, VIDTransaction},
     certificate::{DACertificate, ViewSyncCertificate},
     consensus::{BlockStore, Consensus, ConsensusMetrics, View, ViewInner, ViewQueue},
     data::{DAProposal, DeltasType, LeafType, QuorumProposal, SequencingLeaf},
@@ -634,7 +634,7 @@ pub trait HotShotType<TYPES: NodeType, I: NodeImplementation<TYPES>> {
 
 #[async_trait]
 impl<
-        TYPES: NodeType,
+        TYPES: NodeType<Transaction = VIDTransaction, BlockType = VIDBlockPayload>,
         I: NodeImplementation<
             TYPES,
             Leaf = SequencingLeaf<TYPES>,

crates/hotshot/src/tasks/mod.rs

@@ -27,6 +27,7 @@ use hotshot_task_impls::{
     view_sync::{ViewSyncTaskState, ViewSyncTaskStateTypes},
 };
 use hotshot_types::{
+    block_impl::{VIDBlockPayload, VIDTransaction},
     certificate::ViewSyncCertificate,
     data::{ProposalType, QuorumProposal, SequencingLeaf},
     event::Event,
@@ -38,7 +39,6 @@ use hotshot_types::{
             CommitteeEx, ExchangesType, NodeImplementation, NodeType, ViewSyncEx,
         },
         state::ConsensusTime,
-        BlockPayload,
     },
     vote::{ViewSyncData, VoteType},
 };
@@ -250,7 +250,7 @@ where
 /// # Panics
 /// Is unable to panic. This section here is just to satisfy clippy
 pub async fn add_consensus_task<
-    TYPES: NodeType,
+    TYPES: NodeType<BlockType = VIDBlockPayload>,
     I: NodeImplementation<
         TYPES,
         Leaf = SequencingLeaf<TYPES>,
@@ -288,7 +288,7 @@ where
         consensus,
         timeout: handle.hotshot.inner.config.next_view_timeout,
         cur_view: TYPES::Time::new(0),
-        block: TYPES::BlockType::new(),
+        block: VIDBlockPayload::genesis(),
         quorum_exchange: c_api.inner.exchanges.quorum_exchange().clone().into(),
         api: c_api.clone(),
         committee_exchange: c_api.inner.exchanges.committee_exchange().clone().into(),
@@ -416,7 +416,7 @@ where
 /// # Panics
 /// Is unable to panic. This section here is just to satisfy clippy
 pub async fn add_transaction_task<
-    TYPES: NodeType,
+    TYPES: NodeType<Transaction = VIDTransaction, BlockType = VIDBlockPayload>,
     I: NodeImplementation<
         TYPES,
         Leaf = SequencingLeaf<TYPES>,

crates/orchestrator/default-libp2p-run-config.toml

@@ -69,6 +69,8 @@ num_bootstrap = 5
 secs = 0
 nanos = 0
 
+# TODO (Keyao) Clean up configuration parameters.
+# <https://github.com/EspressoSystems/HotShot/issues/1823>
 [config.propose_max_round_time]
-secs = 1
+secs = 2
 nanos = 0

crates/orchestrator/default-run-config.toml

@@ -55,8 +55,10 @@ num_bootstrap = 4
 secs = 0
 nanos = 0
 
+# TODO (Keyao) Clean up configuration parameters.
+# <https://github.com/EspressoSystems/HotShot/issues/1823>
 [config.propose_max_round_time]
-secs = 1
+secs = 2
 nanos = 0
 
 [web_server_config]

crates/orchestrator/default-web-server-run-config.toml

@@ -56,8 +56,10 @@ num_bootstrap = 4
 secs = 0
 nanos = 0
 
+# TODO (Keyao) Clean up configuration parameters.
+# <https://github.com/EspressoSystems/HotShot/issues/1823>
 [config.propose_max_round_time]
-secs = 1
+secs = 2
 nanos = 0
 
 [web_server_config]

crates/orchestrator/src/config.rs

@@ -230,7 +230,7 @@ fn default_config() -> HotShotConfigFile {
         total_nodes: NonZeroUsize::new(10).unwrap(),
         committee_nodes: 5,
         max_transactions: NonZeroUsize::new(100).unwrap(),
-        min_transactions: 0,
+        min_transactions: 1,
         next_view_timeout: 10000,
         timeout_ratio: (11, 10),
         round_start_delay: 1,

crates/task-impls/src/consensus.rs

@@ -1151,9 +1151,6 @@ where
         }
 
         let block_commitment = self.block.commit();
-        if block_commitment == TYPES::BlockType::new().commit() {
-            debug!("BlockPayload is generic block! {:?}", self.cur_view);
-        }
 
         let leaf = SequencingLeaf {
             view_number: view,

crates/task-impls/src/da.rs

@@ -18,7 +18,7 @@ use hotshot_types::vote::VoteType;
 use hotshot_types::{
     certificate::DACertificate,
     consensus::{Consensus, View},
-    data::{DAProposal, ProposalType, SequencingLeaf, VidDisperse, VidScheme, VidSchemeTrait},
+    data::{DAProposal, ProposalType, SequencingLeaf},
     message::{Message, Proposal, SequencingMessage},
     traits::{
         consensus_api::SequencingConsensusApi,
@@ -646,51 +646,13 @@ where
                 self.event_stream
                     .publish(SequencingHotShotEvent::SendDABlockData(block.clone()))
                     .await;
-                // if let Err(e) = self.api.send_da_broadcast(message.clone()).await {
-                //     consensus.metrics.failed_to_send_messages.add(1);
-                //     warn!(?message, ?e, "Could not broadcast leader proposal");
-                // } else {
-                //     consensus.metrics.outgoing_broadcast_messages.add(1);
-                // }
+
                 self.event_stream
                     .publish(SequencingHotShotEvent::DAProposalSend(
                         message.clone(),
                         self.committee_exchange.public_key().clone(),
                     ))
                     .await;
-
-                debug!("Prepare VID shares");
-                {
-                    /// TODO https://github.com/EspressoSystems/HotShot/issues/1693
-                    const NUM_STORAGE_NODES: usize = 10;
-                    /// TODO https://github.com/EspressoSystems/HotShot/issues/1693
-                    const NUM_CHUNKS: usize = 5;
-
-                    // TODO https://github.com/EspressoSystems/HotShot/issues/1686
-                    let srs = hotshot_types::data::test_srs(NUM_STORAGE_NODES);
-
-                    let vid = VidScheme::new(NUM_CHUNKS, NUM_STORAGE_NODES, &srs).unwrap();
-                    let message_bytes = bincode::serialize(&message).unwrap();
-                    let vid_disperse = vid.disperse(&message_bytes).unwrap();
-                    // TODO for now reuse the same block commitment and signature as DA committee
-                    // https://github.com/EspressoSystems/jellyfish/issues/369
-
-                    self.event_stream
-                        .publish(SequencingHotShotEvent::VidDisperseSend(
-                            Proposal {
-                                data: VidDisperse {
-                                    view_number: view,
-                                    commitment: block.commit(), // TODO GG should be vid_disperse.commit but that's a big change
-                                    shares: vid_disperse.shares,
-                                    common: vid_disperse.common,
-                                },
-                                signature: message.signature,
-                            },
-                            // TODO don't send to committee, send to quorum (consensus.rs) https://github.com/EspressoSystems/HotShot/issues/1696
-                            self.committee_exchange.public_key().clone(),
-                        ))
-                        .await;
-                }
             }
 
             SequencingHotShotEvent::Timeout(view) => {