Enforce max_block_size

Cargo.toml

@@ -46,19 +46,19 @@ dotenvy = "0.15"
 ethers = { version = "2.0", features = ["solc"] }
 futures = "0.3"
 
-hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.48" }
+hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.50" }
+
 # Hotshot imports
-hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.48" }
-hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.17" }
-hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.18" }
-hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.48" }
-hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.16" }
-hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.48" }
+hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.50" }
+hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "0.1.18" }
+hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.19" }
+hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.50" }
+hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.17" }
+hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.50" }
 hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" }
-hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.48" }
-hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.48" }
-hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.48" }
-hotshot-web-server = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.48" }
+hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.50" }
+hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.50" }
+hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.50" }
 
 # Push CDN imports
 cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", features = [

builder/src/lib.rs

@@ -10,8 +10,7 @@ use futures::{
 };
 use hotshot::{
     traits::{
-        election::static_committee::GeneralStaticCommittee,
-        implementations::{NetworkingMetricsValue, WebServerNetwork},
+        election::static_committee::GeneralStaticCommittee, implementations::NetworkingMetricsValue,
     },
     types::{SignatureKey, SystemContextHandle},
     HotShotInitializer, Memberships, Networks, SystemContext,
@@ -706,7 +705,7 @@ mod test {
         let mut parent = {
             // TODO refactor repeated code from other tests
             let (genesis_payload, genesis_ns_table) =
-                Payload::from_transactions([], Arc::new(genesis_state.clone()))
+                Payload::from_transactions([], &genesis_state)
                     .expect("unable to create genesis payload");
             let builder_commitment = genesis_payload.builder_commitment(&genesis_ns_table);
             let genesis_commitment = {

builder/src/non_permissioned.rs

@@ -101,9 +101,8 @@ impl BuilderConfig {
         // builder api request channel
         let (req_sender, req_receiver) = broadcast::<MessageType<SeqTypes>>(channel_capacity.get());
 
-        let (genesis_payload, genesis_ns_table) =
-            Payload::from_transactions([], Arc::new(instance_state.clone()))
-                .expect("genesis payload construction failed");
+        let (genesis_payload, genesis_ns_table) = Payload::from_transactions([], &instance_state)
+            .expect("genesis payload construction failed");
 
         let builder_commitment = genesis_payload.builder_commitment(&genesis_ns_table);
 
@@ -165,37 +164,21 @@ impl BuilderConfig {
         // start the hotshot api service
         run_builder_api_service(hotshot_builder_apis_url.clone(), proxy_global_state);
 
-        // create a client for it
-        // Start Client for the event streaming api
-        tracing::info!(
-            "Builder client connecting to hotshot events API at {}",
-            hotshot_events_api_url.to_string()
-        );
-        let client = Client::<EventStreamApiError, Version01>::new(hotshot_events_api_url.clone());
-
-        assert!(client.connect(None).await);
-
-        tracing::info!("Builder client connected to the hotshot events api");
-
-        // client subscrive to hotshot events
-        let subscribed_events = client
-            .socket("hotshot-events/events")
-            .subscribe::<BuilderEvent<SeqTypes>>()
-            .await
-            .unwrap();
-
-        tracing::info!("Builder client subscribed to hotshot events");
-
         // spawn the builder service
+        let events_url = hotshot_events_api_url
+            .clone()
+            .join("hotshot-events/events")
+            .unwrap();
         async_spawn(async move {
-            run_non_permissioned_standalone_builder_service(
+            let res = run_non_permissioned_standalone_builder_service(
                 tx_sender,
                 da_sender,
                 qc_sender,
                 decide_sender,
-                subscribed_events,
+                events_url,
             )
             .await;
+            tracing::error!(?res, "builder service exited")
         });
 
         tracing::info!("Builder init finished");

builder/src/permissioned.rs

@@ -13,7 +13,7 @@ use hotshot::{
         election::static_committee::GeneralStaticCommittee,
         implementations::{
             derive_libp2p_peer_id, CombinedNetworks, KeyPair, Libp2pNetwork,
-            NetworkingMetricsValue, PushCdnNetwork, WebServerNetwork, WrappedSignatureKey,
+            NetworkingMetricsValue, PushCdnNetwork, Topic, WrappedSignatureKey,
         },
     },
     types::{SignatureKey, SystemContextHandle},
@@ -188,7 +188,7 @@ pub async fn init_node<P: SequencerPersistence, Ver: StaticVersionType + 'static
     // Initialize the push CDN network (and perform the initial connection)
     let cdn_network = PushCdnNetwork::new(
         network_params.cdn_endpoint,
-        vec!["Global".into(), "DA".into()],
+        vec![Topic::Global, Topic::DA],
         KeyPair {
             public_key: WrappedSignatureKey(my_config.public_key),
             private_key: my_config.private_key.clone(),
@@ -400,9 +400,8 @@ impl<N: network::Type, P: SequencerPersistence, Ver: StaticVersionType + 'static
         // builder api request channel
         let (req_sender, req_receiver) = broadcast::<MessageType<SeqTypes>>(channel_capacity.get());
 
-        let (genesis_payload, genesis_ns_table) =
-            Payload::from_transactions([], Arc::new(instance_state.clone()))
-                .expect("genesis payload construction failed");
+        let (genesis_payload, genesis_ns_table) = Payload::from_transactions([], &instance_state)
+            .expect("genesis payload construction failed");
 
         let builder_commitment = genesis_payload.builder_commitment(&genesis_ns_table);
 

sequencer/Cargo.toml

@@ -60,7 +60,6 @@ hotshot-task = { workspace = true }
 # Dependencies for feature `testing`
 hotshot-testing = { workspace = true, optional = true }
 hotshot-types = { workspace = true }
-hotshot-web-server = { workspace = true }
 include_dir = "0.7"
 itertools = { workspace = true }
 

sequencer/src/block.rs

@@ -1,13 +1,12 @@
-use std::sync::Arc;
-
 use crate::{BlockBuildingSnafu, NodeState, Transaction};
 use committable::{Commitment, Committable};
 use hotshot_query_service::availability::QueryablePayload;
-use hotshot_types::traits::{states::InstanceState, BlockPayload};
+use hotshot_types::traits::BlockPayload;
 use hotshot_types::utils::BuilderCommitment;
 use serde::{Deserialize, Serialize};
 use sha2::Digest;
 use snafu::OptionExt;
+use std::sync::Arc;
 
 pub mod entry;
 pub mod payload;
@@ -24,6 +23,7 @@ pub type NsTable = NameSpaceTable<TxTableEntryWord>;
 impl BlockPayload for Payload<TxTableEntryWord> {
     type Error = crate::Error;
     type Transaction = Transaction;
+    type Instance = NodeState;
     type Metadata = NsTable;
 
     /// Returns (Self, metadata).
@@ -48,9 +48,9 @@ impl BlockPayload for Payload<TxTableEntryWord> {
     /// TODO(746) refactor and make pretty "table" code for tx, namespace tables?
     fn from_transactions(
         txs: impl IntoIterator<Item = Self::Transaction>,
-        _state: Arc<dyn InstanceState>,
+        instance_state: &Self::Instance,
     ) -> Result<(Self, Self::Metadata), Self::Error> {
-        let payload = Payload::from_txs(txs)?;
+        let payload = Payload::from_txs(txs, &instance_state.chain_config)?;
         let ns_table = payload.get_ns_table().clone(); // TODO don't clone ns_table
         Some((payload, ns_table)).context(BlockBuildingSnafu)
     }
@@ -69,7 +69,7 @@ impl BlockPayload for Payload<TxTableEntryWord> {
         // use the mock NodeState. A future update to HotShot should
         // make a change there to remove the need for this workaround.
 
-        Self::from_transactions([], Arc::new(NodeState::mock())).unwrap()
+        Self::from_transactions([], &NodeState::mock()).unwrap()
     }
 
     fn encode(&self) -> Result<Arc<[u8]>, Self::Error> {

sequencer/src/block/payload.rs

@@ -1,6 +1,8 @@
 use crate::block::entry::{TxTableEntry, TxTableEntryWord};
 use crate::block::payload;
-use crate::{BlockBuildingSnafu, Error, NamespaceId, NodeState, Transaction};
+use crate::block::tables::NameSpaceTable;
+use crate::block::tables::TxTable;
+use crate::{BlockBuildingSnafu, ChainConfig, Error, NamespaceId, Transaction};
 use ark_serialize::{CanonicalDeserialize, CanonicalSerialize};
 use derivative::Derivative;
 use hotshot::traits::BlockPayload;
@@ -15,14 +17,10 @@ use num_traits::PrimInt;
 use serde::{Deserialize, Serialize};
 use snafu::OptionExt;
 use std::default::Default;
-use std::sync::Arc;
+use std::mem::size_of;
 use std::{collections::HashMap, fmt::Display};
-
-use crate::block::tables::NameSpaceTable;
 use trait_set::trait_set;
 
-use crate::block::tables::TxTable;
-
 trait_set! {
 
     pub trait TableWordTraits = CanonicalSerialize
@@ -155,13 +153,27 @@ impl<TableWord: TableWordTraits> Payload<TableWord> {
 
     pub fn from_txs(
         txs: impl IntoIterator<Item = <payload::Payload<TxTableEntryWord> as BlockPayload>::Transaction>,
+        chain_config: &ChainConfig,
     ) -> Result<Self, Error> {
         let mut namespaces: HashMap<NamespaceId, NamespaceInfo> = Default::default();
         let mut structured_payload = Self {
             raw_payload: vec![],
             ns_table: NameSpaceTable::default(),
         };
+
+        let mut block_size = 0u64;
         for tx in txs.into_iter() {
+            block_size += (tx.payload().len() + size_of::<TxTableEntry>()) as u64;
+
+            // block_size is updated when we encounter a new namespace
+            if namespaces.get(&tx.namespace()).is_none() {
+                block_size += size_of::<TxTableEntry>() as u64;
+            }
+
+            if block_size > chain_config.max_block_size() {
+                break;
+            }
+
             Payload::<TableWord>::update_namespace_with_tx(&mut namespaces, tx);
         }
 
@@ -309,7 +321,7 @@ impl hotshot_types::traits::block_contents::TestableBlock
     for Payload<crate::block::entry::TxTableEntryWord>
 {
     fn genesis() -> Self {
-        BlockPayload::from_transactions([], Arc::new(NodeState::mock()))
+        BlockPayload::from_transactions([], &Default::default())
             .unwrap()
             .0
     }
@@ -332,18 +344,59 @@ mod test {
             tx_iterator::TxIndex,
         },
         transaction::NamespaceId,
-        Transaction,
+        ChainConfig, NodeState, Transaction,
     };
     use async_compatibility_layer::logging::{setup_backtrace, setup_logging};
     use helpers::*;
     use hotshot_query_service::availability::QueryablePayload;
-    use hotshot_types::{traits::BlockPayload, vid::vid_scheme};
+    use hotshot_types::{
+        traits::{block_contents::TestableBlock, BlockPayload},
+        vid::vid_scheme,
+    };
     use jf_primitives::vid::{payload_prover::PayloadProver, VidScheme};
     use rand::RngCore;
-    use std::{collections::HashMap, marker::PhantomData, ops::Range, sync::Arc};
+    use std::{collections::HashMap, marker::PhantomData, mem::size_of, ops::Range};
 
     const NUM_STORAGE_NODES: usize = 10;
 
+    #[test]
+    fn enforce_max_block_size() {
+        // sum of all payloads + table entry of each
+        let target_payload_total = 1000usize;
+        // include name space entry in max_block_size
+        let max_block_size = (target_payload_total + size_of::<TxTableEntry>()) as u64;
+        let payload_size = 6;
+        // `tx_size` is payload + table entry size
+        let tx_size = (payload_size + size_of::<TxTableEntry>()) as u64;
+        // check our sanity
+        assert_eq!(tx_size, 10);
+
+        let n_txs = target_payload_total as u64 / tx_size;
+        let chain_config = ChainConfig::new(1, max_block_size, 1);
+
+        let mut txs = (0..n_txs)
+            .map(|_| Transaction::of_size(payload_size.unwrap()))
+            .collect::<Vec<Transaction>>();
+
+        assert_eq!(txs.len(), 100);
+
+        txs.push(Transaction::of_size(payload_size.unwrap()));
+
+        // The final txn will be omitted
+        let payload = Payload::<TxTableEntryWord>::from_txs(txs.clone(), &chain_config).unwrap();
+        let txs_len: usize = txs
+            .iter()
+            .map(|tx| tx.payload().len() + size_of::<TxTableEntry>())
+            .sum();
+        assert_eq!(payload.txn_count(), txs.len() as u64 - 1u64);
+
+        txs.pop();
+        // All txns will be included.
+        let payload = Payload::<TxTableEntryWord>::from_txs(txs.clone(), &chain_config).unwrap();
+
+        assert_eq!(payload.txn_count(), txs.len() as u64);
+    }
+
     #[test]
     fn basic_correctness() {
         check_basic_correctness::<TxTableEntryWord>()
@@ -464,8 +517,7 @@ mod test {
                 .iter()
                 .flat_map(|(_ns_id, ns)| ns.txs.iter().cloned());
             let (block, actual_ns_table) =
-                Payload::from_transactions(all_txs_iter, Arc::new(crate::NodeState::mock()))
-                    .unwrap();
+                Payload::from_transactions(all_txs_iter, &NodeState::mock()).unwrap();
             let disperse_data = vid.disperse(&block.raw_payload).unwrap();
 
             // TEST ACTUAL STUFF AGAINST DERIVED STUFF