diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 151e8a8a50..e7ac7ecdb4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: Build, Lint, and Test +name: Build and Test Workflow on: push: @@ -43,10 +43,6 @@ jobs: sudo apt-get install -y protobuf-compiler cargo install just - - name: Run linting - run: | - just ${{ matrix.just_variants }} lint - - name: Build all crates in workspace run: just ${{ matrix.just_variants }} build diff --git a/.github/workflows/lints.yml b/.github/workflows/lints.yml new file mode 100644 index 0000000000..61106f6d0c --- /dev/null +++ b/.github/workflows/lints.yml @@ -0,0 +1,42 @@ +name: Lints Workflow + +on: + push: + branches: + - 'main' + pull_request: + branches: + - 'main' + - '*/*' + workflow_dispatch: + +jobs: + lints: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: styfle/cancel-workflow-action@0.11.0 + name: Cancel Outdated Builds + with: + all_but_latest: true + access_token: ${{ github.token }} + + - name: Install Nix + uses: cachix/install-nix-action@v23 + + - uses: actions/checkout@v4 + name: Checkout Repository + + - uses: Swatinem/rust-cache@v2 + name: Enable Rust Caching + with: + shared-key: "" + prefix-key: lint + + - name: Format Check + run: cargo fmt -- --check + + - name: Clippy + run: | + nix develop -c just async_std lint + nix develop -c just tokio lint diff --git a/Cargo.lock b/Cargo.lock index 97d4568a8d..3c1554b852 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,9 +164,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6cd65a4b849ace0b7f6daeebcc1a1d111282227ca745458c61dbf670e52a597" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" dependencies = [ "anstyle", "anstyle-parse", @@ -202,9 +202,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "3.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0238ca56c96dfa37bdf7c373c8886dd591322500aceeeccdb2216fe06dc2f796" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" dependencies = [ "anstyle", "windows-sys", @@ -1208,9 +1208,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.6" +version = "4.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +checksum = "824956d0dca8334758a5b7f7e50518d66ea319330cbceedcf76905c2f6ab30e3" dependencies = [ "clap_builder", "clap_derive", @@ -1218,9 +1218,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.6" +version = "4.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +checksum = "122ec64120a49b4563ccaedcbea7818d069ed8e9aa6d829b82d8a4128936b2ab" dependencies = [ "anstream", "anstyle", @@ -1282,7 +1282,7 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "commit" version = "0.2.2" -source = "git+https://github.com/EspressoSystems/commit#5f1c28f1a109f2b36cf597e61a222614958db3b2" +source = "git+https://github.com/EspressoSystems/commit#100cb8f402e32d794b4d2213356227b7bfeb0e92" dependencies = [ "arbitrary", "ark-serialize 0.3.0", @@ -1290,6 +1290,7 @@ dependencies = [ "derivative", "derive_more", "funty", + "generic-array", "hex", "serde", "sha3", @@ -1392,7 +1393,7 @@ checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" dependencies = [ "aes-gcm 0.8.0", "base64 0.13.1", - "hkdf 0.10.0", + "hkdf", "hmac 0.10.1", "percent-encoding", "rand 0.8.5", @@ -2528,15 +2529,6 @@ dependencies = [ "hmac 0.10.1", ] -[[package]] -name = "hkdf" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" -dependencies = [ - "hmac 0.12.1", -] - [[package]] name = "hmac" version = "0.8.1" @@ -2557,15 +2549,6 @@ dependencies = [ "digest 0.9.0", ] -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", -] - [[package]] name = "hmac-drbg" version = "0.3.0" @@ -3287,7 +3270,7 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jf-primitives" version = "0.4.0-pre.0" -source = "git+https://github.com/EspressoSystems/jellyfish#1626a8448e7af8ea153241004375deff9fd961da" +source = "git+https://github.com/EspressoSystems/jellyfish#30752c6c1af7b933e2536d7aeb991a40af4b806a" dependencies = [ "anyhow", "ark-bls12-377", @@ -3311,6 +3294,7 @@ dependencies = [ "digest 0.10.7", "displaydoc", "espresso-systems-common 0.4.0", + "generic-array", "hashbrown 0.13.2", "itertools 0.10.5", "jf-relation", @@ -3331,7 +3315,7 @@ dependencies = [ [[package]] name = "jf-relation" version = "0.4.0-pre.0" -source = "git+https://github.com/EspressoSystems/jellyfish#1626a8448e7af8ea153241004375deff9fd961da" +source = "git+https://github.com/EspressoSystems/jellyfish#30752c6c1af7b933e2536d7aeb991a40af4b806a" dependencies = [ "ark-bls12-377", "ark-bls12-381", @@ -3357,7 +3341,7 @@ dependencies = [ [[package]] name = "jf-utils" version = "0.4.0-pre.0" -source = "git+https://github.com/EspressoSystems/jellyfish#1626a8448e7af8ea153241004375deff9fd961da" +source = "git+https://github.com/EspressoSystems/jellyfish#30752c6c1af7b933e2536d7aeb991a40af4b806a" dependencies = [ "ark-ec", "ark-ff", @@ -3649,14 +3633,13 @@ dependencies = [ [[package]] name = "libp2p-identity" -version = "0.2.4" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f37304f29c82ede408db06aaba60cd2f783a111f46414d3fc4beedac19e0c67b" +checksum = "686e73aff5e23efbb99bc85340ea6fd8686986aa7b283a881ba182cfca535ca9" dependencies = [ "asn1_der", "bs58", "ed25519-dalek", - "hkdf 0.12.3", "libsecp256k1", "log", "multihash", @@ -4379,9 +4362,9 @@ dependencies = [ [[package]] name = "multihash" -version = "0.19.1" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076d548d76a0e2a0d4ab471d0b1c36c577786dfc4471242035d97a12a735c492" +checksum = "2fd59dcc2bbe70baabeac52cd22ae52c55eefe6c38ff11a9439f16a350a939f2" dependencies = [ "core2", "serde", diff --git a/Cargo.toml b/Cargo.toml index 688e9c37d0..10990e7437 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,11 +52,6 @@ either = { version = "1.8" } espresso-systems-common = { git = "https://github.com/espressosystems/espresso-systems-common", tag = "0.4.1" } ethereum-types = { version = "0.14.1", features = ["impl-serde"] } futures = "0.3.28" - -# TODO generic-array should not be a direct dependency -# https://github.com/EspressoSystems/HotShot/issues/1850 -generic-array = { version = "0.14.7", features = ["serde"] } - jf-primitives = { git = "https://github.com/EspressoSystems/jellyfish" } jf-relation = { git = "https://github.com/EspressoSystems/jellyfish" } jf-utils = { git = "https://github.com/espressosystems/jellyfish" } diff --git a/crates/hotshot-qc/Cargo.toml b/crates/hotshot-qc/Cargo.toml index 9c53fb335a..0faa1f0a87 100644 --- a/crates/hotshot-qc/Cargo.toml +++ b/crates/hotshot-qc/Cargo.toml @@ -19,7 +19,7 @@ ark-std = { workspace = true } bincode = { workspace = true } bitvec = { workspace = true } ethereum-types = { workspace = true } -generic-array = { workspace = true } +generic-array = "0.14.7" hotshot-types = { path = "../types" } jf-primitives = { workspace = true } jf-relation = { workspace = true } diff --git a/crates/hotshot-stake-table/Cargo.toml b/crates/hotshot-stake-table/Cargo.toml index 4059589a85..8f11e4d66f 100644 --- a/crates/hotshot-stake-table/Cargo.toml +++ b/crates/hotshot-stake-table/Cargo.toml @@ -16,7 +16,7 @@ bitvec = { workspace = true } digest = { workspace = true } displaydoc = { version = "0.2.3", default-features = false } ethereum-types = { workspace = true } -generic-array = { workspace = true } +generic-array = "0.14.7" hotshot-types = { path = "../types" } jf-primitives = { workspace = true } jf-relation = { workspace = true } diff --git a/crates/hotshot/examples/infra/modDA.rs b/crates/hotshot/examples/infra/modDA.rs index ef188a7849..82cc12bf68 100644 --- a/crates/hotshot/examples/infra/modDA.rs +++ b/crates/hotshot/examples/infra/modDA.rs @@ -47,8 +47,6 @@ use libp2p_networking::{ network::{MeshParams, NetworkNodeConfigBuilder, NetworkNodeType}, reexport::Multiaddr, }; -use rand::rngs::StdRng; -use rand::SeedableRng; use std::{collections::BTreeSet, sync::Arc}; use std::{num::NonZeroUsize, str::FromStr}; // use libp2p::{ @@ -61,9 +59,9 @@ use std::{num::NonZeroUsize, str::FromStr}; // }; use libp2p_identity::PeerId; // use libp2p_networking::network::{MeshParams, NetworkNodeConfigBuilder, NetworkNodeType}; -use std::{fmt::Debug, net::Ipv4Addr}; use std::{ //collections::{BTreeSet, VecDeque}, + collections::VecDeque, //fs, mem, net::IpAddr, @@ -73,6 +71,7 @@ use std::{ //time::{Duration, Instant}, time::Instant, }; +use std::{fmt::Debug, net::Ipv4Addr}; //use surf_disco::error::ClientError; //use surf_disco::Client; use tracing::{debug, error, info, warn}; @@ -126,19 +125,6 @@ pub async fn run_orchestrator_da< .await; } -/// Helper function to calculate the nuymber of transactions to send per node per round -fn calculate_num_tx_per_round( - node_index: u64, - total_num_nodes: usize, - transactions_per_round: usize, -) -> usize { - if node_index == 0 { - transactions_per_round / total_num_nodes + transactions_per_round % total_num_nodes - } else { - transactions_per_round / total_num_nodes - } -} - /// Defines the behavior of a "run" of the network with a given configuration #[async_trait] pub trait RunDA< @@ -268,23 +254,38 @@ pub trait RunDA< } = self.get_config(); let size = mem::size_of::(); - let padding = padding.saturating_sub(size); - let mut txn_rng = StdRng::seed_from_u64(node_index); - - debug!("Adjusted padding size is {:?} bytes", padding); + let adjusted_padding = if padding < size { 0 } else { padding - size }; + let mut txns: VecDeque = VecDeque::new(); + + // TODO ED: In the future we should have each node generate transactions every round to simulate a more realistic network + let tx_to_gen = transactions_per_round * rounds * 3; + { + let mut txn_rng = rand::thread_rng(); + for _ in 0..tx_to_gen { + let txn = + <::StateType as TestableState>::create_random_transaction( + None, + &mut txn_rng, + padding as u64, + ); + txns.push_back(txn); + } + } + debug!("Generated {} transactions", tx_to_gen); - let mut total_transactions_committed = 0; - let mut total_transactions_sent = 0; - let transactions_to_send_per_round = - calculate_num_tx_per_round(node_index, total_nodes.get(), transactions_per_round); + debug!("Adjusted padding size is {:?} bytes", adjusted_padding); + let mut round = 0; + let mut total_transactions = 0; - info!("Starting hotshot!"); let start = Instant::now(); + info!("Starting hotshot!"); let (mut event_stream, _streamid) = context.get_event_stream(FilterEvent::default()).await; let mut anchor_view: TYPES::Time = ::genesis(); let mut num_successful_commits = 0; + let total_nodes_u64 = total_nodes.get() as u64; + context.hotshot.start_consensus().await; loop { @@ -313,20 +314,8 @@ pub trait RunDA< } } - // send transactions - for _ in 0..transactions_to_send_per_round { - let txn = - <::StateType as TestableState>::create_random_transaction( - None, - &mut txn_rng, - padding as u64, - ); - _ = context.submit_transaction(txn).await.unwrap(); - total_transactions_sent += 1; - } - if let Some(size) = block_size { - total_transactions_committed += size; + total_transactions += size; } num_successful_commits += leaf_chain.len(); @@ -345,16 +334,39 @@ pub trait RunDA< EventType::NextLeaderViewTimeout { view_number } => { warn!("Timed out as the next leader in view {:?}", view_number); } - EventType::ViewFinished { view_number: _ } => {} + EventType::ViewFinished { view_number } => { + if *view_number > round { + round = *view_number; + info!("view finished: {:?}", view_number); + for _ in 0..transactions_per_round { + if node_index >= total_nodes_u64 - 10 { + let txn = txns.pop_front().unwrap(); + + debug!("Submitting txn on round {}", round); + + let result = context.submit_transaction(txn).await; + + if result.is_err() { + error! ( + "Could not send transaction to web server on round {}", + round + ) + } + } + } + } + } _ => unimplemented!(), } } } + + round += 1; } // Output run results let total_time_elapsed = start.elapsed(); - error!("[{node_index}]: {rounds} rounds completed in {total_time_elapsed:?} - Total transactions sent: {total_transactions_sent} - Total transactions committed: {total_transactions_committed} - Total commitments: {num_successful_commits}"); + error!("{rounds} rounds completed in {total_time_elapsed:?} - Total transactions committed: {total_transactions} - Total commitments: {num_successful_commits}"); } /// Returns the da network for this run diff --git a/crates/hotshot/src/traits/networking/memory_network.rs b/crates/hotshot/src/traits/networking/memory_network.rs index 976186c030..51f7bbbdf7 100644 --- a/crates/hotshot/src/traits/networking/memory_network.rs +++ b/crates/hotshot/src/traits/networking/memory_network.rs @@ -574,3 +574,359 @@ impl, MEMBERSHIP: Membership; + // type Transaction = VDemoTransaction; + // type ElectionConfigType = StaticElectionConfig; + // type StateType = VDemoState; + // } + // + // type TestMembership = GeneralStaticCommittee; + // type TestNetwork = MemoryCommChannel; + // + // impl NodeImplementation for TestImpl { + // type ConsensusMessage = ValidatingMessage; + // type Exchanges = ValidatingExchanges< + // Test, + // Message, + // QuorumExchange< + // Test, + // TestLeaf, + // TestProposal, + // TestMembership, + // TestNetwork, + // Message, + // >, + // ViewSyncExchange>, + // >; + // type Leaf = TestLeaf; + // type Storage = MemoryStorage; + // + // fn new_channel_maps( + // start_view: ViewNumber, + // ) -> (ChannelMaps, Option>) { + // (ChannelMaps::new(start_view), None) + // } + // } + // + // type TestLeaf = ValidatingLeaf; + // type TestVote = QuorumVote; + // type TestProposal = ValidatingProposal; + // + // /// fake Eq + // /// we can't compare the votetokentype for equality, so we can't + // /// derive EQ on `VoteType` and thereby message + // /// we are only sending data messages, though so we compare key and + // /// data message + // fn fake_message_eq(message_1: Message, message_2: Message) { + // assert_eq!(message_1.sender, message_2.sender); + // if let MessageKind::Data(DataMessage::SubmitTransaction(d_1, _)) = message_1.kind { + // if let MessageKind::Data(DataMessage::SubmitTransaction(d_2, _)) = message_2.kind { + // assert_eq!(d_1, d_2); + // } + // } else { + // panic!("Got unexpected message type in memory test!"); + // } + // } + // + // #[instrument] + // fn get_pubkey() -> Ed25519Pub { + // let priv_key = Ed25519Priv::generate(); + // Ed25519Pub::from_private(&priv_key) + // } + // + // /// create a message + // fn gen_messages(num_messages: u64, seed: u64, pk: Ed25519Pub) -> Vec> { + // let mut messages = Vec::new(); + // for i in 0..num_messages { + // let message = Message { + // sender: pk, + // kind: MessageKind::Data(DataMessage::SubmitTransaction( + // VDemoTransaction { + // add: Addition { + // account: "A".to_string(), + // amount: 50 + i + seed, + // }, + // sub: Subtraction { + // account: "B".to_string(), + // amount: 50 + i + seed, + // }, + // nonce: seed + i, + // padding: vec![50; 0], + // }, + // ::new(0), + // )), + // _phantom: PhantomData, + // }; + // messages.push(message); + // } + // messages + // } + // + // // Spawning a single MemoryNetwork should produce no errors + // #[cfg_attr( + // feature = "tokio-executor", + // tokio::test(flavor = "multi_thread", worker_threads = 2) + // )] + // #[cfg_attr(feature = "async-std-executor", async_std::test)] + // #[instrument] + // async fn spawn_single() { + // setup_logging(); + // let group: Arc, ::SignatureKey>> = + // MasterMap::new(); + // trace!(?group); + // let pub_key = get_pubkey(); + // let _network = MemoryNetwork::new(pub_key, NoMetrics::boxed(), group, Option::None); + // } + // + // // // Spawning a two MemoryNetworks and connecting them should produce no errors + // #[cfg_attr( + // feature = "tokio-executor", + // tokio::test(flavor = "multi_thread", worker_threads = 2) + // )] + // #[cfg_attr(feature = "async-std-executor", async_std::test)] + // #[instrument] + // async fn spawn_double() { + // setup_logging(); + // let group: Arc, ::SignatureKey>> = + // MasterMap::new(); + // trace!(?group); + // let pub_key_1 = get_pubkey(); + // let _network_1 = + // MemoryNetwork::new(pub_key_1, NoMetrics::boxed(), group.clone(), Option::None); + // let pub_key_2 = get_pubkey(); + // let _network_2 = MemoryNetwork::new(pub_key_2, NoMetrics::boxed(), group, Option::None); + // } + // + // // Check to make sure direct queue works + // #[cfg_attr( + // feature = "tokio-executor", + // tokio::test(flavor = "multi_thread", worker_threads = 2) + // )] + // #[cfg_attr(feature = "async-std-executor", async_std::test)] + // #[allow(deprecated)] + // #[instrument] + // async fn direct_queue() { + // setup_logging(); + // // Create some dummy messages + // + // // Make and connect the networking instances + // let group: Arc, ::SignatureKey>> = + // MasterMap::new(); + // trace!(?group); + // let pub_key_1 = get_pubkey(); + // let network1 = + // MemoryNetwork::new(pub_key_1, NoMetrics::boxed(), group.clone(), Option::None); + // let pub_key_2 = get_pubkey(); + // let network2 = MemoryNetwork::new(pub_key_2, NoMetrics::boxed(), group, Option::None); + // + // let first_messages: Vec> = gen_messages(5, 100, pub_key_1); + // + // // Test 1 -> 2 + // // Send messages + // for sent_message in first_messages { + // network1 + // .direct_message(sent_message.clone(), pub_key_2) + // .await + // .expect("Failed to message node"); + // let mut recv_messages = network2 + // .recv_msgs(TransmitType::Direct) + // .await + // .expect("Failed to receive message"); + // let recv_message = recv_messages.pop().unwrap(); + // assert!(recv_messages.is_empty()); + // fake_message_eq(sent_message, recv_message); + // } + // + // let second_messages: Vec> = gen_messages(5, 200, pub_key_2); + // + // // Test 2 -> 1 + // // Send messages + // for sent_message in second_messages { + // network2 + // .direct_message(sent_message.clone(), pub_key_1) + // .await + // .expect("Failed to message node"); + // let mut recv_messages = network1 + // .recv_msgs(TransmitType::Direct) + // .await + // .expect("Failed to receive message"); + // let recv_message = recv_messages.pop().unwrap(); + // assert!(recv_messages.is_empty()); + // fake_message_eq(sent_message, recv_message); + // } + // } + // + // // Check to make sure direct queue works + // #[cfg_attr( + // feature = "tokio-executor", + // tokio::test(flavor = "multi_thread", worker_threads = 2) + // )] + // #[cfg_attr(feature = "async-std-executor", async_std::test)] + // #[allow(deprecated)] + // #[instrument] + // async fn broadcast_queue() { + // setup_logging(); + // // Make and connect the networking instances + // let group: Arc, ::SignatureKey>> = + // MasterMap::new(); + // trace!(?group); + // let pub_key_1 = get_pubkey(); + // let network1 = + // MemoryNetwork::new(pub_key_1, NoMetrics::boxed(), group.clone(), Option::None); + // let pub_key_2 = get_pubkey(); + // let network2 = MemoryNetwork::new(pub_key_2, NoMetrics::boxed(), group, Option::None); + // + // let first_messages: Vec> = gen_messages(5, 100, pub_key_1); + // + // // Test 1 -> 2 + // // Send messages + // for sent_message in first_messages { + // network1 + // .broadcast_message( + // sent_message.clone(), + // vec![pub_key_2].into_iter().collect::>(), + // ) + // .await + // .expect("Failed to message node"); + // let mut recv_messages = network2 + // .recv_msgs(TransmitType::Broadcast) + // .await + // .expect("Failed to receive message"); + // let recv_message = recv_messages.pop().unwrap(); + // assert!(recv_messages.is_empty()); + // fake_message_eq(sent_message, recv_message); + // } + // + // let second_messages: Vec> = gen_messages(5, 200, pub_key_2); + // + // // Test 2 -> 1 + // // Send messages + // for sent_message in second_messages { + // network2 + // .broadcast_message( + // sent_message.clone(), + // vec![pub_key_1].into_iter().collect::>(), + // ) + // .await + // .expect("Failed to message node"); + // let mut recv_messages = network1 + // .recv_msgs(TransmitType::Broadcast) + // .await + // .expect("Failed to receive message"); + // let recv_message = recv_messages.pop().unwrap(); + // assert!(recv_messages.is_empty()); + // fake_message_eq(sent_message, recv_message); + // } + // } + // + // #[cfg_attr( + // feature = "tokio-executor", + // tokio::test(flavor = "multi_thread", worker_threads = 2) + // )] + // #[cfg_attr(feature = "async-std-executor", async_std::test)] + // #[instrument] + // #[allow(deprecated)] + // async fn test_in_flight_message_count() { + // // setup_logging(); + // + // // let group: Arc, ::SignatureKey>> = + // // MasterMap::new(); + // // trace!(?group); + // // let pub_key_1 = get_pubkey(); + // // let network1 = + // // MemoryNetwork::new(pub_key_1, NoMetrics::boxed(), group.clone(), Option::None); + // // let pub_key_2 = get_pubkey(); + // // let network2 = MemoryNetwork::new(pub_key_2, NoMetrics::boxed(), group, Option::None); + // + // // // Create some dummy messages + // // let messages: Vec> = gen_messages(5, 100, pub_key_1); + // + // // // assert_eq!(network1.in_flight_message_count(), Some(0)); + // // // assert_eq!(network2.in_flight_message_count(), Some(0)); + // + // // for (_count, message) in messages.iter().enumerate() { + // // network1 + // // .direct_message(message.clone(), pub_key_2) + // // .await + // // .unwrap(); + // // // network 2 has received `count` broadcast messages and `count + 1` direct messages + // // // assert_eq!(network2.in_flight_message_count(), Some(count + count + 1)); + // + // // // network2.broadcast_message(message.clone()).await.unwrap(); + // // // network 1 has received `count` broadcast messages + // // // assert_eq!(network1.in_flight_message_count(), Some(count + 1)); + // + // // // network 2 has received `count + 1` broadcast messages and `count + 1` direct messages + // // // assert_eq!(network2.in_flight_message_count(), Some((count + 1) * 2)); + // // } + // + // // for _count in (0..messages.len()).rev() { + // // network1.recv_msgs(TransmitType::Broadcast).await.unwrap(); + // // // assert_eq!(network1.in_flight_message_count(), Some(count)); + // + // // network2.recv_msgs(TransmitType::Broadcast).await.unwrap(); + // // network2.recv_msgs(TransmitType::Direct).await.unwrap(); + // // // assert_eq!(network2.in_flight_message_count(), Some(count * 2)); + // // } + // + // // // assert_eq!(network1.in_flight_message_count(), Some(0)); + // // // assert_eq!(network2.in_flight_message_count(), Some(0)); + // } +} diff --git a/crates/orchestrator/default-libp2p-run-config.toml b/crates/orchestrator/default-libp2p-run-config.toml index a353ed06f5..c217d24d1c 100644 --- a/crates/orchestrator/default-libp2p-run-config.toml +++ b/crates/orchestrator/default-libp2p-run-config.toml @@ -1,5 +1,5 @@ rounds = 10 -transactions_per_round = 12 +transactions_per_round = 10 node_index = 0 seed = [ 0, diff --git a/crates/testing/tests/memory_network.rs b/crates/testing/tests/memory_network.rs deleted file mode 100644 index 27f6ae6531..0000000000 --- a/crates/testing/tests/memory_network.rs +++ /dev/null @@ -1,371 +0,0 @@ -use std::collections::BTreeSet; -use std::marker::PhantomData; -use std::sync::Arc; - -use async_compatibility_layer::logging::setup_logging; -use hotshot::demo::SDemoState; -use hotshot::traits::election::static_committee::{ - GeneralStaticCommittee, StaticElectionConfig, StaticVoteToken, -}; -use hotshot::traits::implementations::{ - MasterMap, MemoryCommChannel, MemoryNetwork, MemoryStorage, -}; -use hotshot::traits::NodeImplementation; -use hotshot::types::bn254::{BLSPrivKey, BLSPubKey}; -use hotshot::types::SignatureKey; -use hotshot_types::block_impl::{VIDBlockPayload, VIDTransaction}; -use hotshot_types::certificate::ViewSyncCertificate; -use hotshot_types::data::{DAProposal, QuorumProposal, SequencingLeaf}; -use hotshot_types::message::{Message, SequencingMessage}; -use hotshot_types::traits::election::{CommitteeExchange, QuorumExchange, ViewSyncExchange}; -use hotshot_types::traits::metrics::NoMetrics; -use hotshot_types::traits::network::TestableNetworkingImplementation; -use hotshot_types::traits::network::{ConnectedNetwork, TransmitType}; -use hotshot_types::traits::node_implementation::{ChannelMaps, NodeType, SequencingExchanges}; -use hotshot_types::vote::{DAVote, ViewSyncVote}; -use hotshot_types::{ - data::ViewNumber, - message::{DataMessage, MessageKind}, - traits::state::ConsensusTime, - vote::QuorumVote, -}; -use rand::rngs::StdRng; -use rand::{RngCore, SeedableRng}; -use serde::{Deserialize, Serialize}; -use tracing::instrument; -use tracing::trace; - -#[derive( - Copy, - Clone, - Debug, - Default, - Hash, - PartialEq, - Eq, - PartialOrd, - Ord, - serde::Serialize, - serde::Deserialize, -)] -pub struct Test; - -impl NodeType for Test { - type Time = ViewNumber; - type BlockType = VIDBlockPayload; - type SignatureKey = BLSPubKey; - type VoteTokenType = StaticVoteToken; - type Transaction = VIDTransaction; - type ElectionConfigType = StaticElectionConfig; - type StateType = SDemoState; -} - -#[derive(Clone, Debug, Deserialize, Serialize, Hash, PartialEq, Eq)] -pub struct TestImpl {} - -pub type ThisLeaf = SequencingLeaf; -pub type ThisMembership = GeneralStaticCommittee::SignatureKey>; -pub type DANetwork = MemoryCommChannel; -pub type QuorumNetwork = MemoryCommChannel; -pub type ViewSyncNetwork = MemoryCommChannel; - -pub type ThisDAProposal = DAProposal; -pub type ThisDAVote = DAVote; - -pub type ThisQuorumProposal = QuorumProposal; -pub type ThisQuorumVote = QuorumVote; - -pub type ThisViewSyncProposal = ViewSyncCertificate; -pub type ThisViewSyncVote = ViewSyncVote; - -impl NodeImplementation for TestImpl { - type Storage = MemoryStorage; - type Leaf = SequencingLeaf; - type Exchanges = SequencingExchanges< - Test, - Message, - QuorumExchange< - Test, - Self::Leaf, - ThisQuorumProposal, - ThisMembership, - QuorumNetwork, - Message, - >, - CommitteeExchange>, - ViewSyncExchange< - Test, - ThisViewSyncProposal, - ThisMembership, - ViewSyncNetwork, - Message, - >, - >; - type ConsensusMessage = SequencingMessage; - - fn new_channel_maps( - start_view: ::Time, - ) -> (ChannelMaps, Option>) { - (ChannelMaps::new(start_view), None) - } -} - -/// fake Eq -/// we can't compare the votetokentype for equality, so we can't -/// derive EQ on `VoteType` and thereby message -/// we are only sending data messages, though so we compare key and -/// data message -fn fake_message_eq(message_1: Message, message_2: Message) { - assert_eq!(message_1.sender, message_2.sender); - if let MessageKind::Data(DataMessage::SubmitTransaction(d_1, _)) = message_1.kind { - if let MessageKind::Data(DataMessage::SubmitTransaction(d_2, _)) = message_2.kind { - assert_eq!(d_1, d_2); - } - } else { - panic!("Got unexpected message type in memory test!"); - } -} - -#[instrument] -fn get_pubkey() -> BLSPubKey { - // random 32 bytes - let mut bytes = [0; 32]; - rand::thread_rng().fill_bytes(&mut bytes); - BLSPubKey::from_private(&BLSPrivKey::generate_from_seed(bytes)) -} - -/// create a message -fn gen_messages(num_messages: u64, seed: u64, pk: BLSPubKey) -> Vec> { - let mut messages = Vec::new(); - for _ in 0..num_messages { - // create a random transaction from seed - let mut bytes = [0u8; 8]; - let mut rng = StdRng::seed_from_u64(seed); - rng.fill_bytes(&mut bytes); - - let message = Message { - sender: pk, - kind: MessageKind::Data(DataMessage::SubmitTransaction( - VIDTransaction(bytes.to_vec()), - ::new(0), - )), - _phantom: PhantomData, - }; - messages.push(message); - } - messages -} - -// Spawning a single MemoryNetwork should produce no errors -#[cfg_attr( - async_executor_impl = "tokio", - tokio::test(flavor = "multi_thread", worker_threads = 2) -)] -#[cfg_attr(async_executor_impl = "async-std", async_std::test)] -#[instrument] -async fn memory_network_spawn_single() { - setup_logging(); - let group: Arc, ::SignatureKey>> = - MasterMap::new(); - trace!(?group); - let pub_key = get_pubkey(); - let _network = MemoryNetwork::new(pub_key, NoMetrics::boxed(), group, Option::None); -} - -// // Spawning a two MemoryNetworks and connecting them should produce no errors -#[cfg_attr( - async_executor_impl = "tokio", - tokio::test(flavor = "multi_thread", worker_threads = 2) -)] -#[cfg_attr(async_executor_impl = "async-std", async_std::test)] -#[instrument] -async fn memory_network_spawn_double() { - setup_logging(); - let group: Arc, ::SignatureKey>> = - MasterMap::new(); - trace!(?group); - let pub_key_1 = get_pubkey(); - let _network_1 = MemoryNetwork::new(pub_key_1, NoMetrics::boxed(), group.clone(), Option::None); - let pub_key_2 = get_pubkey(); - let _network_2 = MemoryNetwork::new(pub_key_2, NoMetrics::boxed(), group, Option::None); -} - -// Check to make sure direct queue works -#[cfg_attr( - async_executor_impl = "tokio", - tokio::test(flavor = "multi_thread", worker_threads = 2) -)] -#[cfg_attr(async_executor_impl = "async-std", async_std::test)] -#[instrument] -async fn memory_network_direct_queue() { - setup_logging(); - // Create some dummy messages - - // Make and connect the networking instances - let group: Arc, ::SignatureKey>> = - MasterMap::new(); - trace!(?group); - - let pub_key_1 = get_pubkey(); - let network1 = MemoryNetwork::new(pub_key_1, NoMetrics::boxed(), group.clone(), Option::None); - - let pub_key_2 = get_pubkey(); - let network2 = MemoryNetwork::new(pub_key_2, NoMetrics::boxed(), group, Option::None); - - let first_messages: Vec> = gen_messages(5, 100, pub_key_1); - - // Test 1 -> 2 - // Send messages - for sent_message in first_messages { - network1 - .direct_message(sent_message.clone(), pub_key_2) - .await - .expect("Failed to message node"); - let mut recv_messages = network2 - .recv_msgs(TransmitType::Direct) - .await - .expect("Failed to receive message"); - let recv_message = recv_messages.pop().unwrap(); - assert!(recv_messages.is_empty()); - fake_message_eq(sent_message, recv_message); - } - - let second_messages: Vec> = gen_messages(5, 200, pub_key_2); - - // Test 2 -> 1 - // Send messages - for sent_message in second_messages { - network2 - .direct_message(sent_message.clone(), pub_key_1) - .await - .expect("Failed to message node"); - let mut recv_messages = network1 - .recv_msgs(TransmitType::Direct) - .await - .expect("Failed to receive message"); - let recv_message = recv_messages.pop().unwrap(); - assert!(recv_messages.is_empty()); - fake_message_eq(sent_message, recv_message); - } -} - -// Check to make sure direct queue works -#[cfg_attr( - async_executor_impl = "tokio", - tokio::test(flavor = "multi_thread", worker_threads = 2) -)] -#[cfg_attr(async_executor_impl = "async-std", async_std::test)] -#[instrument] -async fn memory_network_broadcast_queue() { - setup_logging(); - // Make and connect the networking instances - let group: Arc, ::SignatureKey>> = - MasterMap::new(); - trace!(?group); - let pub_key_1 = get_pubkey(); - let network1 = MemoryNetwork::new(pub_key_1, NoMetrics::boxed(), group.clone(), Option::None); - let pub_key_2 = get_pubkey(); - let network2 = MemoryNetwork::new(pub_key_2, NoMetrics::boxed(), group, Option::None); - - let first_messages: Vec> = gen_messages(5, 100, pub_key_1); - - // Test 1 -> 2 - // Send messages - for sent_message in first_messages { - network1 - .broadcast_message( - sent_message.clone(), - vec![pub_key_2].into_iter().collect::>(), - ) - .await - .expect("Failed to message node"); - let mut recv_messages = network2 - .recv_msgs(TransmitType::Broadcast) - .await - .expect("Failed to receive message"); - let recv_message = recv_messages.pop().unwrap(); - assert!(recv_messages.is_empty()); - fake_message_eq(sent_message, recv_message); - } - - let second_messages: Vec> = gen_messages(5, 200, pub_key_2); - - // Test 2 -> 1 - // Send messages - for sent_message in second_messages { - network2 - .broadcast_message( - sent_message.clone(), - vec![pub_key_1].into_iter().collect::>(), - ) - .await - .expect("Failed to message node"); - let mut recv_messages = network1 - .recv_msgs(TransmitType::Broadcast) - .await - .expect("Failed to receive message"); - let recv_message = recv_messages.pop().unwrap(); - assert!(recv_messages.is_empty()); - fake_message_eq(sent_message, recv_message); - } -} - -#[cfg_attr( - async_executor_impl = "tokio", - tokio::test(flavor = "multi_thread", worker_threads = 2) -)] -#[cfg_attr(async_executor_impl = "async-std", async_std::test)] -#[instrument] -#[allow(deprecated)] -async fn memory_network_test_in_flight_message_count() { - setup_logging(); - - let group: Arc, ::SignatureKey>> = - MasterMap::new(); - trace!(?group); - let pub_key_1 = get_pubkey(); - let network1 = MemoryNetwork::new(pub_key_1, NoMetrics::boxed(), group.clone(), Option::None); - let pub_key_2 = get_pubkey(); - let network2 = MemoryNetwork::new(pub_key_2, NoMetrics::boxed(), group, Option::None); - - // Create some dummy messages - let messages: Vec> = gen_messages(5, 100, pub_key_1); - let broadcast_recipients = BTreeSet::from([pub_key_1, pub_key_2]); - - assert_eq!(network1.in_flight_message_count(), Some(0)); - assert_eq!(network2.in_flight_message_count(), Some(0)); - - for (count, message) in messages.iter().enumerate() { - network1 - .direct_message(message.clone(), pub_key_2) - .await - .unwrap(); - // network 2 has received `count` broadcast messages and `count + 1` direct messages - assert_eq!(network2.in_flight_message_count(), Some(count + count + 1)); - - network2 - .broadcast_message(message.clone(), broadcast_recipients.clone()) - .await - .unwrap(); - // network 1 has received `count` broadcast messages - assert_eq!(network1.in_flight_message_count(), Some(count + 1)); - - // network 2 has received `count + 1` broadcast messages and `count + 1` direct messages - assert_eq!(network2.in_flight_message_count(), Some((count + 1) * 2)); - } - - while network1.in_flight_message_count().unwrap() > 0 { - network1.recv_msgs(TransmitType::Broadcast).await.unwrap(); - } - - while network2.in_flight_message_count().unwrap() > messages.len() { - network2.recv_msgs(TransmitType::Direct).await.unwrap(); - } - - while network2.in_flight_message_count().unwrap() > 0 { - network2.recv_msgs(TransmitType::Broadcast).await.unwrap(); - } - - assert_eq!(network1.in_flight_message_count(), Some(0)); - assert_eq!(network2.in_flight_message_count(), Some(0)); -} diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index c0eb7a11e0..dbc1b7b009 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -31,7 +31,7 @@ displaydoc = { version = "0.2.3", default-features = false } either = { workspace = true, features = ["serde"] } espresso-systems-common = { workspace = true } futures = { workspace = true } -generic-array = { workspace = true } +generic-array = "0.14.7" hex_fmt = "0.3.0" hotshot-constants = { path = "../constants" } hotshot-utils = { path = "../utils" } diff --git a/flake.lock b/flake.lock index 99bccee207..c29e31ee27 100644 --- a/flake.lock +++ b/flake.lock @@ -41,11 +41,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1696054802, - "narHash": "sha256-VTON/WlYeyzFoYwwsb8KveqJJCfWEI6NtZYHcAFKBuo=", + "lastModified": 1695449989, + "narHash": "sha256-A4KxWCgaT1BSVY99o+FZ841U/5yf5hKJmztwrBdvboM=", "owner": "nix-community", "repo": "fenix", - "rev": "3116ee073ab3931c78328ca126224833c95e6227", + "rev": "3605f6c31ac62e055c0ab0dbfa251fa96285109d", "type": "github" }, "original": { @@ -72,11 +72,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1695978539, - "narHash": "sha256-lta5HToBZMWZ2hl5CautNSUgIZViR41QxN7JKbMAjgQ=", + "lastModified": 1695318763, + "narHash": "sha256-FHVPDRP2AfvsxAdc+AsgFJevMz5VBmnZglFUMlxBkcY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "bd9b686c0168041aea600222be0805a0de6e6ab8", + "rev": "e12483116b3b51a185a33a272bf351e357ba9a99", "type": "github" }, "original": { @@ -99,11 +99,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1696014495, - "narHash": "sha256-TcDFXRK9weJ1yCQyo6zxRhLfYHe/GcXDbSUpiszNCuw=", + "lastModified": 1695398941, + "narHash": "sha256-BejjVu84wXk8G1dZuu66r8kz567aAymfaiYgXpMRNqU=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "4791a5de21735e3d9414d131a4f973da9bae0537", + "rev": "2b580a1f3c7bf7f2dd6d1462282362b80a6a3d91", "type": "github" }, "original": {