Merge remote-tracking branch 'origin/main' into rm/remove-orchestrato…

…r-part-2

Cargo.toml

@@ -64,7 +64,7 @@ jf-utils = { version = "0.4.4", git = "https://github.com/espressosystems/jellyf
 lazy_static = "1"
 libp2p-identity = "0.2"
 libp2p-networking = { path = "./crates/libp2p-networking", version = "0.5", default-features = false }
-libp2p-swarm-derive = { version = "0.34" }
+libp2p-swarm-derive = { version = "0.35" }
 lru = "0.12"
 multiaddr = { version = "0.18" }
 portpicker = "0.1"
@@ -91,7 +91,7 @@ vec1 = { version = "1", features = ["serde"] }
 reqwest = { version = "0.12", features = ["json"] }
 parking_lot = "0.12"
 
-libp2p = { package = "libp2p", version = "0.53", default-features = false, features = [
+libp2p = { package = "libp2p", version = "0.54", default-features = false, features = [
     "macros",
     "autonat",
     "cbor",

audits/README.md

@@ -5,4 +5,4 @@ Internal audits
 
 | Scope & Delivery date       | Report                                                        |
 |-----------------------------|---------------------------------------------------------------|
-| HotShot - July 29, 2024   | [Report](./internal-reviews/EspressoHotShot-2024internal.pdf) |
+| HotShot - July 29, 2024   | [Report](./internal-reviews/EspressoHotshot-2024internal.pdf) |

crates/builder-api/src/v0_1/builder.rs

@@ -262,8 +262,7 @@ where
                     .body_auto::<<Types as NodeType>::Transaction, Ver>(Ver::instance())
                     .map_err(Error::TxnUnpack)?;
                 let hash = tx.commit();
-                state.txn_status(hash).await.map_err(Error::TxnStat)?;
-                Ok(hash)
+                state.txn_status(hash).await.map_err(Error::TxnStat)
             }
             .boxed()
         })?;

crates/example-types/src/storage_types.rs

@@ -14,7 +14,9 @@ use async_lock::RwLock;
 use async_trait::async_trait;
 use hotshot_types::{
     consensus::CommitmentMap,
-    data::{DaProposal, Leaf, Leaf2, QuorumProposal, QuorumProposal2, VidDisperseShare},
+    data::{
+        DaProposal, DaProposal2, Leaf, Leaf2, QuorumProposal, QuorumProposal2, VidDisperseShare,
+    },
     event::HotShotAction,
     message::Proposal,
     simple_certificate::{QuorumCertificate2, UpgradeCertificate},
@@ -38,6 +40,7 @@ type VidShares<TYPES> = HashMap<
 pub struct TestStorageState<TYPES: NodeType> {
     vids: VidShares<TYPES>,
     das: HashMap<TYPES::View, Proposal<TYPES, DaProposal<TYPES>>>,
+    da2s: HashMap<TYPES::View, Proposal<TYPES, DaProposal2<TYPES>>>,
     proposals: BTreeMap<TYPES::View, Proposal<TYPES, QuorumProposal<TYPES>>>,
     proposals2: BTreeMap<TYPES::View, Proposal<TYPES, QuorumProposal2<TYPES>>>,
     high_qc: Option<hotshot_types::simple_certificate::QuorumCertificate<TYPES>>,
@@ -51,6 +54,7 @@ impl<TYPES: NodeType> Default for TestStorageState<TYPES> {
         Self {
             vids: HashMap::new(),
             das: HashMap::new(),
+            da2s: HashMap::new(),
             proposals: BTreeMap::new(),
             proposals2: BTreeMap::new(),
             high_qc: None,
@@ -142,6 +146,21 @@ impl<TYPES: NodeType> Storage<TYPES> for TestStorage<TYPES> {
             .insert(proposal.data.view_number, proposal.clone());
         Ok(())
     }
+    async fn append_da2(
+        &self,
+        proposal: &Proposal<TYPES, DaProposal2<TYPES>>,
+        _vid_commit: <VidSchemeType as VidScheme>::Commit,
+    ) -> Result<()> {
+        if self.should_return_err {
+            bail!("Failed to append VID proposal to storage");
+        }
+        Self::run_delay_settings_from_config(&self.delay_config).await;
+        let mut inner = self.inner.write().await;
+        inner
+            .da2s
+            .insert(proposal.data.view_number, proposal.clone());
+        Ok(())
+    }
     async fn append_proposal(
         &self,
         proposal: &Proposal<TYPES, QuorumProposal<TYPES>>,

crates/libp2p-networking/src/network/node.rs

@@ -217,7 +217,7 @@ impl<T: NodeType> NetworkNode<T> {
             let identify = IdentifyBehaviour::new(identify_cfg);
 
             // Configure the Kademlia behaviour
-            let mut kconfig = Config::default();
+            let mut kconfig = Config::new(StreamProtocol::new("/ipfs/kad/1.0.0"));
             kconfig
                 .set_parallelism(NonZeroUsize::new(5).unwrap())
                 .set_provider_publication_interval(config.kademlia_config.publication_interval)
@@ -584,6 +584,7 @@ impl<T: NodeType> NetworkNode<T> {
                                     agent_version: _,
                                     observed_addr: _,
                                 },
+                            connection_id: _,
                         } = *e
                         {
                             let behaviour = self.swarm.behaviour_mut();

crates/libp2p-networking/src/network/transport.rs

@@ -14,7 +14,11 @@ use hotshot_types::traits::{
     signature_key::SignatureKey,
 };
 use libp2p::{
-    core::{muxing::StreamMuxerExt, transport::TransportEvent, StreamMuxer},
+    core::{
+        muxing::StreamMuxerExt,
+        transport::{DialOpts, TransportEvent},
+        StreamMuxer,
+    },
     identity::PeerId,
     Transport,
 };
@@ -314,9 +318,10 @@ where
     fn dial(
         &mut self,
         addr: libp2p::Multiaddr,
+        opts: DialOpts,
     ) -> Result<Self::Dial, libp2p::TransportError<Self::Error>> {
         // Perform the inner dial
-        let res = self.inner.dial(addr);
+        let res = self.inner.dial(addr, opts);
 
         // Clone the necessary fields
         let auth_message = Arc::clone(&self.auth_message);
@@ -329,27 +334,6 @@ where
         }
     }
 
-    /// Dial a remote peer as a listener. This function is changed to perform an authentication
-    /// handshake on top. The flow should be the reverse of the `dial` function and the
-    /// same as the `poll` function.
-    fn dial_as_listener(
-        &mut self,
-        addr: libp2p::Multiaddr,
-    ) -> Result<Self::Dial, libp2p::TransportError<Self::Error>> {
-        // Perform the inner dial
-        let res = self.inner.dial(addr);
-
-        // Clone the necessary fields
-        let auth_message = Arc::clone(&self.auth_message);
-        let stake_table = Arc::clone(&self.stake_table);
-
-        // If the dial was successful, perform the authentication handshake on top
-        match res {
-            Ok(dial) => Ok(Self::gen_handshake(dial, false, stake_table, auth_message)),
-            Err(err) => Err(err),
-        }
-    }
-
     /// This function is where we perform the authentication handshake for _incoming_ connections.
     /// The flow in this case is the reverse of the `dial` function: we first verify the remote peer's
     /// authentication, and then authenticate with them.
@@ -420,13 +404,6 @@ where
     fn remove_listener(&mut self, id: libp2p::core::transport::ListenerId) -> bool {
         self.inner.remove_listener(id)
     }
-    fn address_translation(
-        &self,
-        listen: &libp2p::Multiaddr,
-        observed: &libp2p::Multiaddr,
-    ) -> Option<libp2p::Multiaddr> {
-        self.inner.address_translation(listen, observed)
-    }
     fn listen_on(
         &mut self,
         id: libp2p::core::transport::ListenerId,

crates/macros/src/lib.rs

@@ -27,7 +27,7 @@ struct TypePathBracketedArray {
 /// description of a crosstest
 #[derive(derive_builder::Builder, Debug, Clone)]
 struct CrossTestData {
-    /// imlementations
+    /// implementations
     impls: ExprArray,
 
     /// builder impl

crates/task-impls/src/da.rs

@@ -12,11 +12,11 @@ use async_trait::async_trait;
 use hotshot_task::task::TaskState;
 use hotshot_types::{
     consensus::{Consensus, OuterConsensus},
-    data::{DaProposal, PackedBundle},
+    data::{DaProposal2, PackedBundle},
     event::{Event, EventType},
     message::{Proposal, UpgradeLock},
-    simple_certificate::DaCertificate,
-    simple_vote::{DaData, DaVote},
+    simple_certificate::DaCertificate2,
+    simple_vote::{DaData2, DaVote2},
     traits::{
         block_contents::vid_commitment,
         election::Membership,
@@ -61,7 +61,7 @@ pub struct DaTaskState<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Version
     pub network: Arc<I::Network>,
 
     /// A map of `DaVote` collector tasks.
-    pub vote_collectors: VoteCollectorsMap<TYPES, DaVote<TYPES>, DaCertificate<TYPES>, V>,
+    pub vote_collectors: VoteCollectorsMap<TYPES, DaVote2<TYPES>, DaCertificate2<TYPES>, V>,
 
     /// This Nodes public key
     pub public_key: TYPES::SignatureKey,
@@ -140,8 +140,11 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> DaTaskState<TYP
             }
             HotShotEvent::DaProposalValidated(proposal, sender) => {
                 let cur_view = self.consensus.read().await.cur_view();
+                let view_number = proposal.data.view_number();
+                let epoch_number = proposal.data.epoch_number;
+
                 ensure!(
-                  cur_view <= proposal.data.view_number() + 1,
+                  cur_view <= view_number + 1,
                   debug!(
                     "Validated DA proposal for prior view but it's too old now Current view {:?}, DA Proposal view {:?}", 
                     cur_view,
@@ -152,7 +155,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> DaTaskState<TYP
                 // Proposal is fresh and valid, notify the application layer
                 broadcast_event(
                     Event {
-                        view_number: self.cur_view,
+                        view_number,
                         event: EventType::DaProposal {
                             proposal: proposal.clone(),
                             sender: sender.clone(),
@@ -163,31 +166,31 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> DaTaskState<TYP
                 .await;
 
                 ensure!(
-                    self.membership
-                        .has_da_stake(&self.public_key, self.cur_epoch),
+                    self.membership.has_da_stake(&self.public_key, epoch_number),
                     debug!(
-                        "We were not chosen for consensus committee on {:?}",
-                        self.cur_view
+                        "We were not chosen for consensus committee for view {:?} in epoch {:?}",
+                        view_number, epoch_number
                     )
                 );
 
                 let txns = Arc::clone(&proposal.data.encoded_transactions);
-                let num_nodes = self.membership.total_nodes(self.cur_epoch);
+                let num_nodes = self.membership.total_nodes(epoch_number);
                 let payload_commitment =
                     spawn_blocking(move || vid_commitment(&txns, num_nodes)).await;
                 let payload_commitment = payload_commitment.unwrap();
+
                 self.storage
                     .write()
                     .await
-                    .append_da(proposal, payload_commitment)
+                    .append_da2(proposal, payload_commitment)
                     .await
                     .wrap()
                     .context(error!("Failed to append DA proposal to storage"))?;
-                let view_number = proposal.data.view_number();
                 // Generate and send vote
-                let vote = DaVote::create_signed_vote(
-                    DaData {
+                let vote = DaVote2::create_signed_vote(
+                    DaData2 {
                         payload_commit: payload_commitment,
+                        epoch: epoch_number,
                     },
                     view_number,
                     &self.public_key,
@@ -222,14 +225,13 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> DaTaskState<TYP
                     let pk = self.private_key.clone();
                     let public_key = self.public_key.clone();
                     let chan = event_stream.clone();
-                    let current_epoch = self.cur_epoch;
                     spawn(async move {
                         Consensus::calculate_and_update_vid(
                             OuterConsensus::new(Arc::clone(&consensus.inner_consensus)),
                             view_number,
                             membership,
                             &pk,
-                            current_epoch,
+                            epoch_number,
                         )
                         .await;
                         if let Some(Some(vid_share)) = consensus
@@ -299,6 +301,7 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> DaTaskState<TYP
                     encoded_transactions,
                     metadata,
                     view_number,
+                    epoch_number,
                     ..
                 } = packed_bundle;
                 let view_number = *view_number;
@@ -311,11 +314,12 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, V: Versions> DaTaskState<TYP
                     TYPES::SignatureKey::sign(&self.private_key, &encoded_transactions_hash)
                         .wrap()?;
 
-                let data: DaProposal<TYPES> = DaProposal {
+                let data: DaProposal2<TYPES> = DaProposal2 {
                     encoded_transactions: Arc::clone(encoded_transactions),
                     metadata: metadata.clone(),
                     // Upon entering a new view we want to send a DA Proposal for the next view -> Is it always the case that this is cur_view + 1?
                     view_number,
+                    epoch_number: *epoch_number,
                 };
 
                 let message = Proposal {