WIP: hotshot 0.5.83

Cargo.toml

@@ -168,3 +168,30 @@ opt-level = 3
 opt-level = 3
 [profile.test.package.diff-test-hotshot]
 opt-level = 3
+
+[patch.'https://github.com/EspressoSystems/hotshot.git']
+hotshot = { git = 'https://github.com/EspressoSystems/HotShot//', branch = 'bump/0.5.83' }
+hotshot-builder-api = { git = 'https://github.com/EspressoSystems/hotshot//', branch = 'bump/0.5.83' }
+hotshot-orchestrator ={ git = 'https://github.com/EspressoSystems/hotshot//', branch = 'bump/0.5.83' }
+hotshot-stake-table = { git = 'https://github.com/EspressoSystems/hotshot//', branch = 'bump/0.5.83' }
+hotshot-task = { git = 'https://github.com/EspressoSystems/hotshot//', branch = 'bump/0.5.83' }
+hotshot-task-impls = { git = 'https://github.com/EspressoSystems/hotshot//', branch = 'bump/0.5.83' }
+hotshot-testing = { git = 'https://github.com/EspressoSystems/hotshot//', branch = 'bump/0.5.83' }
+hotshot-types ={ git = 'https://github.com/EspressoSystems/hotshot//', branch = 'bump/0.5.83' }
+libp2p-networking = { git = "https://github.com/EspressoSystems/hotshot//", branch = 'bump/0.5.83'  }
+hotshot-example-types = { git = "https://github.com/EspressoSystems/hotshot//", branch = 'bump/0.5.83' }
+utils = { git = "https://github.com/EspressoSystems/hotshot//", branch = 'bump/0.5.83' }
+
+[patch.'https://github.com/EspressoSystems/marketplace-builder-core.git']
+marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core//", branch = "ma/hotshot-0.5.83" }
+marketplace-builder-shared = { git = "https://github.com/EspressoSystems/marketplace-builder-core//", branch = "ma/hotshot-0.5.83" }
+hotshot-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core//", branch = "ma/hotshot-0.5.83" }
+
+[patch.'https://github.com/EspressoSystems/hotshot-events-service.git']
+hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service//", branch = "ma/hotshot-0.5.83" }
+
+[patch.'https://github.com/EspressoSystems/hotshot-query-service']
+hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service//", branch = "ma/hotshot-0.5.83" }
+
+# patch
+# https://github.com/rust-lang/cargo/issues/5478

builder/src/non_permissioned.rs

@@ -145,6 +145,8 @@ impl BuilderConfig {
                 vid_commitment,
                 leaf_commit: fake_commitment(),
                 builder_commitment,
+                tx_count: todo!(),
+                last_nonempty_view: todo!(),
             },
             decide_receiver,
             da_receiver,

marketplace-builder/src/builder.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashSet, num::NonZeroUsize, time::Duration};
+use std::{arch::global_asm, collections::HashSet, num::NonZeroUsize, time::Duration};
 
 use anyhow::Context;
 use async_broadcast::{
@@ -169,15 +169,16 @@ impl BuilderConfig {
         };
 
         // create the global state
-        let global_state: Arc<GlobalState<SeqTypes, DynamicHooks>> = GlobalState::new(
-            (builder_key_pair.fee_account(), builder_key_pair),
-            api_timeout,
-            maximize_txns_count_timeout_duration,
-            Duration::from_secs(60),
-            tx_channel_capacity.get(),
-            base_fee.as_u64().expect("Base fee too high"),
-            hooks,
-        );
+        // let global_state: Arc<GlobalState<SeqTypes, DynamicHooks>> = GlobalState::new(
+        //     (builder_key_pair.fee_account(), builder_key_pair),
+        //     api_timeout,
+        //     maximize_txns_count_timeout_duration,
+        //     Duration::from_secs(60),
+        //     tx_channel_capacity.get(),
+        //     base_fee.as_u64().expect("Base fee too high"),
+        //     hooks,
+        // );
+        let global_state = todo!();
 
         Self::start_service(
             Arc::clone(&global_state),

sequencer/src/api.rs

@@ -1061,6 +1061,7 @@ mod api_tests {
     use hotshot_query_service::availability::{
         AvailabilityDataSource, BlockQueryData, VidCommonQueryData,
     };
+    use hotshot_types::data::EpochNumber;
     use hotshot_types::drb::{INITIAL_DRB_RESULT, INITIAL_DRB_SEED_INPUT};
     use hotshot_types::{
         data::{DaProposal, QuorumProposal2, VidDisperseShare},
@@ -1261,6 +1262,7 @@ mod api_tests {
             view_change_evidence: None,
             drb_seed: INITIAL_DRB_SEED_INPUT,
             drb_result: INITIAL_DRB_RESULT,
+            next_epoch_justify_qc: None,
         };
         let mut qc = QuorumCertificate::genesis::<MockSequencerVersions>(
             &ValidatedState::default(),
@@ -1313,6 +1315,7 @@ mod api_tests {
                 encoded_transactions: payload_bytes_arc.clone(),
                 metadata: payload.ns_table().clone(),
                 view_number: leaf.view_number(),
+                epoch: EpochNumber::new(1),
             };
             let da_proposal = Proposal {
                 data: da_proposal_inner,
@@ -1468,6 +1471,7 @@ mod api_tests {
             view_change_evidence: None,
             drb_seed: INITIAL_DRB_SEED_INPUT,
             drb_result: INITIAL_DRB_RESULT,
+            next_epoch_justify_qc: None,
         };
 
         let leaf = Leaf2::from_quorum_proposal(&qp);

sequencer/src/genesis.rs

@@ -60,7 +60,7 @@ pub struct Genesis {
     pub header: GenesisHeader,
     #[serde(rename = "upgrade", with = "upgrade_ser")]
     #[serde(default)]
-    pub upgrades: BTreeMap<Version, Upgrade>,                                                                                                                             
+    pub upgrades: BTreeMap<Version, Upgrade>,
 }
 
 impl Genesis {

sequencer/src/lib.rs

@@ -637,18 +637,19 @@ pub mod testing {
             .parse()
             .expect("Failed to parse builder URL");
 
-        let hooks = NoHooks(PhantomData);
+        // let hooks = NoHooks(PhantomData);
 
         // create the global state
-        let global_state: Arc<GlobalState<SeqTypes, NoHooks<SeqTypes>>> = GlobalState::new(
-            (builder_key_pair.fee_account(), builder_key_pair),
-            Duration::from_secs(60),
-            Duration::from_millis(100),
-            Duration::from_secs(60),
-            BUILDER_CHANNEL_CAPACITY_FOR_TEST,
-            10,
-            hooks,
-        );
+        // let global_state: Arc<GlobalState<SeqTypes, NoHooks<SeqTypes>>> = GlobalState::new(
+        //     (builder_key_pair.fee_account(), builder_key_pair),
+        //     Duration::from_secs(60),
+        //     Duration::from_millis(100),
+        //     Duration::from_secs(60),
+        //     BUILDER_CHANNEL_CAPACITY_FOR_TEST,
+        //     10,
+        //     hooks,
+        // );
+        let global_state: Arc<GlobalState<SeqTypes, NoHooks<SeqTypes>>> = todo!();
 
         // create the proxy global state it will server the builder apis
         let app = Arc::clone(&global_state)

sequencer/src/message_compat_tests.rs

@@ -28,14 +28,10 @@ use hotshot_types::{
         SequencingMessage,
     },
     simple_certificate::{
-        DaCertificate, QuorumCertificate, TimeoutCertificate, UpgradeCertificate,
-        ViewSyncCommitCertificate2, ViewSyncFinalizeCertificate2, ViewSyncPreCommitCertificate2,
-    },
-    simple_vote::{
-        DaData, DaVote, QuorumData, QuorumVote, TimeoutData, TimeoutVote, UpgradeProposalData,
-        UpgradeVote, ViewSyncCommitData, ViewSyncCommitVote, ViewSyncFinalizeData,
-        ViewSyncFinalizeVote, ViewSyncPreCommitData, ViewSyncPreCommitVote,
+        DaCertificate, QuorumCertificate, UpgradeCertificate, ViewSyncCommitCertificate2,
+        ViewSyncFinalizeCertificate2, ViewSyncPreCommitCertificate2,
     },
+    simple_vote::{DaData, DaVote, QuorumData, QuorumVote, UpgradeProposalData, UpgradeVote},
     traits::{
         node_implementation::ConsensusTime, signature_key::SignatureKey, BlockPayload, EncodeBytes,
     },
@@ -53,7 +49,15 @@ use vbs::{
 async fn test_message_compat<Ver: StaticVersionType>(_ver: Ver) {
     use espresso_types::{EpochCommittees, Leaf, Payload, SeqTypes, Transaction};
     use hotshot_example_types::node_types::TestVersions;
-    use hotshot_types::PeerConfig;
+    use hotshot_types::{
+        simple_certificate::TimeoutCertificate2,
+        simple_vote::{
+            TimeoutData2, TimeoutVote2, ViewSyncCommitData2, ViewSyncCommitVote2,
+            ViewSyncFinalizeData2, ViewSyncFinalizeVote2, ViewSyncPreCommitData2,
+            ViewSyncPreCommitVote2,
+        },
+        PeerConfig,
+    };
 
     let (sender, priv_key) = PubKey::generated_from_seed_indexed(Default::default(), 0);
     let signature = PubKey::sign(&priv_key, &[]).unwrap();
@@ -82,20 +86,24 @@ async fn test_message_compat<Ver: StaticVersionType>(_ver: Ver) {
     )
     .await
     .unwrap();
-    let view_sync_pre_commit_data = ViewSyncPreCommitData {
+    let view_sync_pre_commit_data = ViewSyncPreCommitData2 {
         relay: 0,
         round: ViewNumber::genesis(),
+        epoch: EpochNumber::genesis(),
     };
-    let view_sync_commit_data = ViewSyncCommitData {
+    let view_sync_commit_data = ViewSyncCommitData2 {
         relay: 0,
         round: ViewNumber::genesis(),
+        epoch: EpochNumber::genesis(),
     };
-    let view_sync_finalize_data = ViewSyncFinalizeData {
+    let view_sync_finalize_data = ViewSyncFinalizeData2 {
         relay: 0,
         round: ViewNumber::genesis(),
+        epoch: EpochNumber::genesis(),
     };
-    let timeout_data = TimeoutData {
+    let timeout_data = TimeoutData2 {
         view: ViewNumber::genesis(),
+        epoch: EpochNumber::genesis(),
     };
     let da_data = DaData {
         payload_commit: block_header.payload_commitment(),
@@ -118,7 +126,7 @@ async fn test_message_compat<Ver: StaticVersionType>(_ver: Ver) {
                     Default::default(),
                     Default::default(),
                 )),
-                proposal_certificate: Some(ViewChangeEvidence::Timeout(TimeoutCertificate::new(
+                proposal_certificate: Some(ViewChangeEvidence::Timeout(TimeoutCertificate2::new(
                     timeout_data.clone(),
                     timeout_data.commit(),
                     ViewNumber::genesis(),
@@ -136,43 +144,43 @@ async fn test_message_compat<Ver: StaticVersionType>(_ver: Ver) {
             },
             view_number: ViewNumber::genesis(),
         }),
-        GeneralConsensusMessage::ViewSyncPreCommitVote(ViewSyncPreCommitVote {
+        GeneralConsensusMessage::ViewSyncPreCommitVote2(ViewSyncPreCommitVote2 {
             signature: (sender, signature.clone()),
             data: view_sync_pre_commit_data.clone(),
             view_number: ViewNumber::genesis(),
         }),
-        GeneralConsensusMessage::ViewSyncCommitVote(ViewSyncCommitVote {
+        GeneralConsensusMessage::ViewSyncCommitVote2(ViewSyncCommitVote2 {
             signature: (sender, signature.clone()),
             data: view_sync_commit_data.clone(),
             view_number: ViewNumber::genesis(),
         }),
-        GeneralConsensusMessage::ViewSyncFinalizeVote(ViewSyncFinalizeVote {
+        GeneralConsensusMessage::ViewSyncFinalizeVote2(ViewSyncFinalizeVote2 {
             signature: (sender, signature.clone()),
             data: view_sync_finalize_data.clone(),
             view_number: ViewNumber::genesis(),
         }),
-        GeneralConsensusMessage::ViewSyncPreCommitCertificate(ViewSyncPreCommitCertificate2::new(
+        GeneralConsensusMessage::ViewSyncPreCommitCertificate2(ViewSyncPreCommitCertificate2::new(
             view_sync_pre_commit_data.clone(),
             view_sync_pre_commit_data.commit(),
             ViewNumber::genesis(),
             Default::default(),
             Default::default(),
         )),
-        GeneralConsensusMessage::ViewSyncCommitCertificate(ViewSyncCommitCertificate2::new(
+        GeneralConsensusMessage::ViewSyncCommitCertificate2(ViewSyncCommitCertificate2::new(
             view_sync_commit_data.clone(),
             view_sync_commit_data.commit(),
             ViewNumber::genesis(),
             Default::default(),
             Default::default(),
         )),
-        GeneralConsensusMessage::ViewSyncFinalizeCertificate(ViewSyncFinalizeCertificate2::new(
+        GeneralConsensusMessage::ViewSyncFinalizeCertificate2(ViewSyncFinalizeCertificate2::new(
             view_sync_finalize_data.clone(),
             view_sync_finalize_data.commit(),
             ViewNumber::genesis(),
             Default::default(),
             Default::default(),
         )),
-        GeneralConsensusMessage::TimeoutVote(TimeoutVote {
+        GeneralConsensusMessage::TimeoutVote2(TimeoutVote2 {
             signature: (sender, signature.clone()),
             data: timeout_data,
             view_number: ViewNumber::genesis(),
@@ -181,6 +189,7 @@ async fn test_message_compat<Ver: StaticVersionType>(_ver: Ver) {
             data: UpgradeProposal {
                 upgrade_proposal: upgrade_data.clone(),
                 view_number: ViewNumber::genesis(),
+                epoch: EpochNumber::genesis(),
             },
             signature: signature.clone(),
             _pd: Default::default(),
@@ -197,6 +206,7 @@ async fn test_message_compat<Ver: StaticVersionType>(_ver: Ver) {
                 encoded_transactions: payload.encode(),
                 metadata,
                 view_number: ViewNumber::genesis(),
+                epoch: EpochNumber::genesis(),
             },
             signature: signature.clone(),
             _pd: Default::default(),
@@ -219,6 +229,7 @@ async fn test_message_compat<Ver: StaticVersionType>(_ver: Ver) {
                 vid_scheme(1).disperse(payload.encode()).unwrap(),
                 &membership,
                 EpochNumber::genesis(),
+                Some(EpochNumber::new(1)),
             ))
             .remove(0),
             signature: signature.clone(),

sequencer/src/persistence.rs

@@ -55,7 +55,7 @@ mod persistence_tests {
     use hotshot::types::{BLSPubKey, SignatureKey};
     use hotshot_example_types::node_types::TestVersions;
     use hotshot_types::{
-        data::{DaProposal, QuorumProposal2, VidDisperseShare, ViewNumber},
+        data::{DaProposal, EpochNumber, QuorumProposal2, VidDisperseShare, ViewNumber},
         drb::{INITIAL_DRB_RESULT, INITIAL_DRB_SEED_INPUT},
         event::{EventType, HotShotAction, LeafInfo},
         message::Proposal,
@@ -197,6 +197,7 @@ mod persistence_tests {
                 view_change_evidence: None,
                 drb_seed: INITIAL_DRB_SEED_INPUT,
                 drb_result: INITIAL_DRB_RESULT,
+                next_epoch_justify_qc: None,
             },
             signature,
             _pd: Default::default(),
@@ -248,6 +249,7 @@ mod persistence_tests {
             encoded_transactions: leaf_payload_bytes_arc.clone(),
             metadata: leaf_payload.ns_table().clone(),
             view_number: ViewNumber::new(0),
+            epoch: EpochNumber::new(1),
         };
 
         let da_proposal = Proposal {
@@ -610,6 +612,7 @@ mod persistence_tests {
             view_change_evidence: None,
             drb_seed: INITIAL_DRB_SEED_INPUT,
             drb_result: INITIAL_DRB_RESULT,
+            next_epoch_justify_qc: None,
         };
         let mut qc = QuorumCertificate::genesis::<TestVersions>(
             &ValidatedState::default(),
@@ -625,6 +628,7 @@ mod persistence_tests {
                 encoded_transactions: leaf_payload_bytes_arc.clone(),
                 metadata: leaf_payload.ns_table().clone(),
                 view_number: ViewNumber::new(0),
+                epoch: EpochNumber::new(1),
             },
             signature: block_payload_signature,
             _pd: Default::default(),
@@ -796,6 +800,7 @@ mod persistence_tests {
             view_change_evidence: None,
             drb_seed: INITIAL_DRB_SEED_INPUT,
             drb_result: INITIAL_DRB_RESULT,
+            next_epoch_justify_qc: None,
         };
         let quorum_proposal_signature =
             BLSPubKey::sign(&privkey, &bincode::serialize(&quorum_proposal).unwrap())
@@ -813,6 +818,7 @@ mod persistence_tests {
                 encoded_transactions: leaf_payload_bytes_arc,
                 metadata: leaf_payload.ns_table().clone(),
                 view_number: ViewNumber::new(0),
+                epoch: EpochNumber::new(1),
             },
             signature: block_payload_signature,
             _pd: Default::default(),

sequencer/src/persistence/fs.rs

@@ -313,7 +313,7 @@ impl Inner {
 
             let info = LeafInfo {
                 leaf: leaf.into(),
-                vid_share,
+                vid_share: vid_share.map(Into::into),
 
                 // Note: the following fields are not used in Decide event processing, and should be
                 // removed. For now, we just default them.