WIP: hotshot 0.5.83

- Make rustc happy.
- Still contains some todo!.
- `cargo check` passes
- `cargo check --tests` fails, needs a fix in query service

Cargo.toml

@@ -168,3 +168,27 @@ opt-level = 3
 opt-level = 3
 [profile.test.package.diff-test-hotshot]
 opt-level = 3
+
+[patch.'https://github.com/EspressoSystems/hotshot.git']
+hotshot = { git = 'https://github.com/EspressoSystems/HotShot//', branch = 'bump/0.5.83' }
+hotshot-builder-api = { git = 'https://github.com/EspressoSystems/hotshot//', branch = 'bump/0.5.83' }
+hotshot-orchestrator ={ git = 'https://github.com/EspressoSystems/hotshot//', branch = 'bump/0.5.83' }
+hotshot-stake-table = { git = 'https://github.com/EspressoSystems/hotshot//', branch = 'bump/0.5.83' }
+hotshot-task = { git = 'https://github.com/EspressoSystems/hotshot//', branch = 'bump/0.5.83' }
+hotshot-task-impls = { git = 'https://github.com/EspressoSystems/hotshot//', branch = 'bump/0.5.83' }
+hotshot-testing = { git = 'https://github.com/EspressoSystems/hotshot//', branch = 'bump/0.5.83' }
+hotshot-types ={ git = 'https://github.com/EspressoSystems/hotshot//', branch = 'bump/0.5.83' }
+libp2p-networking = { git = "https://github.com/EspressoSystems/hotshot//", branch = 'bump/0.5.83'  }
+hotshot-example-types = { git = "https://github.com/EspressoSystems/hotshot//", branch = 'bump/0.5.83' }
+utils = { git = "https://github.com/EspressoSystems/hotshot//", branch = 'bump/0.5.83' }
+
+[patch.'https://github.com/EspressoSystems/marketplace-builder-core.git']
+marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core//", branch = "ma/hotshot-0.5.83" }
+marketplace-builder-shared = { git = "https://github.com/EspressoSystems/marketplace-builder-core//", branch = "ma/hotshot-0.5.83" }
+hotshot-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core//", branch = "ma/hotshot-0.5.83" }
+
+[patch.'https://github.com/EspressoSystems/hotshot-events-service.git']
+hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service//", branch = "ma/hotshot-0.5.83" }
+
+# patch
+# https://github.com/rust-lang/cargo/issues/5478

builder/src/non_permissioned.rs

@@ -145,6 +145,8 @@ impl BuilderConfig {
                 vid_commitment,
                 leaf_commit: fake_commitment(),
                 builder_commitment,
+                tx_count: todo!(),
+                last_nonempty_view: todo!(),
             },
             decide_receiver,
             da_receiver,

marketplace-builder/src/builder.rs

@@ -1,4 +1,4 @@
-use std::{collections::HashSet, num::NonZeroUsize, time::Duration};
+use std::{arch::global_asm, collections::HashSet, num::NonZeroUsize, time::Duration};
 
 use anyhow::Context;
 use async_broadcast::{
@@ -169,15 +169,16 @@ impl BuilderConfig {
         };
 
         // create the global state
-        let global_state: Arc<GlobalState<SeqTypes, DynamicHooks>> = GlobalState::new(
-            (builder_key_pair.fee_account(), builder_key_pair),
-            api_timeout,
-            maximize_txns_count_timeout_duration,
-            Duration::from_secs(60),
-            tx_channel_capacity.get(),
-            base_fee.as_u64().expect("Base fee too high"),
-            hooks,
-        );
+        // let global_state: Arc<GlobalState<SeqTypes, DynamicHooks>> = GlobalState::new(
+        //     (builder_key_pair.fee_account(), builder_key_pair),
+        //     api_timeout,
+        //     maximize_txns_count_timeout_duration,
+        //     Duration::from_secs(60),
+        //     tx_channel_capacity.get(),
+        //     base_fee.as_u64().expect("Base fee too high"),
+        //     hooks,
+        // );
+        let global_state = todo!();
 
         Self::start_service(
             Arc::clone(&global_state),

sequencer/src/persistence/fs.rs

@@ -313,7 +313,7 @@ impl Inner {
 
             let info = LeafInfo {
                 leaf: leaf.into(),
-                vid_share,
+                vid_share: vid_share.map(Into::into),
 
                 // Note: the following fields are not used in Decide event processing, and should be
                 // removed. For now, we just default them.

sequencer/src/persistence/sql.rs

@@ -746,7 +746,7 @@ impl Persistence {
 
                     LeafInfo {
                         leaf: leaf.into(),
-                        vid_share,
+                        vid_share: vid_share.map(Into::into),
                         // Note: the following fields are not used in Decide event processing, and
                         // should be removed. For now, we just default them.
                         state: Default::default(),
@@ -1508,6 +1508,7 @@ mod test {
     use futures::stream::TryStreamExt;
     use hotshot_example_types::node_types::TestVersions;
     use hotshot_types::{
+        data::EpochNumber,
         drb::{INITIAL_DRB_RESULT, INITIAL_DRB_SEED_INPUT},
         simple_certificate::QuorumCertificate,
         traits::{block_contents::vid_commitment, signature_key::SignatureKey, EncodeBytes},
@@ -1540,6 +1541,7 @@ mod test {
                 view_change_evidence: None,
                 drb_seed: INITIAL_DRB_SEED_INPUT,
                 drb_result: INITIAL_DRB_RESULT,
+                next_epoch_justify_qc: None,
             },
             signature,
             _pd: Default::default(),
@@ -1629,6 +1631,7 @@ mod test {
             view_change_evidence: None,
             drb_seed: INITIAL_DRB_SEED_INPUT,
             drb_result: INITIAL_DRB_RESULT,
+            next_epoch_justify_qc: None,
         };
         let quorum_proposal_signature =
             BLSPubKey::sign(&privkey, &bincode::serialize(&quorum_proposal).unwrap())
@@ -1646,6 +1649,7 @@ mod test {
                 encoded_transactions: leaf_payload_bytes_arc,
                 metadata: leaf_payload.ns_table().clone(),
                 view_number: ViewNumber::new(0),
+                epoch: EpochNumber::new(1),
             },
             signature: block_payload_signature,
             _pd: Default::default(),
@@ -1753,6 +1757,7 @@ mod test {
             view_change_evidence: None,
             drb_seed: INITIAL_DRB_SEED_INPUT,
             drb_result: INITIAL_DRB_RESULT,
+            next_epoch_justify_qc: None,
         };
         let quorum_proposal_signature =
             BLSPubKey::sign(&privkey, &bincode::serialize(&quorum_proposal).unwrap())
@@ -1770,6 +1775,7 @@ mod test {
                 encoded_transactions: leaf_payload_bytes_arc.clone(),
                 metadata: leaf_payload.ns_table().clone(),
                 view_number: data_view,
+                epoch: EpochNumber::new(1),
             },
             signature: block_payload_signature,
             _pd: Default::default(),

sequencer/src/proposal_fetcher.rs

@@ -10,7 +10,7 @@ use espresso_types::{parse_duration, v0::traits::SequencerPersistence, PubKey, V
 use futures::stream::StreamExt;
 use hotshot::types::EventType;
 use hotshot_types::{
-    data::{EpochNumber, Leaf2, ViewNumber},
+    data::{Leaf2, ViewNumber},
     traits::{
         metrics::{Counter, Gauge, Metrics},
         network::ConnectedNetwork,
@@ -190,11 +190,7 @@ where
                 }
             }
 
-            let future =
-                self.consensus
-                    .read()
-                    .await
-                    .request_proposal(view, EpochNumber::genesis(), leaf)?;
+            let future = self.consensus.read().await.request_proposal(view, leaf)?;
             let proposal = timeout(self.cfg.fetch_timeout, future)
                 .await
                 .context("timed out fetching proposal")?

types/src/v0/mod.rs

@@ -143,6 +143,8 @@ impl NodeType for SeqTypes {
     type Membership = EpochCommittees;
     type BuilderSignatureKey = FeeAccount;
     type AuctionResult = SolverAuctionResults;
+    // TODO this associated constant will be removed to make it configurable
+    const EPOCH_HEIGHT: u64 = 42;
 }
 
 #[derive(Clone, Default, Debug, Copy)]

types/src/v0/traits.rs

@@ -10,11 +10,14 @@ use hotshot::{types::EventType, HotShotInitializer};
 use hotshot_types::{
     consensus::CommitmentMap,
     data::{
-        DaProposal, EpochNumber, QuorumProposal, QuorumProposal2, VidDisperseShare, ViewNumber,
+        DaProposal, DaProposal2, EpochNumber, QuorumProposal, QuorumProposal2, VidDisperseShare,
+        VidDisperseShare2, ViewNumber,
     },
     event::{HotShotAction, LeafInfo},
     message::{convert_proposal, Proposal},
-    simple_certificate::{QuorumCertificate, QuorumCertificate2, UpgradeCertificate},
+    simple_certificate::{
+        NextEpochQuorumCertificate2, QuorumCertificate, QuorumCertificate2, UpgradeCertificate,
+    },
     traits::{
         node_implementation::{ConsensusTime, Versions},
         storage::Storage,
@@ -493,6 +496,9 @@ pub trait SequencerPersistence: Sized + Send + Sync + Clone + 'static {
                 ViewNumber::genesis()
             }
         };
+
+        // TODO load from storage
+        let next_epoch_high_qc = None;
         let (leaf, high_qc, anchor_view) = match self
             .load_anchor_leaf()
             .await
@@ -581,6 +587,7 @@ pub trait SequencerPersistence: Sized + Send + Sync + Clone + 'static {
                 highest_voted_view,
                 saved_proposals,
                 high_qc,
+                next_epoch_high_qc,
                 upgrade_certificate,
                 undecided_leaves.into_values().collect(),
                 undecided_state,
@@ -726,6 +733,14 @@ impl<P: SequencerPersistence> Storage<SeqTypes> for Arc<P> {
         (**self).append_vid(proposal).await
     }
 
+    async fn append_vid2(
+        &self,
+        proposal: &Proposal<SeqTypes, VidDisperseShare2<SeqTypes>>,
+    ) -> anyhow::Result<()> {
+        let proposal_1 = convert_proposal(proposal.clone());
+        (**self).append_vid(&proposal_1).await
+    }
+
     async fn append_da(
         &self,
         proposal: &Proposal<SeqTypes, DaProposal<SeqTypes>>,
@@ -734,6 +749,15 @@ impl<P: SequencerPersistence> Storage<SeqTypes> for Arc<P> {
         (**self).append_da(proposal, vid_commit).await
     }
 
+    async fn append_da2(
+        &self,
+        proposal: &Proposal<SeqTypes, DaProposal2<SeqTypes>>,
+        vid_commit: <VidSchemeType as VidScheme>::Commit,
+    ) -> anyhow::Result<()> {
+        let proposal_1 = convert_proposal(proposal.clone());
+        (**self).append_da(&proposal_1, vid_commit).await
+    }
+
     async fn record_action(&self, view: ViewNumber, action: HotShotAction) -> anyhow::Result<()> {
         (**self).record_action(view, action).await
     }
@@ -809,6 +833,14 @@ impl<P: SequencerPersistence> Storage<SeqTypes> for Arc<P> {
             .migrate_consensus(migrate_leaf, migrate_proposal)
             .await
     }
+
+    async fn update_next_epoch_high_qc2(
+        &self,
+        _high_qc: NextEpochQuorumCertificate2<SeqTypes>,
+    ) -> anyhow::Result<()> {
+        // TODO
+        Ok(())
+    }
 }
 
 /// Data that can be deserialized from a subslice of namespace payload bytes.