Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/hotshot/0.5.82' into ma/weekly-u…
Browse files Browse the repository at this point in the history 
…pdate-plus-move-marketplace-version
  • Loading branch information
@tbro
tbro committed Dec 4, 2024
2 parents 7c63882 + b7e7e65 commit 96f2271
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 33 deletions.
40 changes: 39 additions & 1 deletion sequencer/src/persistence.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@
//! persistence which is _required_ to run a node.
use async_trait::async_trait;
use espresso_types::v0_99::ChainConfig;
use committable::Committable;
use espresso_types::{v0_99::ChainConfig, Leaf, Leaf2};
use hotshot_types::{consensus::CommitmentMap, data::QuorumProposal};

pub mod fs;
pub mod no_storage;
Expand All @@ -20,6 +22,42 @@ pub trait ChainConfigPersistence: Sized + Send + Sync {
async fn insert_chain_config(&mut self, chain_config: ChainConfig) -> anyhow::Result<()>;
}

fn downgrade_leaf(leaf2: Leaf2) -> Leaf {
if leaf2.drb_seed != [0; 96] && leaf2.drb_result != [0; 32] {
tracing::error!("Losing DRB information!");
}
let quorum_proposal = QuorumProposal {
block_header: leaf2.block_header().clone(),
view_number: leaf2.view_number(),
justify_qc: leaf2.justify_qc().to_qc(),
upgrade_certificate: leaf2.upgrade_certificate(),
proposal_certificate: None,
};
let mut leaf = Leaf::from_quorum_proposal(&quorum_proposal);
if let Some(payload) = leaf2.block_payload() {
leaf.fill_block_payload_unchecked(payload);
}
leaf
}

fn upgrade_commitment_map(map: CommitmentMap<Leaf>) -> CommitmentMap<Leaf2> {
map.into_values()
.map(|leaf| {
let leaf2: Leaf2 = leaf.into();
(leaf2.commit(), leaf2)
})
.collect()
}

fn downgrade_commitment_map(map: CommitmentMap<Leaf2>) -> CommitmentMap<Leaf> {
map.into_values()
.map(|leaf2| {
let leaf = downgrade_leaf(leaf2);
(<Leaf as Committable>::commit(&leaf), leaf)
})
.collect()
}

#[cfg(any(test, feature = "testing"))]
mod testing {

Expand Down
47 changes: 31 additions & 16 deletions sequencer/src/persistence/fs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ use hotshot_types::{
consensus::CommitmentMap,
data::{DaProposal, QuorumProposal, QuorumProposal2, VidDisperseShare},
event::{Event, EventType, HotShotAction, LeafInfo},
message::Proposal,
simple_certificate::{QuorumCertificate2, UpgradeCertificate},
message::{convert_proposal, Proposal},
simple_certificate::{QuorumCertificate, QuorumCertificate2, UpgradeCertificate},
traits::{block_contents::BlockPayload, node_implementation::ConsensusTime},
utils::View,
vid::VidSchemeType,
Expand All @@ -28,6 +28,8 @@ use std::{

use crate::ViewNumber;

use super::{downgrade_commitment_map, downgrade_leaf, upgrade_commitment_map};

/// Options for file system backed persistence.
#[derive(Parser, Clone, Debug)]
pub struct Options {
Expand Down Expand Up @@ -238,7 +240,7 @@ impl Inner {
let bytes =
fs::read(&path).context(format!("reading decided leaf {}", path.display()))?;
let (mut leaf, qc) =
bincode::deserialize::<(Leaf2, QuorumCertificate2<SeqTypes>)>(&bytes)
bincode::deserialize::<(Leaf, QuorumCertificate<SeqTypes>)>(&bytes)
.context(format!("parsing decided leaf {}", path.display()))?;

// Include the VID share if available.
Expand All @@ -261,7 +263,7 @@ impl Inner {
}

let info = LeafInfo {
leaf,
leaf: leaf.into(),
vid_share,

// Note: the following fields are not used in Decide event processing, and should be
Expand Down Expand Up @@ -289,7 +291,7 @@ impl Inner {
.handle_event(&Event {
view_number: ViewNumber::new(view),
event: EventType::Decide {
qc: Arc::new(qc),
qc: Arc::new(qc.to_qc2()),
leaf_chain: Arc::new(vec![leaf]),
block_size: None,
},
Expand Down Expand Up @@ -349,14 +351,18 @@ impl Inner {
let bytes =
fs::read(&file).context(format!("reading decided leaf {}", file.display()))?;
let (leaf, qc) =
bincode::deserialize::<(Leaf2, QuorumCertificate2<SeqTypes>)>(&bytes)
bincode::deserialize::<(Leaf, QuorumCertificate<SeqTypes>)>(&bytes)
.context(format!("parsing decided leaf {}", file.display()))?;
if let Some((anchor_leaf, _)) = &anchor {
if leaf.view_number() > anchor_leaf.view_number() {
anchor = Some((leaf, qc));
let leaf2 = leaf.into();
let qc2 = qc.to_qc2();
anchor = Some((leaf2, qc2));
}
} else {
anchor = Some((leaf, qc));
let leaf2 = leaf.into();
let qc2 = qc.to_qc2();
anchor = Some((leaf2, qc2));
}
}

Expand Down Expand Up @@ -450,7 +456,7 @@ impl SequencerPersistence for Persistence {
fs::remove_file(&legacy_path).context("removing legacy anchor leaf file")?;
}

for (info, qc) in leaf_chain {
for (info, qc2) in leaf_chain {
let view = info.leaf.view_number().u64();
let file_path = path.join(view.to_string()).with_extension("txt");
inner.replace(
Expand All @@ -462,7 +468,9 @@ impl SequencerPersistence for Persistence {
Ok(false)
},
|mut file| {
let bytes = bincode::serialize(&(&info.leaf, qc))?;
let leaf = downgrade_leaf(info.leaf.clone());
let qc = qc2.to_qc();
let bytes = bincode::serialize(&(&leaf, qc))?;
file.write_all(&bytes)?;
Ok(())
},
Expand Down Expand Up @@ -503,7 +511,9 @@ impl SequencerPersistence for Persistence {
return Ok(None);
}
let bytes = fs::read(&path).context("read")?;
Ok(Some(bincode::deserialize(&bytes).context("deserialize")?))
let value: (CommitmentMap<Leaf>, _) =
bincode::deserialize(&bytes).context("deserialize")?;
Ok(Some((upgrade_commitment_map(value.0), value.1)))
}

async fn load_da_proposal(
Expand Down Expand Up @@ -604,6 +614,8 @@ impl SequencerPersistence for Persistence {
leaves: CommitmentMap<Leaf2>,
state: BTreeMap<ViewNumber, View<SeqTypes>>,
) -> anyhow::Result<()> {
let leaves = downgrade_commitment_map(leaves);

if !self.store_undecided_state {
return Ok(());
}
Expand All @@ -628,6 +640,8 @@ impl SequencerPersistence for Persistence {
&self,
proposal: &Proposal<SeqTypes, QuorumProposal2<SeqTypes>>,
) -> anyhow::Result<()> {
let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> =
convert_proposal(proposal.clone());
let mut inner = self.inner.write().await;
let view_number = proposal.data.view_number().u64();
let dir_path = inner.quorum_proposals_dir_path();
Expand Down Expand Up @@ -692,11 +706,12 @@ impl SequencerPersistence for Persistence {
let proposal_bytes = fs::read(file)?;

// Then, deserialize.
let proposal: Proposal<SeqTypes, QuorumProposal2<SeqTypes>> =
let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> =
bincode::deserialize(&proposal_bytes)?;
let proposal2 = convert_proposal(proposal);

// Push to the map and we're done.
map.insert(view_number, proposal);
map.insert(view_number, proposal2);
}
}

Expand All @@ -711,8 +726,9 @@ impl SequencerPersistence for Persistence {
let dir_path = inner.quorum_proposals_dir_path();
let file_path = dir_path.join(view.to_string()).with_extension("txt");
let bytes = fs::read(file_path)?;
let proposal = bincode::deserialize(&bytes)?;
Ok(proposal)
let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> = bincode::deserialize(&bytes)?;
let proposal2 = convert_proposal(proposal);
Ok(proposal2)
}

async fn load_upgrade_certificate(
Expand Down Expand Up @@ -761,7 +777,6 @@ impl SequencerPersistence for Persistence {
Proposal<SeqTypes, QuorumProposal<SeqTypes>>,
) -> Proposal<SeqTypes, QuorumProposal2<SeqTypes>>,
) -> anyhow::Result<()> {
// TODO:
Ok(())
}
}
Expand Down
43 changes: 27 additions & 16 deletions sequencer/src/persistence/sql.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ use hotshot_types::{
consensus::CommitmentMap,
data::{DaProposal, QuorumProposal, QuorumProposal2, VidDisperseShare},
event::{Event, EventType, HotShotAction, LeafInfo},
message::Proposal,
simple_certificate::{QuorumCertificate2, UpgradeCertificate},
message::{convert_proposal, Proposal},
simple_certificate::{QuorumCertificate, QuorumCertificate2, UpgradeCertificate},
traits::{node_implementation::ConsensusTime, BlockPayload},
utils::View,
vid::VidSchemeType,
Expand All @@ -36,6 +36,8 @@ use std::{collections::BTreeMap, path::PathBuf, str::FromStr, sync::Arc, time::D

use crate::{catchup::SqlStateCatchup, SeqTypes, ViewNumber};

use super::{downgrade_commitment_map, downgrade_leaf, upgrade_commitment_map};

/// Options for Postgres-backed persistence.
#[derive(Parser, Clone, Derivative, Default)]
#[derivative(Debug)]
Expand Down Expand Up @@ -486,9 +488,9 @@ impl Persistence {
if hash.is_none() {
let view: i64 = row.try_get("view")?;
let data: Vec<u8> = row.try_get("data")?;
let proposal: Proposal<SeqTypes, QuorumProposal2<SeqTypes>> =
let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> =
bincode::deserialize(&data)?;
let leaf = Leaf2::from_quorum_proposal(&proposal.data);
let leaf = Leaf::from_quorum_proposal(&proposal.data);
let leaf_hash = Committable::commit(&leaf);
tracing::info!(view, %leaf_hash, "populating quorum proposal leaf hash");
updates.push((view, leaf_hash.to_string()));
Expand Down Expand Up @@ -548,14 +550,16 @@ impl SequencerPersistence for Persistence {
) -> anyhow::Result<()> {
let values = leaf_chain
.into_iter()
.map(|(info, qc)| {
.map(|(info, qc2)| {
// The leaf may come with a large payload attached. We don't care about this payload
// because we already store it separately, as part of the DA proposal. Storing it
// here contributes to load on the DB for no reason, so we remove it before
// serializing the leaf.
let mut leaf = info.leaf.clone();
let mut leaf = downgrade_leaf(info.leaf.clone());
leaf.unfill_block_payload();

let qc = qc2.to_qc();

let view = qc.view_number.u64() as i64;
let leaf_bytes = bincode::serialize(&leaf)?;
let qc_bytes = bincode::serialize(&qc)?;
Expand Down Expand Up @@ -612,12 +616,14 @@ impl SequencerPersistence for Persistence {
};

let leaf_bytes: Vec<u8> = row.get("leaf");
let leaf = bincode::deserialize(&leaf_bytes)?;
let leaf: Leaf = bincode::deserialize(&leaf_bytes)?;
let leaf2: Leaf2 = leaf.into();

let qc_bytes: Vec<u8> = row.get("qc");
let qc = bincode::deserialize(&qc_bytes)?;
let qc: QuorumCertificate<SeqTypes> = bincode::deserialize(&qc_bytes)?;
let qc2 = qc.to_qc2();

Ok(Some((leaf, qc)))
Ok(Some((leaf2, qc2)))
}

async fn load_anchor_view(&self) -> anyhow::Result<ViewNumber> {
Expand All @@ -642,12 +648,13 @@ impl SequencerPersistence for Persistence {
};

let leaves_bytes: Vec<u8> = row.get("leaves");
let leaves = bincode::deserialize(&leaves_bytes)?;
let leaves: CommitmentMap<Leaf> = bincode::deserialize(&leaves_bytes)?;
let leaves2 = upgrade_commitment_map(leaves);

let state_bytes: Vec<u8> = row.get("state");
let state = bincode::deserialize(&state_bytes)?;

Ok(Some((leaves, state)))
Ok(Some((leaves2, state)))
}

async fn load_da_proposal(
Expand Down Expand Up @@ -708,9 +715,9 @@ impl SequencerPersistence for Persistence {
let view: i64 = row.get("view");
let view_number: ViewNumber = ViewNumber::new(view.try_into()?);
let bytes: Vec<u8> = row.get("data");
let proposal: Proposal<SeqTypes, QuorumProposal2<SeqTypes>> =
let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> =
bincode::deserialize(&bytes)?;
Ok((view_number, proposal))
Ok((view_number, convert_proposal(proposal)))
})
.collect::<anyhow::Result<Vec<_>>>()?,
))
Expand All @@ -726,7 +733,8 @@ impl SequencerPersistence for Persistence {
.bind(view.u64() as i64)
.fetch_one(tx.as_mut())
.await?;
let proposal = bincode::deserialize(&data)?;
let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> = bincode::deserialize(&data)?;
let proposal = convert_proposal(proposal);
Ok(proposal)
}

Expand Down Expand Up @@ -787,6 +795,8 @@ impl SequencerPersistence for Persistence {
leaves: CommitmentMap<Leaf2>,
state: BTreeMap<ViewNumber, View<SeqTypes>>,
) -> anyhow::Result<()> {
let leaves = downgrade_commitment_map(leaves);

if !self.store_undecided_state {
return Ok(());
}
Expand All @@ -808,9 +818,11 @@ impl SequencerPersistence for Persistence {
&self,
proposal: &Proposal<SeqTypes, QuorumProposal2<SeqTypes>>,
) -> anyhow::Result<()> {
let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> =
convert_proposal(proposal.clone());
let view_number = proposal.data.view_number().u64();
let proposal_bytes = bincode::serialize(&proposal).context("serializing proposal")?;
let leaf_hash = Committable::commit(&Leaf2::from_quorum_proposal(&proposal.data));
let leaf_hash = Committable::commit(&Leaf::from_quorum_proposal(&proposal.data));
let mut tx = self.db.write().await?;
tx.upsert(
"quorum_proposals",
Expand Down Expand Up @@ -868,7 +880,6 @@ impl SequencerPersistence for Persistence {
Proposal<SeqTypes, QuorumProposal<SeqTypes>>,
) -> Proposal<SeqTypes, QuorumProposal2<SeqTypes>>,
) -> anyhow::Result<()> {
// TODO:
Ok(())
}
}
Expand Down

0 comments on commit 96f2271

Please sign in to comment.