Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

VID share distribution algorithm (2) #2163

Merged
merged 11 commits into from
Dec 8, 2023
12 changes: 11 additions & 1 deletion crates/hotshot/src/tasks/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ use hotshot_task_impls::{
vid::{VIDTaskState, VIDTaskTypes},
view_sync::{ViewSyncTaskState, ViewSyncTaskStateTypes},
};
use hotshot_types::traits::election::Membership;
use hotshot_types::{
event::Event,
message::Messages,
Expand Down Expand Up @@ -206,7 +207,15 @@ pub async fn add_consensus_task<TYPES: NodeType, I: NodeImplementation<TYPES>>(
let registry = task_runner.registry.clone();
let (payload, metadata) = <TYPES::BlockPayload as BlockPayload>::genesis();
// Impossible for `unwrap` to fail on the genesis payload.
let payload_commitment = vid_commitment(&payload.encode().unwrap().collect());
let payload_commitment = vid_commitment(
&payload.encode().unwrap().collect(),
handle
.hotshot
.inner
.memberships
.quorum_membership
.total_nodes(),
);
// build the consensus task
let consensus_state = ConsensusTaskState {
registry: registry.clone(),
Expand Down Expand Up @@ -352,6 +361,7 @@ pub async fn add_da_task<TYPES: NodeType, I: NodeImplementation<TYPES>>(
api: c_api.clone(),
consensus: handle.hotshot.get_consensus(),
da_membership: c_api.inner.memberships.da_membership.clone().into(),
quorum_membership: c_api.inner.memberships.quorum_membership.clone().into(),
da_network: c_api.inner.networks.da_network.clone().into(),
cur_view: TYPES::Time::new(0),
vote_collector: None,
Expand Down
7 changes: 6 additions & 1 deletion crates/task-impls/src/consensus.rs
Original file line number Diff line number Diff line change
Expand Up @@ -614,11 +614,16 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
if let Some(encoded_txns) =
consensus.saved_payloads.get(leaf.get_payload_commitment())
{
let num_quorum_committee = self.quorum_membership.total_nodes();

let payload = BlockPayload::from_bytes(
encoded_txns.clone().into_iter(),
leaf.get_block_header().metadata(),
num_quorum_committee,
);
if let Err(e) = leaf.fill_block_payload(payload) {
if let Err(e) =
leaf.fill_block_payload(payload, num_quorum_committee)
{
error!(
"Saved block payload and commitment don't match: {:?}",
e
Expand Down
8 changes: 7 additions & 1 deletion crates/task-impls/src/da.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ pub struct DATaskState<
/// Membership for the DA committee
pub da_membership: Arc<TYPES::Membership>,

/// Membership for the quorum committee
pub quorum_membership: Arc<TYPES::Membership>,
rob-maron marked this conversation as resolved.
Show resolved Hide resolved

/// Network for DA
pub da_network: Arc<I::CommitteeNetwork>,

Expand Down Expand Up @@ -113,7 +116,10 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
return None;
}

let payload_commitment = vid_commitment(&proposal.data.encoded_transactions);
let payload_commitment = vid_commitment(
&proposal.data.encoded_transactions,
self.quorum_membership.total_nodes(),
);
let encoded_transactions_hash = Sha256::digest(&proposal.data.encoded_transactions);

// ED Is this the right leader?
Expand Down
5 changes: 4 additions & 1 deletion crates/task-impls/src/transactions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,10 @@ impl<TYPES: NodeType, I: NodeImplementation<TYPES>, A: ConsensusApi<TYPES, I> +
// <https://github.com/EspressoSystems/HotShot/issues/1822>
let txns = self.wait_for_transactions(parent_leaf).await?;
let (payload, metadata) =
match <TYPES::BlockPayload as BlockPayload>::from_transactions(txns) {
match <TYPES::BlockPayload as BlockPayload>::from_transactions(
txns,
self.membership.total_nodes(),
) {
Ok((payload, metadata)) => (payload, metadata),
Err(e) => {
error!("Failed to build the block payload: {:?}.", e);
Expand Down
10 changes: 9 additions & 1 deletion crates/testing/src/task_helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,15 @@ async fn build_quorum_proposal_and_signature(

// every event input is seen on the event stream in the output.
let block = <VIDBlockPayload as TestableBlock>::genesis();
let payload_commitment = vid_commitment(&block.encode().unwrap().collect());
let payload_commitment = vid_commitment(
&block.encode().unwrap().collect(),
handle
.hotshot
.inner
.memberships
.quorum_membership
.total_nodes(),
);
let block_header = VIDBlockHeader::new(payload_commitment, (), &parent_leaf.block_header);
let leaf = Leaf {
view_number: ViewNumber::new(view),
Expand Down
12 changes: 10 additions & 2 deletions crates/testing/tests/da_task.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use hotshot_types::{
data::{DAProposal, ViewNumber},
simple_vote::{DAData, DAVote},
traits::{
block_contents::vid_commitment, consensus_api::ConsensusSharedApi,
block_contents::vid_commitment, consensus_api::ConsensusSharedApi, election::Membership,
node_implementation::NodeType, state::ConsensusTime,
},
};
Expand Down Expand Up @@ -35,7 +35,15 @@ async fn test_da_task() {
let pub_key = *api.public_key();
let transactions = vec![VIDTransaction(vec![0])];
let encoded_transactions = VIDTransaction::encode(transactions.clone()).unwrap();
let payload_commitment = vid_commitment(&encoded_transactions);
let payload_commitment = vid_commitment(
&encoded_transactions,
handle
.hotshot
.inner
.memberships
.quorum_membership
.total_nodes(),
);
let encoded_transactions_hash = Sha256::digest(&encoded_transactions);

let signature =
Expand Down
15 changes: 10 additions & 5 deletions crates/types/src/block_impl.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use std::{
use crate::{
data::{BlockError, VidCommitment, VidScheme, VidSchemeTrait},
traits::{
block_contents::{vid_commitment, BlockHeader, Transaction},
block_contents::{vid_commitment, BlockHeader, Transaction, NUM_STORAGE_NODES},
state::TestableBlock,
BlockPayload,
},
Expand Down Expand Up @@ -86,7 +86,7 @@ impl VIDBlockPayload {
let encoded = VIDTransaction::encode(vec![VIDTransaction(txns.clone())]).unwrap();
VIDBlockPayload {
transactions: vec![VIDTransaction(txns)],
payload_commitment: vid_commitment(&encoded),
payload_commitment: vid_commitment(&encoded, NUM_STORAGE_NODES),
}
}
}
Expand Down Expand Up @@ -115,19 +115,24 @@ impl BlockPayload for VIDBlockPayload {

fn from_transactions(
transactions: impl IntoIterator<Item = Self::Transaction>,
num_storage_nodes: usize,
) -> Result<(Self, Self::Metadata), Self::Error> {
let txns_vec: Vec<VIDTransaction> = transactions.into_iter().collect();
let encoded = VIDTransaction::encode(txns_vec.clone())?;
Ok((
Self {
transactions: txns_vec,
payload_commitment: vid_commitment(&encoded),
payload_commitment: vid_commitment(&encoded, num_storage_nodes),
},
(),
))
}

fn from_bytes<E>(encoded_transactions: E, _metadata: Self::Metadata) -> Self
fn from_bytes<E>(
encoded_transactions: E,
_metadata: Self::Metadata,
num_storage_nodes: usize,
) -> Self
where
E: Iterator<Item = u8>,
{
Expand All @@ -151,7 +156,7 @@ impl BlockPayload for VIDBlockPayload {

Self {
transactions,
payload_commitment: vid_commitment(&encoded_vec),
payload_commitment: vid_commitment(&encoded_vec, num_storage_nodes),
}
}

Expand Down
3 changes: 2 additions & 1 deletion crates/types/src/data.rs
Original file line number Diff line number Diff line change
Expand Up @@ -403,14 +403,15 @@ impl<TYPES: NodeType> Leaf<TYPES> {
pub fn fill_block_payload(
&mut self,
block_payload: TYPES::BlockPayload,
num_storage_nodes: usize,
) -> Result<(), BlockError> {
let encoded_txns = match block_payload.encode() {
// TODO (Keyao) [VALIDATED_STATE] - Avoid collect/copy on the encoded transaction bytes.
// <https://github.com/EspressoSystems/HotShot/issues/2115>
Ok(encoded) => encoded.into_iter().collect(),
Err(_) => return Err(BlockError::InvalidTransactionLength),
};
let commitment = vid_commitment(&encoded_txns);
let commitment = vid_commitment(&encoded_txns, num_storage_nodes);
rob-maron marked this conversation as resolved.
Show resolved Hide resolved
if commitment != self.block_header.payload_commitment() {
return Err(BlockError::InconsistentPayloadCommitment);
}
Expand Down
25 changes: 17 additions & 8 deletions crates/types/src/traits/block_contents.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,18 @@ pub trait BlockPayload:
/// If the transaction length conversion fails.
fn from_transactions(
transactions: impl IntoIterator<Item = Self::Transaction>,
num_storage_nodes: usize,
rob-maron marked this conversation as resolved.
Show resolved Hide resolved
) -> Result<(Self, Self::Metadata), Self::Error>;

/// Build a payload with the encoded transaction bytes and metadata.
/// Build a payload with the encoded transaction bytes, metadata,
/// and the associated number of VID storage nodes
///
/// `I` may be, but not necessarily is, the `Encode` type directly from `fn encode`.
fn from_bytes<I>(encoded_transactions: I, metadata: Self::Metadata) -> Self
fn from_bytes<I>(
encoded_transactions: I,
metadata: Self::Metadata,
num_storage_nodes: usize,
) -> Self
where
I: Iterator<Item = u8>;

Expand All @@ -83,13 +89,16 @@ pub trait BlockPayload:
/// # Panics
/// If the VID computation fails.
#[must_use]
pub fn vid_commitment(encoded_transactions: &Vec<u8>) -> <VidScheme as VidSchemeTrait>::Commit {
pub fn vid_commitment(
encoded_transactions: &Vec<u8>,
num_storage_nodes: usize,
rob-maron marked this conversation as resolved.
Show resolved Hide resolved
) -> <VidScheme as VidSchemeTrait>::Commit {
let num_chunks = 1 << num_storage_nodes.ilog2();

// TODO <https://github.com/EspressoSystems/HotShot/issues/1686>
let srs = test_srs(NUM_STORAGE_NODES);
// TODO We are using constant numbers for now, but they will change as the quorum size
// changes.
// TODO <https://github.com/EspressoSystems/HotShot/issues/1693>
let vid = VidScheme::new(NUM_CHUNKS, NUM_STORAGE_NODES, srs).unwrap();
let srs = test_srs(num_storage_nodes);

let vid = VidScheme::new(num_chunks, num_storage_nodes, srs).unwrap();
vid.commit_only(encoded_transactions).unwrap()
}

Expand Down
Loading