diff --git a/crates/core-primitives/src/header/mod.rs b/crates/core-primitives/src/header/mod.rs index 57b8943..2c82bc2 100644 --- a/crates/core-primitives/src/header/mod.rs +++ b/crates/core-primitives/src/header/mod.rs @@ -58,7 +58,7 @@ pub struct Header + TryFrom, Hash: HashT> { pub extrinsics_root: Hash::Output, /// A chain-specific digest of data useful for light clients or referencing auxiliary data. pub digest: Digest, - /// + /// Extension data. pub extension: HeaderExtension, } @@ -96,7 +96,6 @@ where + Into + TryFrom + sp_std::str::FromStr, - // + MaybeMallocSizeOf, Hash: HashT, Hash::Output: Default + sp_std::hash::Hash @@ -108,7 +107,6 @@ where + MaybeDisplay + SimpleBitOps + Codec, - // + MaybeMallocSizeOf, { type Number = Number; type Hash = ::Output; diff --git a/crates/core-primitives/src/lib.rs b/crates/core-primitives/src/lib.rs index c402ab7..bc15dad 100644 --- a/crates/core-primitives/src/lib.rs +++ b/crates/core-primitives/src/lib.rs @@ -26,8 +26,8 @@ use sp_runtime::generic::Digest; pub mod header; pub use header::*; -pub mod sidercar; -pub use sidercar::*; +pub mod sidecar; +pub use sidecar::*; pub mod localstorage; diff --git a/crates/core-primitives/src/localstorage.rs b/crates/core-primitives/src/localstorage.rs index 6773277..6a9e3ca 100644 --- a/crates/core-primitives/src/localstorage.rs +++ b/crates/core-primitives/src/localstorage.rs @@ -25,18 +25,21 @@ use sp_runtime::traits::Block; use sp_core::offchain::StorageKind; +/// Save a key-value pair to local storage with the provided prefix. pub fn save_to_localstorage_with_prefix(key: &[u8], value: &[u8], prefix: &[u8]) { let mut prefixed_key = prefix.to_vec(); prefixed_key.extend_from_slice(key); sp_io::offchain::local_storage_set(StorageKind::PERSISTENT, &prefixed_key, value); } +/// Retrieve a value from local storage using the provided key and prefix. pub fn get_from_localstorage_with_prefix(key: &[u8], prefix: &[u8]) -> Option> { let mut prefixed_key = prefix.to_vec(); prefixed_key.extend_from_slice(key); sp_io::offchain::local_storage_get(StorageKind::PERSISTENT, &prefixed_key) } +/// Save a key-value pair to local storage (usable outside the runtime) with the provided prefix. #[cfg(feature = "outside")] pub fn save_to_localstorage_with_prefix_outside>( db: &mut OffchainDb, @@ -49,6 +52,7 @@ pub fn save_to_localstorage_with_prefix_outside>( db.local_storage_set(StorageKind::PERSISTENT, &prefixed_key, value); } +/// Retrieve a value from local storage (usable outside the runtime) using the provided key and prefix. #[cfg(feature = "outside")] pub fn get_from_localstorage_with_prefix_outside>( db: &mut OffchainDb, diff --git a/crates/core-primitives/src/sidercar.rs b/crates/core-primitives/src/sidecar.rs similarity index 58% rename from crates/core-primitives/src/sidercar.rs rename to crates/core-primitives/src/sidecar.rs index c0918d5..3a82afb 100644 --- a/crates/core-primitives/src/sidercar.rs +++ b/crates/core-primitives/src/sidecar.rs @@ -36,12 +36,12 @@ use sp_io::hashing; use melo_das_primitives::config::FIELD_ELEMENTS_PER_BLOB; -const SIDERCAR_PREFIX: &[u8] = b"sidercar"; +const SIDERCAR_PREFIX: &[u8] = b"sidecar"; -// Status of the sidercar, including failure to retrieve data and attestation errors +/// Represents the possible statuses of the sidecar, including failures and success cases. #[derive(Encode, Decode, Debug, Clone, PartialEq, Eq)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] -pub enum SidercarStatus { +pub enum SidecarStatus { // Failed to retrieve data NotFound, // Proof error @@ -50,24 +50,27 @@ pub enum SidercarStatus { Success, } +/// Contains essential metadata for the sidecar, such as data length, hash, commitments, and proofs. #[derive(Encode, Debug, Decode, Clone, PartialEq, Eq)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] -pub struct SidercarMetadata { - // Data length +pub struct SidecarMetadata { + /// Length of the data. pub data_len: u32, - // Hash of the data + /// Hash representation of the data. pub blobs_hash: sp_core::H256, - // Commitments + /// Commitments related to the data. pub commitments: Vec, - // Proofs + /// Proofs confirming the validity of the data. pub proofs: Vec, } -impl SidercarMetadata { +impl SidecarMetadata { + /// Calculates and returns the ID (hash) of the metadata. pub fn id(&self) -> [u8; 32] { hashing::blake2_256(&self.encode()) } + /// Verifies the provided bytes against the stored commitments and proofs. pub fn verify_bytes(&self, bytes: &[u8]) -> Result { let kzg = KZG::default_embedded(); bytes_to_blobs(bytes, FIELD_ELEMENTS_PER_BLOB).and_then(|blobs| { @@ -81,11 +84,12 @@ impl SidercarMetadata { }) } + /// Attempts to generate a `SidecarMetadata` instance from given application data bytes. pub fn try_from_app_data(bytes: &[u8]) -> Result { let kzg = KZG::default_embedded(); let data_len = bytes.len() as u32; - let blobs_hash = Sidercar::calculate_id(bytes); + let blobs_hash = Sidecar::calculate_id(bytes); let blobs = bytes_to_blobs(bytes, FIELD_ELEMENTS_PER_BLOB)?; @@ -107,10 +111,12 @@ impl SidercarMetadata { #[cfg(not(feature = "std"))] { - let mut commitments = Vec::new(); - let mut proofs = Vec::new(); - - for blob in blobs.iter() { + let blob_count = blobs.len(); + + let mut commitments = Vec::with_capacity(blob_count); + let mut proofs = Vec::with_capacity(blob_count); + + for blob in &blobs { match blob.commit_and_proof(&kzg, FIELD_ELEMENTS_PER_BLOB) { Ok((commitment, proof)) => { commitments.push(commitment); @@ -119,37 +125,43 @@ impl SidercarMetadata { Err(e) => return Err(format!("Failed to commit and proof: {}", e)), } } - + Ok(Self { data_len, blobs_hash: blobs_hash.into(), commitments, proofs }) } + } } +/// Represents a sidecar, encapsulating its metadata, potential data, and its current status. #[derive(Encode, Decode, Clone, PartialEq, Eq)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] -pub struct Sidercar { - // Metadata - pub metadata: SidercarMetadata, - // Data +pub struct Sidecar { + /// Metadata associated with the sidecar. + pub metadata: SidecarMetadata, + /// Data blob associated with the sidecar, if any. pub blobs: Option>, - // Status; None means an unhandled edge case and data errors should not be reported at this time - pub status: Option, + /// Current status of the sidecar; `None` means an unhandled edge case, so data errors shouldn't be reported. + pub status: Option, } -impl Sidercar { - pub fn new(metadata: SidercarMetadata, blobs: Option>) -> Self { +impl Sidecar { + /// Constructs a new sidecar instance with the provided metadata and data. + pub fn new(metadata: SidecarMetadata, blobs: Option>) -> Self { Self { metadata, blobs, status: None } } + /// Calculates and returns the ID (hash) of the sidecar based on its metadata. pub fn id(&self) -> [u8; 32] { - // Returns hash of sidercar metadata converted to bytes + // Returns hash of sidecar metadata converted to bytes self.metadata.id() } + /// Calculates and returns the ID (hash) based on a given blob. pub fn calculate_id(blob: &[u8]) -> [u8; 32] { hashing::blake2_256(blob) } + /// Checks the hash of the stored blobs against the metadata's blob hash. pub fn check_hash(&self) -> bool { match self.blobs { Some(ref blobs) => self.metadata.blobs_hash[..] == Self::calculate_id(blobs), @@ -157,40 +169,67 @@ impl Sidercar { } } + /// Determines if the sidecar status represents an unavailability scenario. pub fn is_unavailability(&self) -> bool { - self.status != Some(SidercarStatus::Success) && self.status.is_some() + self.status != Some(SidecarStatus::Success) && self.status.is_some() } + /// Sets the status of the sidecar to 'NotFound'. pub fn set_not_found(&mut self) { - self.status = Some(SidercarStatus::NotFound); + self.status = Some(SidecarStatus::NotFound); } + /// Retrieves a sidecar instance from local storage based on a given key. + /// + /// # Parameters + /// + /// * `key`: Byte slice that represents the key used to store the sidecar. + /// + /// # Returns + /// + /// An `Option` that contains a `Sidecar` if found, otherwise `None`. pub fn from_local(key: &[u8]) -> Option { - let maybe_sidercar = get_from_localstorage_with_prefix(key, SIDERCAR_PREFIX); - match maybe_sidercar { - Some(data) => Sidercar::decode(&mut &data[..]).ok(), + let maybe_sidecar = get_from_localstorage_with_prefix(key, SIDERCAR_PREFIX); + match maybe_sidecar { + Some(data) => Sidecar::decode(&mut &data[..]).ok(), None => None, } } + /// Saves the sidecar instance to local storage. pub fn save_to_local(&self) { save_to_localstorage_with_prefix(&self.id(), &self.encode(), SIDERCAR_PREFIX); } #[cfg(feature = "outside")] + /// Retrieves a sidecar instance from an external local storage based on a given key and database reference. + /// + /// # Parameters + /// + /// * `key`: Byte slice that represents the key used to store the sidecar. + /// * `db`: Mutable reference to the offchain database. + /// + /// # Returns + /// + /// An `Option` that contains a `Sidecar` if found, otherwise `None`. pub fn from_local_outside>( key: &[u8], db: &mut OffchainDb, - ) -> Option { - let maybe_sidercar = + ) -> Option { + let maybe_sidecar = get_from_localstorage_with_prefix_outside::(db, key, SIDERCAR_PREFIX); - match maybe_sidercar { - Some(data) => Sidercar::decode(&mut &data[..]).ok(), + match maybe_sidecar { + Some(data) => Sidecar::decode(&mut &data[..]).ok(), None => None, } } #[cfg(feature = "outside")] + /// Saves the sidecar instance to an external local storage using a given database reference. + /// + /// # Parameters + /// + /// * `db`: Mutable reference to the offchain database. pub fn save_to_local_outside>( &self, db: &mut OffchainDb, @@ -213,8 +252,8 @@ mod tests { // Mock your `KZGCommitment` and `KZGProof` here if needed #[test] - fn test_sidercar_metadata_id() { - let metadata = SidercarMetadata { + fn test_sidecar_metadata_id() { + let metadata = SidecarMetadata { data_len: 42, blobs_hash: H256::from([1u8; 32]), commitments: vec![], // Populate this with real or mocked data @@ -226,8 +265,8 @@ mod tests { } #[test] - fn test_sidercar_new() { - let metadata = SidercarMetadata { + fn test_sidecar_new() { + let metadata = SidecarMetadata { data_len: 42, blobs_hash: H256::from([1u8; 32]), commitments: vec![], // Populate this with real or mocked data @@ -235,51 +274,51 @@ mod tests { }; let blobs = Some(vec![1, 2, 3]); - let sidercar = Sidercar::new(metadata.clone(), blobs.clone()); + let sidecar = Sidecar::new(metadata.clone(), blobs.clone()); - assert_eq!(sidercar.metadata, metadata); - assert_eq!(sidercar.blobs, blobs); - assert_eq!(sidercar.status, None); + assert_eq!(sidecar.metadata, metadata); + assert_eq!(sidecar.blobs, blobs); + assert_eq!(sidecar.status, None); } #[test] - fn test_sidercar_id() { - let metadata = SidercarMetadata { + fn test_sidecar_id() { + let metadata = SidecarMetadata { data_len: 42, blobs_hash: H256::from([1u8; 32]), commitments: vec![], // Populate this with real or mocked data proofs: vec![], // Populate this with real or mocked data }; - let sidercar = Sidercar::new(metadata.clone(), None); - assert_eq!(sidercar.id(), metadata.id()); + let sidecar = Sidecar::new(metadata.clone(), None); + assert_eq!(sidecar.id(), metadata.id()); } #[test] - fn test_sidercar_check_hash() { - let metadata = SidercarMetadata { + fn test_sidecar_check_hash() { + let metadata = SidecarMetadata { data_len: 3, blobs_hash: H256::from(hashing::blake2_256(&[1, 2, 3])), commitments: vec![], // Populate this with real or mocked data proofs: vec![], // Populate this with real or mocked data }; - let sidercar = Sidercar::new(metadata.clone(), Some(vec![1, 2, 3])); - assert!(sidercar.check_hash()); + let sidecar = Sidecar::new(metadata.clone(), Some(vec![1, 2, 3])); + assert!(sidecar.check_hash()); } #[test] - fn test_sidercar_is_unavailability() { - let metadata = SidercarMetadata { + fn test_sidecar_is_unavailability() { + let metadata = SidecarMetadata { data_len: 3, blobs_hash: H256::from([1u8; 32]), commitments: vec![], proofs: vec![], }; - let mut sidercar = Sidercar::new(metadata, None); - sidercar.status = Some(SidercarStatus::NotFound); + let mut sidecar = Sidecar::new(metadata, None); + sidecar.status = Some(SidecarStatus::NotFound); - assert!(sidercar.is_unavailability()); + assert!(sidecar.is_unavailability()); } } diff --git a/crates/core-primitives/src/testing.rs b/crates/core-primitives/src/testing.rs index 7e3b172..f5834e7 100644 --- a/crates/core-primitives/src/testing.rs +++ b/crates/core-primitives/src/testing.rs @@ -16,10 +16,10 @@ // limitations under the License. //! Testing utilities. -use crate::HeaderExtension; use crate::traits::ExtendedHeader; use crate::traits::HeaderCommitList; use crate::Header as HeaderT; +use crate::HeaderExtension; use lazy_static::lazy_static; use melo_das_primitives::KZGCommitment; @@ -47,6 +47,7 @@ use std::{ }; lazy_static! { + /// A static reference containing test commitments. pub static ref TEST_COMMITMENTS: Vec = vec![ KZGCommitment::rand(), KZGCommitment::rand(), @@ -57,38 +58,40 @@ lazy_static! { ]; } +/// `CommitListTest` is a mock structure that implements `HeaderCommitList` with no data. pub struct CommitListTest(); impl HeaderCommitList for CommitListTest { + // Always returns an empty list of `KZGCommitment`. fn last() -> Vec { vec![] } } +/// `CommitListTestWithData` is a mock structure that implements `HeaderCommitList` with predefined data. pub struct CommitListTestWithData(); impl HeaderCommitList for CommitListTestWithData { + // Returns a predefined list of `KZGCommitment` for testing. fn last() -> Vec { TEST_COMMITMENTS.to_vec() } } impl CommitListTestWithData { + /// Converts the static `TEST_COMMITMENTS` into bytes. pub fn commit_bytes() -> Vec { - TEST_COMMITMENTS - .iter() - .map(|c| c.to_bytes()) - .flatten() - .collect() + TEST_COMMITMENTS.iter().map(|c| c.to_bytes()).flatten().collect() } + /// Creates a `HeaderExtension` with the bytes representation of `TEST_COMMITMENTS`. pub fn header_extension() -> HeaderExtension { - HeaderExtension { - commitments_bytes: Self::commit_bytes(), - } + HeaderExtension { commitments_bytes: Self::commit_bytes() } } } + +/// From substrate sp_runtime test utils /// A dummy type which can be used instead of regular cryptographic primitives. /// /// 1. Wraps a `u64` `AccountId` and is able to `IdentifyAccount`. @@ -213,6 +216,7 @@ impl traits::IdentifyAccount for UintAuthorityId { } } +/// From substrate sp_runtime test utils /// A dummy signature type, to match `UintAuthorityId`. #[derive(Eq, PartialEq, Clone, Debug, Hash, Serialize, Deserialize, Encode, Decode, TypeInfo)] pub struct TestSignature(pub u64, pub Vec); @@ -248,6 +252,7 @@ impl Header { } } +/// From substrate sp_runtime test utils /// An opaque extrinsic wrapper type. #[derive(PartialEq, Eq, Clone, Debug, Encode, Decode)] pub struct ExtrinsicWrapper(Xt); @@ -284,6 +289,7 @@ impl Deref for ExtrinsicWrapper { } } +/// From substrate sp_runtime test utils /// Testing block #[derive(PartialEq, Eq, Clone, Serialize, Debug, Encode, Decode)] pub struct Block { @@ -329,6 +335,7 @@ where } } +/// From substrate sp_runtime test utils /// Test transaction, tuple of (sender, call, signed_extra) /// with index only used if sender is some. /// diff --git a/crates/das-network/protocol/src/lib.rs b/crates/das-network/protocol/src/lib.rs index b19a6f2..1d4e540 100644 --- a/crates/das-network/protocol/src/lib.rs +++ b/crates/das-network/protocol/src/lib.rs @@ -18,15 +18,26 @@ use std::fmt::Debug; pub use melo_das_network::Service as DasDhtService; +/// `DasDht` trait provides an asynchronous interface for interacting with the DHT (Distributed Hash Table). #[async_trait] pub trait DasDht: Send + Debug + 'static { - /// Get the addresses for the given [`AuthorityId`] from the local address cache. + /// Asynchronously puts a key-value pair into the DHT. + /// + /// # Arguments + /// + /// * `key` - The key to be inserted into the DHT. + /// * `value` - The value associated with the provided key. + /// + /// # Returns + /// + /// An `Option<()>` which is `Some(())` if the operation is successful and `None` otherwise. async fn put_value_to_dht(&mut self, key: KademliaKey, value: Vec) -> Option<()>; } #[async_trait] impl DasDht for DasDhtService { + /// Implementation of the `put_value_to_dht` method for `DasDhtService`. async fn put_value_to_dht(&mut self, key: KademliaKey, value: Vec) -> Option<()> { DasDhtService::put_value_to_dht(self, key, value).await } -} \ No newline at end of file +} diff --git a/crates/das-network/src/dht_work.rs b/crates/das-network/src/dht_work.rs index 98194d7..8c4cf63 100644 --- a/crates/das-network/src/dht_work.rs +++ b/crates/das-network/src/dht_work.rs @@ -23,23 +23,26 @@ use melo_erasure_coding::bytes_vec_to_blobs; /// Logging target for the mmr gadget. pub const LOG_TARGET: &str = "das-network::dht_work"; -use crate::{NetworkProvider, ServicetoWorkerMsg, Sidercar, SidercarStatus}; +use crate::{NetworkProvider, ServicetoWorkerMsg, Sidecar, SidecarStatus}; + +/// Represents the worker responsible for DHT network operations. pub struct Worker> { #[allow(dead_code)] client: Arc, - /// Channel receiver for messages send by a [`crate::Service`]. + /// Channel receiver for messages sent by the main service. from_service: Fuse>, - /// DHT network + /// DHT network instance. network: Arc, - /// Channel we receive Dht events on. + /// Channel receiver for DHT events. dht_event_rx: DhtEventStream, - /// + /// Backend storage instance. pub backend: Arc, + /// Off-chain database instance. pub offchain_db: OffchainDb, } @@ -50,6 +53,7 @@ where DhtEventStream: Stream + Unpin, BE: Backend, { + /// Attempts to create a new worker instance. pub(crate) fn try_build( from_service: mpsc::Receiver, client: Arc, @@ -70,13 +74,14 @@ where warn!( target: LOG_TARGET, // TODO - "Can't spawn a for a node without offchain storage." + "Can't spawn a worker for a node without offchain storage." ); None }, } } + /// Main loop for the worker, where it listens to events and messages. pub async fn run(mut self, start: FStart) where FStart: Fn(), @@ -96,48 +101,50 @@ where } } + /// Handles DHT events. async fn handle_dht_event(&mut self, event: DhtEvent) { match event { DhtEvent::ValueFound(v) => { self.handle_dht_value_found_event(v); }, DhtEvent::ValueNotFound(key) => self.handle_dht_value_not_found_event(key), - // TODO handle other events + // TODO: handle other events _ => {}, } } + // Handles the event where a value is found in the DHT. fn handle_dht_value_found_event(&mut self, values: Vec<(KademliaKey, Vec)>) { for (key, value) in values { - let maybe_sidercar = - Sidercar::from_local_outside::(key.as_ref(), &mut self.offchain_db); - match maybe_sidercar { - Some(sidercar) => { - if sidercar.status.is_none() { - let data_hash = Sidercar::calculate_id(&value); - let mut new_sidercar = sidercar.clone(); - if data_hash != sidercar.metadata.blobs_hash.as_bytes() { - new_sidercar.status = Some(SidercarStatus::ProofError); + let maybe_sidecar = + Sidecar::from_local_outside::(key.as_ref(), &mut self.offchain_db); + match maybe_sidecar { + Some(sidecar) => { + if sidecar.status.is_none() { + let data_hash = Sidecar::calculate_id(&value); + let mut new_sidecar = sidecar.clone(); + if data_hash != sidecar.metadata.blobs_hash.as_bytes() { + new_sidecar.status = Some(SidecarStatus::ProofError); } else { let kzg = KZG::default_embedded(); // TODO bytes to blobs let blobs = bytes_vec_to_blobs(&[value.clone()], 1).unwrap(); let encoding_valid = Blob::verify_batch( &blobs, - &sidercar.metadata.commitments, - &sidercar.metadata.proofs, + &sidecar.metadata.commitments, + &sidecar.metadata.proofs, &kzg, FIELD_ELEMENTS_PER_BLOB, ) .unwrap(); if encoding_valid { - new_sidercar.blobs = Some(value.clone()); - new_sidercar.status = Some(SidercarStatus::Success); + new_sidecar.blobs = Some(value.clone()); + new_sidecar.status = Some(SidecarStatus::Success); } else { - new_sidercar.status = Some(SidercarStatus::ProofError); + new_sidecar.status = Some(SidecarStatus::ProofError); } } - new_sidercar.save_to_local_outside::(&mut self.offchain_db) + new_sidecar.save_to_local_outside::(&mut self.offchain_db) } }, None => {}, @@ -145,21 +152,23 @@ where } } + // Handles the event where a value is not found in the DHT. fn handle_dht_value_not_found_event(&mut self, key: KademliaKey) { - let maybe_sidercar = - Sidercar::from_local_outside::(key.as_ref(), &mut self.offchain_db); - match maybe_sidercar { - Some(sidercar) => { - if sidercar.status.is_none() { - let mut new_sidercar = sidercar.clone(); - new_sidercar.status = Some(SidercarStatus::NotFound); - new_sidercar.save_to_local_outside::(&mut self.offchain_db) + let maybe_sidecar = + Sidecar::from_local_outside::(key.as_ref(), &mut self.offchain_db); + match maybe_sidecar { + Some(sidecar) => { + if sidecar.status.is_none() { + let mut new_sidecar = sidecar.clone(); + new_sidecar.status = Some(SidecarStatus::NotFound); + new_sidecar.save_to_local_outside::(&mut self.offchain_db) } }, None => {}, } } + // Processes messages coming from the main service. fn process_message_from_service(&self, msg: ServicetoWorkerMsg) { match msg { ServicetoWorkerMsg::PutValueToDht(key, value, sender) => { diff --git a/crates/das-network/src/lib.rs b/crates/das-network/src/lib.rs index 6cc65d1..5708d3d 100644 --- a/crates/das-network/src/lib.rs +++ b/crates/das-network/src/lib.rs @@ -12,11 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Necessary imports for the module. use futures::{ channel::{mpsc, oneshot}, Stream, }; +// Logging macro. pub use log::warn; +// Common primitives and traits. pub use node_primitives::AccountId; pub use sc_client_api::Backend; pub use sc_network::{DhtEvent, KademliaKey, NetworkDHTProvider, NetworkSigner, NetworkStateInfo}; @@ -24,6 +27,7 @@ pub use sc_offchain::OffchainDb; pub use sp_runtime::traits::{Block, Header}; pub use std::sync::Arc; +// Internal module imports. pub use crate::{dht_work::Worker, service::Service}; mod dht_work; @@ -32,12 +36,15 @@ mod tx_pool_listener; pub use tx_pool_listener::{start_tx_pool_listener, TPListenerParams}; +/// Trait to encapsulate necessary network-related operations. pub trait NetworkProvider: NetworkDHTProvider + NetworkStateInfo + NetworkSigner {} impl NetworkProvider for T where T: NetworkDHTProvider + NetworkStateInfo + NetworkSigner {} -pub use melo_core_primitives::{Sidercar, SidercarMetadata, SidercarStatus}; +// Import core primitives related to sidecars. +pub use melo_core_primitives::{Sidecar, SidecarMetadata, SidecarStatus}; use sp_core::H256; +/// Instantiates a new DHT Worker with the given parameters. pub fn new_worker( client: Arc, network: Arc, @@ -54,14 +61,17 @@ where Worker::try_build(from_service, client, backend, network, dht_event_rx) } +/// Creates a new channel for communication between the service and worker. pub fn new_workgroup() -> (mpsc::Sender, mpsc::Receiver) { mpsc::channel(0) } +/// Initializes a new Service instance with the specified communication channel. pub fn new_service(to_worker: mpsc::Sender) -> Service { Service::new(to_worker) } +/// Conveniently creates both a Worker and Service with the given parameters. pub fn new_worker_and_service( client: Arc, network: Arc, @@ -82,16 +92,19 @@ where Some((worker, service)) } -pub fn sidercar_kademlia_key(sidercar: &Sidercar) -> KademliaKey { - KademliaKey::from(Vec::from(sidercar.id())) +/// Converts a sidecar instance into a Kademlia key. +pub fn sidecar_kademlia_key(sidecar: &Sidecar) -> KademliaKey { + KademliaKey::from(Vec::from(sidecar.id())) } -pub fn kademlia_key_from_sidercar_id(sidercar_id: &H256) -> KademliaKey { - KademliaKey::from(Vec::from(&sidercar_id[..])) +/// Converts a sidecar ID into a Kademlia key. +pub fn kademlia_key_from_sidecar_id(sidecar_id: &H256) -> KademliaKey { + KademliaKey::from(Vec::from(&sidecar_id[..])) } -/// Message send from the [`Service`] to the [`Worker`]. +/// Enumerated messages that can be sent from the Service to the Worker. pub enum ServicetoWorkerMsg { - /// See [`Service::get_addresses_by_authority_id`]. + /// Request to insert a value into the DHT. + /// Contains the key for insertion, the data to insert, and a sender to acknowledge completion. PutValueToDht(KademliaKey, Vec, oneshot::Sender>), } diff --git a/crates/das-network/src/service.rs b/crates/das-network/src/service.rs index 4b8ade5..6f5b65b 100644 --- a/crates/das-network/src/service.rs +++ b/crates/das-network/src/service.rs @@ -20,33 +20,48 @@ use std::fmt::Debug; use crate::{KademliaKey, ServicetoWorkerMsg}; -/// Service to interact with the [`crate::Worker`]. +/// `Service` serves as an intermediary to interact with the Worker, handling requests and facilitating communication. +/// It mainly operates on the message passing mechanism between service and worker. #[derive(Clone)] pub struct Service { + // Channel sender to send messages to the worker. to_worker: mpsc::Sender, } impl Debug for Service { + /// Provides a human-readable representation of the Service, useful for debugging. fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_tuple("DasNetworkService").finish() } } -/// A [`Service`] allows to interact with a [`crate::Worker`], e.g. by querying the -/// [`crate::Worker`]'s local address cache for a given [`AuthorityId`]. impl Service { + /// Constructs a new `Service` instance with a given channel to communicate with the worker. pub(crate) fn new(to_worker: mpsc::Sender) -> Self { Self { to_worker } } + /// Puts a key-value pair to the DHT (Distributed Hash Table). + /// Sends a message to the worker to perform the DHT insertion and awaits its acknowledgment. + /// + /// # Parameters + /// - `key`: The `KademliaKey` under which the value will be stored in the DHT. + /// - `value`: The actual data to be stored in the DHT. + /// + /// # Returns + /// - An `Option<()>` signaling the success or failure of the operation. + /// The `None` variant indicates a failure. pub async fn put_value_to_dht(&mut self, key: KademliaKey, value: Vec) -> Option<()> { + // Create a one-shot channel for immediate communication. let (tx, rx) = oneshot::channel(); + // Send a request to the worker to put the key-value pair to the DHT. self.to_worker .send(ServicetoWorkerMsg::PutValueToDht(key, value, tx)) .await .ok()?; + // Wait for the worker's response. rx.await.ok().flatten() } } diff --git a/crates/das-network/src/tx_pool_listener.rs b/crates/das-network/src/tx_pool_listener.rs index 9b1df7c..a128a57 100644 --- a/crates/das-network/src/tx_pool_listener.rs +++ b/crates/das-network/src/tx_pool_listener.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{Arc, Backend,OffchainDb,warn}; +use crate::{warn, Arc, Backend, OffchainDb}; use futures::StreamExt; use melo_core_primitives::{traits::Extractor, Encode}; use sc_network::NetworkDHTProvider; @@ -21,10 +21,12 @@ use sp_api::ProvideRuntimeApi; use sp_blockchain::HeaderBackend; use sp_runtime::traits::Block as BlockT; +// Define a constant for logging with a target string const LOG_TARGET: &str = "tx_pool_listener"; -use crate::{sidercar_kademlia_key, NetworkProvider, Sidercar, SidercarMetadata}; +use crate::{sidecar_kademlia_key, NetworkProvider, Sidecar, SidecarMetadata}; +/// Parameters required for the transaction pool listener. #[derive(Clone)] pub struct TPListenerParams { pub client: Arc, @@ -33,6 +35,8 @@ pub struct TPListenerParams { pub backend: Arc, } +/// Main function responsible for starting the transaction pool listener. +/// It monitors the transaction pool for incoming transactions and processes them accordingly. pub async fn start_tx_pool_listener( TPListenerParams { client, network, transaction_pool, backend }: TPListenerParams< Client, @@ -48,10 +52,14 @@ pub async fn start_tx_pool_listener( Client::Api: Extractor, BE: Backend, { + // Log the start of the transaction pool listener tracing::info!( target: LOG_TARGET, "Starting transaction pool listener.", ); + + // Initialize the off-chain database using the backend's off-chain storage. + // If unavailable, log a warning and return without starting the listener. let mut offchain_db = match backend.offchain_storage() { Some(offchain_storage) => OffchainDb::new(offchain_storage), None => { @@ -59,75 +67,69 @@ pub async fn start_tx_pool_listener( target: LOG_TARGET, "Can't spawn a transaction pool listener for a node without offchain storage." ); - return + return; }, }; - // Obtain the import notification event stream from the transaction pool + + // Get the stream of import notifications from the transaction pool let mut import_notification_stream = transaction_pool.import_notification_stream(); - // Handle the transaction pool import notification event stream + // Process each import notification as they arrive in the stream while let Some(notification) = import_notification_stream.next().await { - match transaction_pool.ready_transaction(¬ification) { - Some(transaction) => { - // TODO: Can we avoid decoding the extrinsic here? - let encoded = transaction.data().encode(); - let at = client.info().best_hash; - match client.runtime_api().extract(at, &encoded) { - Ok(res) => match res { - Some(data) => { - data.into_iter().for_each( - |(data_hash, bytes_len, commitments, proofs)| { - tracing::debug!( - target: LOG_TARGET, - "New blob transaction found. Hash: {:?}", data_hash, - ); - - let metadata = SidercarMetadata { - data_len: bytes_len, - blobs_hash: data_hash, - commitments, - proofs, - }; - - let fetch_value_from_network = |sidercar: &Sidercar| { - network.get_value(&sidercar_kademlia_key(sidercar)); - }; + if let Some(transaction) = transaction_pool.ready_transaction(¬ification) { + // Encode the transaction data for processing + let encoded = transaction.data().encode(); + let at = client.info().best_hash; - match Sidercar::from_local_outside::(&metadata.id(), &mut offchain_db) { - Some(sidercar) => { - if sidercar.status.is_none() { - fetch_value_from_network(&sidercar); - } - }, - None => { - let sidercar = - Sidercar { blobs: None, metadata, status: None }; - sidercar.save_to_local_outside::(&mut offchain_db); - fetch_value_from_network(&sidercar); - }, - } - }, - ); - }, - None => { - tracing::debug!( - target: LOG_TARGET, - "Decoding of extrinsic failed. Transaction: {:?}", - transaction.hash(), - ); - }, - }, - Err(err) => { + // Extract relevant information from the encoded transaction data + match client.runtime_api().extract(at, &encoded) { + Ok(Some(data)) => { + for (data_hash, bytes_len, commitments, proofs) in data { tracing::debug!( target: LOG_TARGET, - "Failed to extract data from extrinsic. Transaction: {:?}. Error: {:?}", - transaction.hash(), - err, + "New blob transaction found. Hash: {:?}", data_hash, ); - }, - }; - }, - None => {}, + + let metadata = SidecarMetadata { + data_len: bytes_len, + blobs_hash: data_hash, + commitments, + proofs, + }; + + let fetch_value_from_network = |sidecar: &Sidecar| { + network.get_value(&sidecar_kademlia_key(sidecar)); + }; + + match Sidecar::from_local_outside::(&metadata.id(), &mut offchain_db) { + Some(sidecar) if sidecar.status.is_none() => { + fetch_value_from_network(&sidecar); + }, + None => { + let sidecar = Sidecar { + blobs: None, + metadata: metadata.clone(), + status: None, + }; + sidecar.save_to_local_outside::(&mut offchain_db); + fetch_value_from_network(&sidecar); + }, + _ => {}, + } + } + }, + Ok(None) => tracing::debug!( + target: LOG_TARGET, + "Decoding of extrinsic failed. Transaction: {:?}", + transaction.hash(), + ), + Err(err) => tracing::debug!( + target: LOG_TARGET, + "Failed to extract data from extrinsic. Transaction: {:?}. Error: {:?}", + transaction.hash(), + err, + ), + }; } } -} +} \ No newline at end of file diff --git a/crates/das-rpc/src/lib.rs b/crates/das-rpc/src/lib.rs index 4795fde..91333aa 100644 --- a/crates/das-rpc/src/lib.rs +++ b/crates/das-rpc/src/lib.rs @@ -20,8 +20,8 @@ use jsonrpsee::{ proc_macros::rpc, }; use melo_core_primitives::traits::AppDataApi; -use melo_core_primitives::{Sidercar, SidercarMetadata}; -use melo_das_network::kademlia_key_from_sidercar_id; +use melo_core_primitives::{Sidecar, SidecarMetadata}; +use melo_das_network::kademlia_key_from_sidecar_id; use melo_das_network_protocol::DasDht; use melodot_runtime::{RuntimeCall, UncheckedExtrinsic}; @@ -37,6 +37,8 @@ pub use sc_rpc_api::DenyUnsafe; pub use error::Error; +/// Represents the status of a Blob transaction. +/// Includes the transaction hash and potential error details. #[derive(Eq, PartialEq, Clone, Encode, Decode, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct BlobTxSatus { @@ -44,24 +46,29 @@ pub struct BlobTxSatus { pub err: Option, } +/// Defines the Das API's functionalities. #[rpc(client, server, namespace = "das")] pub trait DasApi { + /// Method for submitting blob transactions. + /// This will take care of encoding, and then submitting the data and extrinsic to the pool. #[method(name = "submitBlobTx")] async fn submit_blob_tx(&self, data: Bytes, extrinsic: Bytes) -> RpcResult>; } +/// Main structure representing the Das system. +/// Holds client connection, transaction pool, and DHT network service. pub struct Das { - /// Substrate client + /// Client interface for interacting with the blockchain. client: Arc, - /// Transactions pool + /// Pool for managing and processing transactions. pool: Arc

, - /// DHT network + /// Service for interacting with the DHT network. pub service: DDS, _marker: std::marker::PhantomData, } impl Das { - /// Creates a new instance of the Das Rpc helper. + /// Constructor: Creates a new instance of Das. pub fn new(client: Arc, pool: Arc

, service: DDS) -> Self { Self { client, pool, service, _marker: Default::default() } } @@ -78,12 +85,27 @@ where C::Api: AppDataApi, DDS: DasDht + Sync + Send + 'static + Clone, { + /// Submits a blob transaction to the transaction pool. + /// The transaction undergoes validation and then gets executed by the runtime. + /// + /// # Arguments + /// * `data` - Raw data intended for DHT network. + /// * `extrinsic` - An unsigned extrinsic to be included in the transaction pool. + /// + /// # Returns + /// A struct containing: + /// * `tx_hash` - The hash of the transaction. + /// * `err` - `Some` error string if the data submission fails. `None` if successful. + /// + /// # Note + /// Ensure proper encoding of the data. Improper encoding can result in a successful transaction submission (if it's valid), + /// but a failed data publication, rendering the data inaccessible. async fn submit_blob_tx( &self, data: Bytes, extrinsic: Bytes, ) -> RpcResult> { - // Decode the extrinsic + // Decode the provided extrinsic. let xt = Decode::decode(&mut &extrinsic[..]) .map_err(|e| Error::DecodingExtrinsicFailed(Box::new(e)))?; @@ -101,8 +123,8 @@ where .map_err(|e| Error::FetchTransactionMetadataFailed(Box::new(e)))? .ok_or(Error::InvalidTransactionFormat)?; - // Validate data_len and data_hash - if data_len != (data.len() as u32) || Sidercar::calculate_id(&data)[..] != data_hash[..] { + // Validate the length and hash of the data. + if data_len != (data.len() as u32) || Sidecar::calculate_id(&data)[..] != data_hash[..] { return Err(Error::DataLengthOrHashError.into()); } @@ -117,30 +139,30 @@ where .unwrap_or_else(|e| Error::TransactionPushFailed(Box::new(e)).into()) })?; - let metadata = SidercarMetadata { data_len, blobs_hash: data_hash, commitments, proofs }; + let metadata = SidecarMetadata { data_len, blobs_hash: data_hash, commitments, proofs }; - let mut blob_tx_status = BlobTxSatus { - tx_hash: tx_hash, - err: None, - }; + let mut blob_tx_status = BlobTxSatus { tx_hash, err: None }; match metadata.verify_bytes(&data) { Ok(true) => { - // Push data to the DHT network + // On successful data verification, push data to DHT network. let mut dht_service = self.service.clone(); let put_res = dht_service - .put_value_to_dht(kademlia_key_from_sidercar_id(&data_hash), data.to_vec()) + .put_value_to_dht(kademlia_key_from_sidecar_id(&data_hash), data.to_vec()) .await .is_some(); if !put_res { - blob_tx_status.err = - Some("Failed to put data to DHT network.".to_string()); + blob_tx_status.err = Some("Failed to put data to DHT network.".to_string()); } }, Ok(false) => { - blob_tx_status.err = Some("Data verification failed. Please check your data and try again.".to_string()); + // Handle cases where data verification failed. + blob_tx_status.err = Some( + "Data verification failed. Please check your data and try again.".to_string(), + ); }, Err(e) => { + // Handle unexpected errors during verification. blob_tx_status.err = Some(e); }, } diff --git a/crates/frame-system-ext/Cargo.toml b/crates/frame-system-ext/Cargo.toml index 85bafe8..7d46a3e 100644 --- a/crates/frame-system-ext/Cargo.toml +++ b/crates/frame-system-ext/Cargo.toml @@ -10,16 +10,17 @@ edition = "2021" targets = ["x86_64-unknown-linux-gnu"] [dependencies] +log = { version = "0.4.17", default-features = false } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.1.1", default-features = false, features = ["derive"]} +serde = { version = "1.0.136", features = ["derive"], optional = true } + melo-core-primitives = { version = "0.1.0", path = "../core-primitives", default-features = false } frame-support = { version = "4.0.0-dev", default-features = false, git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } frame-system = { version = "4.0.0-dev", default-features = false, git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } -log = { version = "0.4.17", default-features = false } sp-application-crypto = { version = "7.0.0", default-features = false, git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } sp-runtime = { version = "7.0.0", default-features = false, git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42"} -codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } -scale-info = { version = "2.1.1", default-features = false, features = ["derive"]} -serde = { version = "1.0.136", features = ["derive"], optional = true } [dev-dependencies] sp-io = { version = "7.0.0", git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } diff --git a/crates/frame-system-ext/src/lib.rs b/crates/frame-system-ext/src/lib.rs index 147cf40..0a51e3c 100644 --- a/crates/frame-system-ext/src/lib.rs +++ b/crates/frame-system-ext/src/lib.rs @@ -12,6 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! # Frame System Extension Module +//! +//! This module provides an extension mechanism for the frame system. +//! It replaces the `finalize()` method in the original `frame_system::Pallet` module, allowing +//! for the generation of new types of block headers, introducing extended fields. +//! +//! An alternative approach would be to directly modify the frame-system pallet, which would prevent +//! modifications to the frame-executive module. However, this would make the frame-system cluttered +//! and often require additional type conversions for blocks and block headers. Our modification to +//! frame-executive is clear, involving simple type adjustments and the introduction of additional +//! block headers and test tools. This ensures compatibility with the Substrate ecosystem. +//! +//! ## Overview +//! +//! The System Extension module introduces an extended block header that includes a commitment list, +//! enhancing the capabilities of the traditional block header. + #![cfg_attr(not(feature = "std"), no_std)] use frame_support::pallet_prelude::*; @@ -22,6 +39,9 @@ use sp_runtime::traits; use melo_core_primitives::traits::{ExtendedHeader, HeaderCommitList}; +// Logger target for this module. +const LOG_TARGET: &str = "runtime::system_ext"; + #[frame_support::pallet] pub mod pallet { use super::*; @@ -29,25 +49,40 @@ pub mod pallet { #[pallet::pallet] pub struct Pallet(_); - /// Configure the pallet by specifying the parameters and types on which it depends. + /// Configuration trait for this pallet. + /// + /// This trait allows the definition of the extended block header and the commit list type. #[pallet::config] pub trait Config: frame_system::Config { - /// The block header. + /// The extended block header. type ExtendedHeader: Parameter + traits::Header + ExtendedHeader; + /// The type of the commit list. type CommitList: HeaderCommitList; } } impl Pallet { - /// Remove temporary "environment" entries in storage, compute the storage root and return the - /// resulting header for this block. + /// Finalizes the block creation process. + /// + /// This function will: + /// - Remove any temporary environmental storage entries. + /// - Compute the storage root. + /// - Return the resulting extended header for the current block. + /// + /// # Returns + /// + /// - `T::ExtendedHeader`: The extended block header with a commitment list. pub fn finalize() -> T::ExtendedHeader { + // Retrieve the base header from the frame_system pallet. let header = >::finalize(); + + // Get the last commit list. let commit_list = T::CommitList::last(); + // Construct an extended header. let mut ext_header = T::ExtendedHeader::new_ext( *header.number(), *header.extrinsics_root(), @@ -57,9 +92,13 @@ impl Pallet { Default::default(), ); + // Set the commitments using the commit list. ext_header.set_commitments(&commit_list); - // log::trace!(target: LOG_TARGET, "Header {:?}", header); + // Log the base header for debugging. + log::trace!(target: LOG_TARGET, "Header {:?}", header); + + // Return the constructed extended header. ext_header } } diff --git a/crates/meloxt/examples/register_app.rs b/crates/meloxt/examples/register_app.rs index dca9b6c..a5125e1 100644 --- a/crates/meloxt/examples/register_app.rs +++ b/crates/meloxt/examples/register_app.rs @@ -25,6 +25,7 @@ pub async fn main() { error!("{}", err); } } + async fn run() -> Result<(), Box> { info!("{} register app ", START_EXAMPLE); let client = ClientBuilder::default().build().await?; diff --git a/crates/meloxt/examples/submit_blob_tx.rs b/crates/meloxt/examples/submit_blob_tx.rs index 50e7e3b..6c2f41a 100644 --- a/crates/meloxt/examples/submit_blob_tx.rs +++ b/crates/meloxt/examples/submit_blob_tx.rs @@ -18,7 +18,7 @@ use melo_das_primitives::crypto::{KZGCommitment as KZGCommitmentT, KZGProof as K use melo_das_rpc::BlobTxSatus; use meloxt::info_msg::*; use meloxt::Client; -use meloxt::{commitments_to_runtime, wait_for_block, init_logger, proofs_to_runtime, sidercar_metadata}; +use meloxt::{commitments_to_runtime, wait_for_block, init_logger, proofs_to_runtime, sidecar_metadata}; use meloxt::{melodot, ClientBuilder}; use primitive_types::H256; use subxt::rpc::rpc_params; @@ -43,7 +43,7 @@ async fn run() -> Result<(), Box> { let app_id = 1; let bytes_len = 123; // Exceeding the limit - let (commitments_t, proofs_t, data_hash, bytes) = sidercar_metadata(bytes_len); + let (commitments_t, proofs_t, data_hash, bytes) = sidecar_metadata(bytes_len); let commitments = commitments_to_runtime(commitments_t.clone()); diff --git a/crates/meloxt/examples/submit_data.rs b/crates/meloxt/examples/submit_data.rs index e42d184..e6c4df4 100644 --- a/crates/meloxt/examples/submit_data.rs +++ b/crates/meloxt/examples/submit_data.rs @@ -15,7 +15,7 @@ use log::{error, info}; use meloxt::info_msg::*; use meloxt::init_logger; -use meloxt::sidercar_metadata_runtime; +use meloxt::sidecar_metadata_runtime; use meloxt::{melodot, ClientBuilder}; use subxt_signer::sr25519::dev::{self}; @@ -35,7 +35,7 @@ async fn run() -> Result<(), Box> { let app_id = 1; let bytes_len = 121; // Exceeding the limit - let (commitments, proofs, data_hash, _) = sidercar_metadata_runtime(bytes_len); + let (commitments, proofs, data_hash, _) = sidecar_metadata_runtime(bytes_len); let submit_data_tx = melodot::tx() diff --git a/crates/meloxt/src/header.rs b/crates/meloxt/src/header.rs index 12ca570..a069b31 100644 --- a/crates/meloxt/src/header.rs +++ b/crates/meloxt/src/header.rs @@ -22,13 +22,19 @@ use melo_core_primitives::HeaderExtension; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)] #[serde(rename_all = "camelCase")] pub struct MelodotHeader + TryFrom, H: Hasher> { + /// The parent hash of this block. pub parent_hash: H::Output, + /// The block number. #[serde(serialize_with = "serialize_number", deserialize_with = "deserialize_number")] #[codec(compact)] pub number: N, + /// The state trie merkle root of this block. pub state_root: H::Output, + /// The extrinsics trie merkle root of this block. pub extrinsics_root: H::Output, + /// The digest of this block. pub digest: Digest, + /// The commitment list of this block. pub extension: HeaderExtension, } diff --git a/crates/meloxt/src/helper.rs b/crates/meloxt/src/helper.rs index da5f400..bfcec8d 100644 --- a/crates/meloxt/src/helper.rs +++ b/crates/meloxt/src/helper.rs @@ -14,24 +14,57 @@ use crate::melodot::runtime_types::melo_das_primitives::crypto::{KZGCommitment, KZGProof}; use crate::Client; -use melo_core_primitives::SidercarMetadata; +use melo_core_primitives::SidecarMetadata; use melo_das_primitives::crypto::{KZGCommitment as KZGCommitmentT, KZGProof as KZGProofT}; pub use primitive_types::H256; -pub fn sidercar_metadata_runtime( +/// Converts a given byte length to sidecar metadata suitable for runtime. +/// +/// # Arguments +/// +/// - `bytes_len`: Length of random bytes to generate. +/// +/// # Returns +/// +/// - `Vec`: Vector of KZG commitments for runtime. +/// - `Vec`: Vector of KZG proofs for runtime. +/// - `H256`: Hash value of the blobs. +/// - `Vec`: Randomly generated bytes. +pub fn sidecar_metadata_runtime( bytes_len: u32, ) -> (Vec, Vec, H256, Vec) { - let (commits, proofs, blobs_hash, bytes) = sidercar_metadata(bytes_len); + let (commits, proofs, blobs_hash, bytes) = sidecar_metadata(bytes_len); (commitments_to_runtime(commits), proofs_to_runtime(proofs), blobs_hash, bytes) } -pub fn sidercar_metadata(bytes_len: u32) -> (Vec, Vec, H256, Vec) { +/// Generates sidecar metadata based on a given byte length. +/// +/// # Arguments +/// +/// - `bytes_len`: Length of random bytes to generate. +/// +/// # Returns +/// +/// - `Vec`: Vector of KZG commitments. +/// - `Vec`: Vector of KZG proofs. +/// - `H256`: Hash value of the blobs. +/// - `Vec`: Randomly generated bytes. +pub fn sidecar_metadata(bytes_len: u32) -> (Vec, Vec, H256, Vec) { let bytes = (0..bytes_len).map(|_| rand::random::()).collect::>(); - let metadata: SidercarMetadata = SidercarMetadata::try_from_app_data(&bytes).unwrap(); + let metadata: SidecarMetadata = SidecarMetadata::try_from_app_data(&bytes).unwrap(); (metadata.commitments, metadata.proofs, metadata.blobs_hash, bytes) } +/// Converts KZG commitments to a runtime-friendly format. +/// +/// # Arguments +/// +/// - `commitments`: A vector of KZG commitments. +/// +/// # Returns +/// +/// - `Vec`: Vector of KZG commitments for runtime. pub fn commitments_to_runtime(commitments: Vec) -> Vec { commitments .iter() @@ -39,10 +72,28 @@ pub fn commitments_to_runtime(commitments: Vec) -> Vec>() } +/// Converts KZG proofs to a runtime-friendly format. +/// +/// # Arguments +/// +/// - `proofs`: A vector of KZG proofs. +/// +/// # Returns +/// +/// - `Vec`: Vector of KZG proofs for runtime. pub fn proofs_to_runtime(proofs: Vec) -> Vec { proofs.iter().map(|c| KZGProof { inner: c.to_bytes() }).collect::>() } +/// Waits for two block confirmations using a client subscription. +/// +/// # Arguments +/// +/// - `client`: A reference to the client object. +/// +/// # Returns +/// +/// - `Result<(), Box>`: A result indicating success or failure. pub async fn wait_for_block(client: &Client) -> Result<(), Box> { let mut sub = client.api.rpc().subscribe_all_block_headers().await?; sub.next().await; @@ -51,10 +102,11 @@ pub async fn wait_for_block(client: &Client) -> Result<(), Box; +// Implement the `Config` trait for `MeloConfig`, mapping Melo-specific types to the substrate types. impl Config for MeloConfig { type Hash = H256; type AccountId = AccountId; @@ -49,27 +54,32 @@ impl Config for MeloConfig { type ExtrinsicParams = ::ExtrinsicParams; } +/// Client structure containing the API for blockchain interactions and a signer for transactions. pub struct Client { pub api: OnlineClient, pub signer: Keypair, } impl Client { + /// Update the signer for the client. pub fn set_signer(&mut self, signer: Keypair) { self.signer = signer; } + /// Update the API client. pub fn set_client(&mut self, api: OnlineClient) { self.api = api; } } +/// A builder pattern for creating a `Client` instance. pub struct ClientBuilder { pub url: String, pub signer: Keypair, } impl ClientBuilder { + /// Constructor for `ClientBuilder`. pub fn new(url: &str, signer: Keypair) -> Self { Self { url: url.to_string(), @@ -77,6 +87,7 @@ impl ClientBuilder { } } + /// Asynchronously build and return a `Client` instance. pub async fn build(&self) -> Result> { let api = OnlineClient::::from_url(&self.url).await?; Ok(Client { @@ -86,6 +97,7 @@ impl ClientBuilder { } } +// Default implementation for `ClientBuilder`. impl Default for ClientBuilder { fn default() -> Self { Self { @@ -93,4 +105,4 @@ impl Default for ClientBuilder { signer: dev::alice(), } } -} \ No newline at end of file +} diff --git a/crates/meloxt/src/run_examples.rs b/crates/meloxt/src/run_examples.rs index cfca0af..c66f54e 100644 --- a/crates/meloxt/src/run_examples.rs +++ b/crates/meloxt/src/run_examples.rs @@ -16,27 +16,51 @@ use std::process::Stdio; use anyhow::{ensure, Result}; use tokio::process::Command as TokioCommand; +/// Entry point for the asynchronous main function. #[tokio::main] async fn main() -> Result<()> { + // Fetch all the available example names. let examples = fetch_all_examples().await?; + + // Iterate through each example and run it. for example in examples.iter() { println!("Running example: {}", example); run_example(example).await?; } + Ok(()) } +/// Asynchronously run a given example using cargo. +/// +/// # Arguments +/// +/// * `example` - The name of the example to run. +/// +/// # Returns +/// +/// * `Result<()>` - A result type indicating success or any potential error. +/// async fn run_example(example: &str) -> Result<()> { + // Execute the example using cargo. It assumes that the example can be run using the cargo command. let status = TokioCommand::new("cargo") .args(&["run", "--release", "--example", example]) .status() .await?; + // Check if the command was successful; if not, return an error. ensure!(status.success(), format!("Example {} failed", example)); Ok(()) } +/// Asynchronously fetch the names of all available examples. +/// +/// # Returns +/// +/// * `Result>` - A result type containing a vector of example names or any potential error. +/// async fn fetch_all_examples() -> Result> { + // Use tokio's spawn_blocking to run a blocking operation in the context of an asynchronous function. let output = tokio::task::spawn_blocking(move || { std::process::Command::new("cargo") .args(&["run", "--release", "--example"]) @@ -45,9 +69,12 @@ async fn fetch_all_examples() -> Result> { }) .await??; + // Process the output to fetch example names. + // We skip the first two lines assuming they may contain info or error messages. + // The following lines are considered to contain the names of the examples. let lines = String::from_utf8(output.stderr)? .lines() - .skip(2) // Skip the first 2 lines which might be info or error messages. + .skip(2) .map(|line| line.trim().to_string()) .filter(|line| !line.is_empty()) .collect(); diff --git a/crates/pallet-melo-store/src/lib.rs b/crates/pallet-melo-store/src/lib.rs index ef9a3cd..d4d39a1 100644 --- a/crates/pallet-melo-store/src/lib.rs +++ b/crates/pallet-melo-store/src/lib.rs @@ -46,31 +46,36 @@ use sp_runtime::{ use sp_std::prelude::*; use melo_core_primitives::traits::HeaderCommitList; -use melo_core_primitives::{Sidercar, SidercarMetadata}; +use melo_core_primitives::{Sidecar, SidecarMetadata}; use melo_das_primitives::crypto::{KZGCommitment, KZGProof}; +// A prefix constant used for the off-chain database. const DB_PREFIX: &[u8] = b"melodot/melo-store/unavailable-data-report"; -// Threshold for delayed acknowledgement of unavailability +// A threshold constant used to determine when to delay the acknowledgment of unavailability. pub const DELAY_CHECK_THRESHOLD: u32 = 1; -// Weight for each blob +// Weight constant for each blob. pub const WEIGHT_PER_BLOB: Weight = Weight::from_parts(1024, 0); +// Typedef for Authorization Index. pub type AuthIndex = u32; +// Struct to represent the report status. #[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo)] struct ReportStatus { - /// The block in which the data is reported + /// The block number when the data was reported. pub at_block: BlockNumber, + /// The block number when the report was sent. pub sent_at: BlockNumber, } impl ReportStatus { + // Check if the report is recent based on given parameters. fn is_recent(&self, at_block: BlockNumber, now: BlockNumber) -> bool { self.at_block == at_block && self.sent_at + DELAY_CHECK_THRESHOLD.into() > now } } -/// Error which may occur while executing the off-chain code. +/// Possible errors that can occur during off-chain execution. #[cfg_attr(test, derive(PartialEq))] enum OffchainErr { WaitingForInclusion(BlockNumber), @@ -92,20 +97,22 @@ impl sp_std::fmt::Debug for OffchainErr = Result>>; +// Struct to represent a report of unavailable data. #[derive(Encode, Decode, Clone, PartialEq, Eq, RuntimeDebug, TypeInfo)] pub struct UnavailableDataReport where BlockNumber: PartialEq + Eq + Decode + Encode, { - /// Block number at the time report is created.. + /// Block number at the time report is created. pub at_block: BlockNumber, - /// An index of the authority on the list of validators. + /// Index of the authority reporting the unavailability. pub authority_index: AuthIndex, - + /// Set of indexes related to the report. pub index_set: Vec, - /// The length of session validator set + /// Total length of session validator set. pub validators_len: u32, } @@ -119,26 +126,42 @@ pub mod pallet { pub type KZGCommitmentListFor = BoundedVec::MaxBlobNum>; pub type KZGProofListFor = BoundedVec::MaxBlobNum>; + /// Represents the metadata of a blob in the system. #[derive(Clone, Eq, Default, PartialEq, Encode, Decode, MaxEncodedLen, TypeInfo)] #[scale_info(skip_type_params(T))] pub struct BlobMetadata { + /// Unique identifier for the application that uses this blob. pub app_id: u32, + + /// Account ID of the entity that created or owns this blob. pub from: T::AccountId, + + /// List of KZG commitments associated with this blob. pub commitments: KZGCommitmentListFor, + + /// List of KZG proofs associated with this blob. pub proofs: KZGProofListFor, + + /// Length of the data in bytes that this metadata represents. pub bytes_len: u32, + + /// Hash of the data associated with this blob. pub data_hash: H256, + + /// Flag indicating whether the blob data is available or not. pub is_available: bool, } - /// Configure the pallet by specifying the parameters and types on which it depends. + /// Provides configuration parameters for the pallet. #[pallet::config] pub trait Config: SendTransactionTypes> + frame_system::Config { - /// Because this pallet emits events, it depends on the runtime's definition of an event. + /// This type represents an event in the runtime, which includes events emitted by this pallet. type RuntimeEvent: From> + IsType<::RuntimeEvent>; - /// Type representing the weight of this pallet + + /// This type represents the computation cost of the pallet's operations. type WeightInfo: WeightInfo; - /// The identifier type for an authority. + + /// This type defines the unique identifier for an authority or a trusted node in the network. type AuthorityId: Member + Parameter + RuntimeAppPublic @@ -146,19 +169,20 @@ pub mod pallet { + MaybeSerializeDeserialize + MaxEncodedLen; - /// The maximum number of keys that can be added. + /// Defines the upper limit for the number of keys that can be stored. type MaxKeys: Get; + /// The maximum number of blobs that can be handled. #[pallet::constant] type MaxBlobNum: Get; + /// This defines the priority for unsigned transactions in the Melo context. #[pallet::constant] type MeloUnsignedPriority: Get; } - // Store metadata of AppData - // TODO: We currently don't delete it because, when farmers submit solutions later, if the - // data is not deleted, it will first validate the Root and then delete the data. + /// Represents metadata associated with the AppData. It's preserved for future verification. + /// Deleting data after a certain point may be beneficial for storage and computational efficiency. #[pallet::storage] #[pallet::getter(fn metadata)] pub(super) type Metadata = StorageMap< @@ -169,15 +193,18 @@ pub mod pallet { ValueQuery, >; + /// Contains the keys for this pallet's use. #[pallet::storage] #[pallet::getter(fn keys)] pub(super) type Keys = StorageValue<_, WeakBoundedVec, ValueQuery>; + /// Holds the unique identifier for the application using this pallet. #[pallet::storage] #[pallet::getter(fn app_id)] pub(super) type AppId = StorageValue<_, u32, ValueQuery>; + /// Represents votes regarding the availability of certain data. #[pallet::storage] #[pallet::getter(fn unavailable_vote)] pub(super) type UnavailableVote = StorageDoubleMap< @@ -189,76 +216,62 @@ pub mod pallet { WeakBoundedVec, >; + /// Enumerates all the possible events that can be emitted by this pallet. #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// Data has been received + /// Indicates that data was successfully received. DataReceived { - /// Hash of the data data_hash: H256, - /// Length of the data bytes_len: u32, - /// Submitter of the data from: T::AccountId, - /// App ID of the data app_id: u32, - /// Index of the data index: u32, - /// Commitments commitments: Vec, - /// Proofs proofs: Vec, }, - - /// Report has been received - ReportReceived { - /// Block number of the report - at_block: BlockNumberFor, - /// Node that submitted the report - from: AuthIndex, - }, - /// New app ID has been registered - AppIdRegistered { - /// New app ID - app_id: u32, - /// From - from: T::AccountId, - }, + /// Signifies that a report has been submitted. + ReportReceived { at_block: BlockNumberFor, from: AuthIndex }, + /// Denotes the successful registration of a new application ID. + AppIdRegistered { app_id: u32, from: T::AccountId }, } - // Errors inform users that something went wrong. + /// Enumerates all possible errors that might occur while using this pallet. #[pallet::error] pub enum Error { - /// Exceeds the maximum limit for the number of Blobs. + /// The system has reached its limit for the number of Blobs. ExceedMaxBlobLimit, - /// There's an error with the app ID. + /// Something's wrong with the given app ID. AppIdError, - /// Exceeds the maximum number of Blobs for a single block. + /// Too many Blobs were added in a single block. ExceedMaxBlobPerBlock, - /// Exceeds the confirmation time for unavailable data. + /// The time for confirming unavailable data has passed. ExceedUnavailableDataConfirmTime, - /// The index set is empty. + /// No indices have been set. IndexSetIsEmpty, - /// The data doesn't exist. + /// The requested data doesn't exist. DataNotExist, - /// The report has been submitted more than once. + /// The same report was submitted more than once. DuplicateReportSubmission, - /// Exceeds the maximum total votes. + /// The total votes have exceeded the allowed maximum. ExceedMaxTotalVotes, - /// The report is for a block in the future. + /// A report was made for a block that hasn't occurred yet. ReportForFutureBlock, - /// The submitted availability data is empty. + /// No data was provided in the submission. SubmittedDataIsEmpty, - /// The number of commitments does not match the expected blob number. + /// The number of provided commitments doesn't match the expected number. MismatchedCommitmentsCount, - /// The number of proofs does not match the expected blob number. + /// The number of provided proofs doesn't match the expected number. MismatchedProofsCount, - /// Non existent public key. + /// The provided public key is not valid. InvalidKey, } #[pallet::call] impl Pallet { + /// Submit data for a particular app. + /// This call allows a user to submit data, its commitments, and proofs. + /// The function ensures various constraints like the length of the data, validity of the app id, and other integrity checks. #[pallet::call_index(0)] #[pallet::weight( WEIGHT_PER_BLOB @@ -332,6 +345,10 @@ pub mod pallet { Ok(()) } + /// Report on the unavailability of certain data. + /// Validators can use this function to report any data that they find unavailable. + /// The function does checks like making sure the data isn't being reported for a future block, + /// the report is within the acceptable delay, and that the reporting key is valid. #[pallet::call_index(1)] #[pallet::weight(::WeightInfo::validate_unsigned_and_then_report( unavailable_data_report.validators_len, @@ -355,7 +372,7 @@ pub mod pallet { >= current_block_number, Error::::ExceedUnavailableDataConfirmTime ); - + ensure!(!unavailable_data_report.index_set.is_empty(), Error::::IndexSetIsEmpty); let keys = Keys::::get(); @@ -390,6 +407,8 @@ pub mod pallet { Ok(()) } + /// Register a new app with the system. + /// This function allows a user to register a new app and increments the app ID. #[pallet::call_index(2)] #[pallet::weight(::WeightInfo::register_app())] pub fn register_app(origin: OriginFor) -> DispatchResult { @@ -480,20 +499,24 @@ pub mod pallet { } impl Pallet { + /// Retrieve the list of indexes representing data unavailability at a given block. + /// + /// # Arguments + /// * `at_block` - The block number to check for data unavailability. pub fn get_unavailability_data(at_block: BlockNumberFor) -> Vec { Metadata::::get(at_block) .iter() .enumerate() .filter_map(|(i, metadata)| { - let sidercar_metadata = SidercarMetadata { + let sidecar_metadata = SidecarMetadata { commitments: metadata.commitments.to_vec(), data_len: metadata.bytes_len, blobs_hash: metadata.data_hash, proofs: metadata.proofs.to_vec(), }; - let id = sidercar_metadata.id(); - if let Some(sidercar) = Sidercar::from_local(&id) { - if sidercar.is_unavailability() { + let id = sidecar_metadata.id(); + if let Some(sidecar) = Sidecar::from_local(&id) { + if sidecar.is_unavailability() { Some(i as u32) } else { None @@ -505,6 +528,10 @@ impl Pallet { .collect::>() } + /// Fetch the list of commitments at a given block. + /// + /// # Arguments + /// * `at_block` - The block number to fetch commitments from. pub fn get_commitment_list(at_block: BlockNumberFor) -> Vec { Metadata::::get(at_block) .iter() @@ -513,6 +540,10 @@ impl Pallet { .collect::>() } + /// Assemble and send unavailability reports for any data that is unavailable. + /// + /// # Arguments + /// * `now` - The current block number. pub(crate) fn send_unavailability_report( now: BlockNumberFor, ) -> OffchainResult>> { @@ -543,6 +574,7 @@ impl Pallet { Ok(reports) } + // Helper method to send a single unavailability report. fn send_single_unavailability_report( authority_index: u32, key: T::AuthorityId, @@ -579,46 +611,42 @@ impl Pallet { }) } + // Locking mechanism to prevent double reporting by the same authority. fn with_report_lock( authority_index: u32, at_block: BlockNumberFor, now: BlockNumberFor, f: impl FnOnce() -> OffchainResult, ) -> OffchainResult { - let key = { - let mut key = DB_PREFIX.to_vec(); - key.extend(authority_index.encode()); - key - }; + let mut key = DB_PREFIX.to_vec(); + key.extend(authority_index.encode()); + let storage = StorageValueRef::persistent(&key); - let res = storage.mutate( + + match storage.mutate( |status: Result>>, StorageRetrievalError>| { - match status { - // we are still waiting for inclusion. - Ok(Some(status)) if status.is_recent(at_block, now) => { - Err(OffchainErr::WaitingForInclusion(status.sent_at)) - }, - // attempt to set new status - _ => Ok(ReportStatus { at_block, sent_at: now }), + if let Ok(Some(status)) = status { + if status.is_recent(at_block, now) { + return Err(OffchainErr::WaitingForInclusion(status.sent_at)); + } } + Ok(ReportStatus { at_block, sent_at: now }) }, - ); - if let Err(MutateStorageError::ValueFunctionFailed(err)) = res { - return Err(err); - } - - let mut new_status = res.map_err(|_| OffchainErr::FailedToAcquireLock)?; - - let res = f(); - - if res.is_err() { - new_status.sent_at = 0u32.into(); - storage.set(&new_status); + ) { + Err(MutateStorageError::ValueFunctionFailed(err)) => Err(err), + res => { + let mut new_status = res.map_err(|_| OffchainErr::FailedToAcquireLock)?; + let result = f(); + if result.is_err() { + new_status.sent_at = 0u32.into(); + storage.set(&new_status); + } + result + } } - - res } + // Fetch all local authority keys. fn local_authority_keys() -> impl Iterator { let authorities = Keys::::get(); @@ -634,6 +662,7 @@ impl Pallet { }) } + // Handle an unavailability vote for a particular piece of data. fn handle_vote( at_block: BlockNumberFor, authority_index: AuthIndex, @@ -679,6 +708,7 @@ impl Pallet { }) } + // Initialize the authority keys. fn initialize_keys(keys: &[T::AuthorityId]) { if !keys.is_empty() { assert!(Keys::::get().is_empty(), "Keys are already initialized!"); @@ -688,6 +718,7 @@ impl Pallet { } } + // Set the authority keys (used for testing purposes). #[cfg(test)] fn set_keys(keys: Vec) { let bounded_keys = WeakBoundedVec::<_, T::MaxKeys>::try_from(keys) diff --git a/crates/pallet-melo-store/src/tests.rs b/crates/pallet-melo-store/src/tests.rs index 573a90b..573c6f7 100644 --- a/crates/pallet-melo-store/src/tests.rs +++ b/crates/pallet-melo-store/src/tests.rs @@ -18,7 +18,7 @@ use super::*; use crate as pallet_melo_store; use crate::mock::*; use frame_support::{assert_noop, assert_ok}; -use melo_core_primitives::SidercarMetadata; +use melo_core_primitives::SidecarMetadata; use sp_core::{ offchain::{ testing::{TestOffchainExt, TestTransactionPoolExt}, @@ -632,13 +632,13 @@ fn should_send_unavailability_report_correctly() { commitments.clone(), proofs.clone(), )); - let sidercar_metadata = - SidercarMetadata { data_len: bytes_len, blobs_hash: data_hash, commitments, proofs }; + let sidecar_metadata = + SidecarMetadata { data_len: bytes_len, blobs_hash: data_hash, commitments, proofs }; - let mut sidercar = Sidercar::new(sidercar_metadata, None); - sidercar.set_not_found(); - sidercar.save_to_local(); - assert!(sidercar.is_unavailability()); + let mut sidecar = Sidecar::new(sidecar_metadata, None); + sidecar.set_not_found(); + sidecar.save_to_local(); + assert!(sidecar.is_unavailability()); // Test get_unavailability_data let unavailability_data = MeloStore::get_unavailability_data(now);