diff --git a/Cargo.lock b/Cargo.lock index ce21c42..a7a996e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4548,7 +4548,6 @@ version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" dependencies = [ - "cc", "pkg-config", "vcpkg", ] @@ -4744,6 +4743,7 @@ version = "0.1.0" dependencies = [ "lazy_static", "log", + "melo-das-db", "melo-das-primitives", "melo-erasure-coding", "parity-scale-codec", diff --git a/crates/core-primitives/Cargo.toml b/crates/core-primitives/Cargo.toml index e678d27..165e7b6 100644 --- a/crates/core-primitives/Cargo.toml +++ b/crates/core-primitives/Cargo.toml @@ -12,12 +12,14 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] melo-das-primitives = { version = "0.1.0", path = "../das-primitives", default-features = false} melo-erasure-coding = { version = "0.1.0", path = "../melo-erasure-coding", default-features = false} +melo-das-db = { version = "0.0.1", path = "../das-db", default-features = false} rayon = "1.5.1" codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false, features = ["derive"] } log = { version = "0.4.17", default-features = false } scale-info = { version = "2.1.1", default-features = false, features = ["derive"] } serde = { version = "1.0.136", optional = true, features = ["alloc", "derive"] } +rand = { version = "0.8", default-features = false, features = ["alloc"] } sp-core = { version = "7.0.0", default-features = false, git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } sp-std = { version = "5.0.0", default-features = false, git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } @@ -31,11 +33,10 @@ sc-offchain = {optional = true, default-features = false, git = "https://github. # For testing lazy_static = "1.4" -rand = { version = "0.8.5", optional = true } sp-application-crypto = { default-features = false, git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } [dev-dependencies] -rand = "0.8.5" +rand = "0.8" serde_json = "1.0.85" zstd = { version = "0.12.3", default-features = false } sp-state-machine = { git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } @@ -47,9 +48,9 @@ default = ["std", "outside"] std = [ "codec/std", "log/std", - "rand", "scale-info/std", "serde", + "rand/std", "sp-core/std", "sp-runtime/std", "melo-das-primitives/std", @@ -62,11 +63,13 @@ std = [ "sp-application-crypto/std", "sc-client-api", "sc-offchain", + "melo-das-db/std", ] outside = [ "melo-das-primitives/serde", "sc-client-api", "sc-offchain", + # "melo-das-db/outside", ] parallel = [ "melo-das-primitives/parallel", diff --git a/crates/core-primitives/src/confidence.rs b/crates/core-primitives/src/confidence.rs new file mode 100644 index 0000000..017505e --- /dev/null +++ b/crates/core-primitives/src/confidence.rs @@ -0,0 +1,165 @@ +// Copyright 2023 ZeroDAO +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +extern crate alloc; +use alloc::vec::Vec; +use codec::{Decode, Encode}; +use rand::Rng; + +use melo_das_db::traits::DasKv; +use melo_das_primitives::{config::FIELD_ELEMENTS_PER_BLOB, Position}; + +const CHUNK_COUNT: usize = 2 ^ 4; +const SAMPLES_PER_BLOB: usize = FIELD_ELEMENTS_PER_BLOB / CHUNK_COUNT; + +/// Confidence trait defines methods related to the confidence of an application. +pub trait Confidence { + /// Returns the confidence value. + fn value(&self, base_factor: f64) -> f32; + + /// Constructs a unique ID. + fn id(&self) -> Vec; + + /// Fetches `n` random sample positions from the `SAMPLES_PER_BLOB * total_rows` matrix. + fn set_sample(&mut self, n: usize); + + /// Checks if the availability exceeds the provided threshold. + fn exceeds_threshold(&self, base_factor: f64, threshold: f32) -> bool; + + /// Saves the current instance to the database. + fn save(&self, db: &mut impl DasKv); + + /// Retrieves an instance from the database. + fn get(id: &[u8], db: &mut impl DasKv) -> Option + where + Self: Sized; + + /// Removes the current instance from the database. + fn remove(&self, db: &mut impl DasKv); +} + +#[derive(Debug, Clone, Default, Decode, Encode)] +pub struct Sample { + position: Position, + is_availability: bool, +} + +impl Sample { + pub fn set_success(&mut self) { + self.is_availability = true; + } +} + +pub const AVAILABILITY_THRESHOLD: f32 = 0.8; + +#[derive(Debug, Clone, Decode, Encode, Default)] +pub struct ConfidenceBase { + id: Vec, + total_rows: u32, + samples: Vec, +} + +impl Confidence for ConfidenceBase { + fn value(&self, base_factor: f64) -> f32 { + let success_count = self.samples.iter().filter(|&sample| sample.is_availability).count(); + calculate_confidence(success_count as u32, base_factor) as f32 + } + + fn id(&self) -> Vec { + self.id.clone() + } + + fn set_sample(&mut self, n: usize) { + let mut rng = rand::thread_rng(); + let mut positions = Vec::with_capacity(n); + + while positions.len() < n { + let x = rng.gen_range(0..SAMPLES_PER_BLOB) as u32; + let y = rng.gen_range(0..self.total_rows); + + let pos = Position { x, y }; + + if !positions.contains(&pos) { + positions.push(pos); + } + } + + self.samples = positions + .into_iter() + .map(|pos| Sample { position: pos, is_availability: false }) + .collect(); + } + + fn exceeds_threshold(&self, base_factor: f64, threshold: f32) -> bool { + self.value(base_factor) > threshold + } + + fn save(&self, db: &mut impl DasKv) { + db.set(&self.id(), &self.encode()); + } + + fn get(id: &[u8], db: &mut impl DasKv) -> Option + where + Self: Sized, + { + db.get(id).and_then(|encoded_data| Decode::decode(&mut &encoded_data[..]).ok()) + } + + fn remove(&self, db: &mut impl DasKv) { + db.remove(&self.id()); + } +} + +fn calculate_confidence(samples: u32, base_factor: f64) -> f64 { + 100f64 * (1f64 - base_factor.powi(samples as i32)) +} + +pub mod app_confidence { + use super::*; + + pub const BASE_FACTOR: f64 = 0.5; + + pub fn id(block_num: BlockNum, app_id: Vec) -> Vec { + let mut id = app_id.clone(); + id.extend_from_slice(&block_num.encode()); + id + } + + pub fn new_confidence( + block_num: BlockNum, + app_id: Vec, + total_rows: u32, + ) -> ConfidenceBase { + let id = id(block_num.clone(), app_id.clone()); + ConfidenceBase { id, total_rows, samples: Vec::new() } + } +} + +pub mod block_confidence { + use super::*; + + pub const BASE_FACTOR: f64 = 0.25; + + pub fn id(block_hash: Vec) -> Vec { + block_hash + } + + pub fn new_confidence( + block_hash: Vec, + total_rows: u32, + ) -> ConfidenceBase { + let id = id(block_hash); + ConfidenceBase { id, total_rows, samples: Vec::new() } + } +} \ No newline at end of file diff --git a/crates/core-primitives/src/lib.rs b/crates/core-primitives/src/lib.rs index bc15dad..51fc6df 100644 --- a/crates/core-primitives/src/lib.rs +++ b/crates/core-primitives/src/lib.rs @@ -30,7 +30,7 @@ pub mod sidecar; pub use sidecar::*; pub mod localstorage; - +pub mod confidence; pub mod traits; #[cfg(feature = "std")] diff --git a/crates/das-db/Cargo.toml b/crates/das-db/Cargo.toml index de890f9..5cd2eb9 100644 --- a/crates/das-db/Cargo.toml +++ b/crates/das-db/Cargo.toml @@ -6,19 +6,23 @@ license = "Apache-2.0" authors = ["DKLee "] edition = "2021" +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + [dependencies] -sp-core = { git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42"} -sp-io = { git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42"} -sc-client-api = { git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42"} -sc-offchain = { git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42"} -sp-runtime = { git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42"} +sp-core = { git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42", default-features = false} +sp-io = { git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42", default-features = false} +sp-runtime = { git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42", default-features = false} + +sc-client-api = { git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42", optional = true} +sc-offchain = { git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42", optional = true} -rusqlite = { version = "0.28.0", optional = true, features = ["bundled"] } +rusqlite = { version = "0.28.0", optional = true } [dev-dependencies] [features] -default = ["std","outside"] +default = ["std"] std = [ "sp-core/std", "sp-io/std", @@ -26,8 +30,6 @@ std = [ "rusqlite", ] outside = [ - "sp-core/std", - "sp-io/std", - "sp-runtime/std", - "rusqlite", + "sc-client-api", + "sc-offchain", ] \ No newline at end of file diff --git a/crates/das-db/src/lib.rs b/crates/das-db/src/lib.rs index 3bab60f..4d69875 100644 --- a/crates/das-db/src/lib.rs +++ b/crates/das-db/src/lib.rs @@ -13,6 +13,12 @@ // limitations under the License. #![cfg_attr(not(feature = "std"), no_std)] +extern crate alloc; +pub use alloc::{ + vec::Vec, + vec, +}; + pub mod traits; #[cfg(feature = "std")] pub mod sqlite; diff --git a/crates/das-db/src/offchain.rs b/crates/das-db/src/offchain.rs index cb4cda8..7950801 100644 --- a/crates/das-db/src/offchain.rs +++ b/crates/das-db/src/offchain.rs @@ -13,6 +13,7 @@ // limitations under the License. use crate::traits::DasKv; +use crate::Vec; use sp_core::offchain::StorageKind; diff --git a/crates/das-db/src/traits.rs b/crates/das-db/src/traits.rs index 6dbcb67..dc0b48a 100644 --- a/crates/das-db/src/traits.rs +++ b/crates/das-db/src/traits.rs @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +use crate::Vec; /// `DasKv` is a trait representing a key-value store interface. pub trait DasKv { diff --git a/crates/das-primitives/src/crypto/mod.rs b/crates/das-primitives/src/crypto/mod.rs index 68fcc1d..c414dc8 100644 --- a/crates/das-primitives/src/crypto/mod.rs +++ b/crates/das-primitives/src/crypto/mod.rs @@ -639,7 +639,7 @@ impl KZG { } } -#[derive(Debug, Default, Clone, PartialEq, Eq, From)] +#[derive(Debug, Default, Clone, PartialEq, Eq, From, Decode, Encode)] pub struct Position { pub x: u32, pub y: u32, diff --git a/runtime/build.rs b/runtime/build.rs index c03d618..af6a69c 100644 --- a/runtime/build.rs +++ b/runtime/build.rs @@ -1,10 +1,10 @@ fn main() { #[cfg(feature = "std")] { - substrate_wasm_builder::WasmBuilder::new() - .with_current_project() - .export_heap_base() - .import_memory() - .build(); + // substrate_wasm_builder::WasmBuilder::new() + // .with_current_project() + // .export_heap_base() + // .import_memory() + // .build(); } }