diff --git a/Cargo.toml b/Cargo.toml index 8da86a90f..e6efc60fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,7 +71,6 @@ dyn-clone = "1.0" uuid = { version = "1", features = ["serde", "v4"] } derive_more = "0.99" paste = "1.0" -sata = { git = "https://github.com/Satellite-im/Sata", rev = "9b5a5d441c816968bc75ea8b99335f728be259ef" } tracing = { version = "0.1" } either = "1" void = "1" diff --git a/extensions/warp-ipfs/Cargo.toml b/extensions/warp-ipfs/Cargo.toml index 6b13a241d..c825e8d9b 100644 --- a/extensions/warp-ipfs/Cargo.toml +++ b/extensions/warp-ipfs/Cargo.toml @@ -22,10 +22,10 @@ anyhow.workspace = true serde.workspace = true serde_json.workspace = true either = { workspace = true, features = ["serde"] } -bs58 = "0.4" +bs58.workspace = true parking_lot.workspace = true -tracing = "0.1" +tracing.workspace = true async-recursion = "1" @@ -38,7 +38,6 @@ derive_more.workspace = true mediatype.workspace = true bincode.workspace = true -bayespam = "1.1.0" bytes.workspace = true shuttle = { path = "../../tools/shuttle", default-features = false } diff --git a/extensions/warp-pd-flatfile/Cargo.toml b/extensions/warp-pd-flatfile/Cargo.toml deleted file mode 100644 index 1b4730b8b..000000000 --- a/extensions/warp-pd-flatfile/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "warp-pd-flatfile" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib", "rlib", "staticlib"] - -[dependencies] -warp = { path = "../../warp" } - -serde = { version = "1.0", features = ["derive"] } -serde_json = { version = "1.0" } -anyhow = "1.0" -uuid = { version = "1.0", features = ["serde", "v4"] } -parking_lot.workspace = true - -[features] -default = [] -build-header = [] - -[build-dependencies] -cbindgen = "0.23" diff --git a/extensions/warp-pd-flatfile/examples/flatfile-cache.rs b/extensions/warp-pd-flatfile/examples/flatfile-cache.rs deleted file mode 100644 index dff531fcb..000000000 --- a/extensions/warp-pd-flatfile/examples/flatfile-cache.rs +++ /dev/null @@ -1,128 +0,0 @@ -use std::path::PathBuf; -use warp::data::DataType; -use warp::error::Error; -use warp::module::Module; -use warp::pocket_dimension::query::{Comparator, QueryBuilder}; -use warp::pocket_dimension::{DimensionData, PocketDimension}; -use warp::sata::Sata; -use warp_pd_flatfile::FlatfileStorage; - -fn main() -> anyhow::Result<()> { - let mut root = std::env::temp_dir(); - root.push("pd-cache"); - - let index = { - let mut index = PathBuf::new(); - index.push("cache-index"); - - index - }; - - let mut storage = FlatfileStorage::new_with_index_file(root, index)?; - let data = Sata::default().encode( - warp::libipld::IpldCodec::DagCbor, - warp::sata::Kind::Reference, - DimensionData::from("Cargo.toml"), - )?; - - let data_account = Sata::default().encode( - warp::libipld::IpldCodec::DagJson, - warp::sata::Kind::Reference, - b"Hi Account", - ); - let data_messaging = Sata::default().encode( - warp::libipld::IpldCodec::DagJson, - warp::sata::Kind::Reference, - b"Hi Messaging", - ); - let data_http = Sata::default().encode( - warp::libipld::IpldCodec::DagJson, - warp::sata::Kind::Reference, - b"Hi Http", - ); - let data_export = Sata::default().encode( - warp::libipld::IpldCodec::DagJson, - warp::sata::Kind::Reference, - b"Hi Data Export", - ); - - storage.add_data(DataType::from(Module::FileSystem), &data)?; - storage.add_data(DataType::from(Module::Accounts), &data_account.unwrap())?; - storage.add_data(DataType::from(Module::Messaging), &data_messaging.unwrap())?; - storage.add_data(DataType::from("http"), &data_http.unwrap())?; - storage.add_data(DataType::from("data_export"), &data_export.unwrap())?; - - let bufdata = Sata::default().encode( - warp::libipld::IpldCodec::DagCbor, - warp::sata::Kind::Reference, - DimensionData::from_buffer_nofile("testbin", b"Hello, World"), - )?; - - storage.add_data(DataType::from(Module::FileSystem), &bufdata)?; - - let bufdata = Sata::default().encode( - warp::libipld::IpldCodec::DagCbor, - warp::sata::Kind::Reference, - DimensionData::from_buffer_nofile("test", b"Hello, World"), - )?; - - storage.add_data(DataType::FileSystem, &bufdata)?; - - let mut query = QueryBuilder::default(); - query.filter(Comparator::Eq, "name", "testbin")?; - - let arr = storage - .get_data(DataType::from(Module::FileSystem), Some(&query))? - .last() - .ok_or(Error::InvalidDataType)? - .decode::()?; - - let arr_account = storage - .get_data(DataType::Accounts, None)? - .last() - .ok_or(Error::InvalidDataType) - .unwrap() - .decode::>() - .unwrap(); - - let arr_messaging = storage - .get_data(DataType::Messaging, None)? - .last() - .ok_or(Error::InvalidDataType) - .unwrap() - .decode::>() - .unwrap(); - - let arr_http = storage - .get_data(DataType::Http, None)? - .last() - .ok_or(Error::InvalidDataType) - .unwrap() - .decode::>() - .unwrap(); - - let arr_data_export = storage - .get_data(DataType::DataExport, None)? - .last() - .ok_or(Error::InvalidDataType) - .unwrap() - .decode::>() - .unwrap(); - - let mut buf: Vec = vec![]; - - arr.write_from_path(&mut buf)?; - - println!("Contents: {}", String::from_utf8_lossy(&buf)); - println!("Contents: {}", String::from_utf8_lossy(&arr_account)); - println!("Contents: {}", String::from_utf8_lossy(&arr_messaging)); - println!("Contents: {}", String::from_utf8_lossy(&arr_http)); - println!("Contents: {}", String::from_utf8_lossy(&arr_data_export)); - - storage.empty(DataType::from(Module::FileSystem))?; - storage.empty(DataType::from(Module::Accounts))?; - storage.empty(DataType::from(Module::Messaging))?; - storage.empty(DataType::from("data_export"))?; - storage.empty(DataType::from("http"))?; - Ok(()) -} diff --git a/extensions/warp-pd-flatfile/src/config.rs b/extensions/warp-pd-flatfile/src/config.rs deleted file mode 100644 index b0ba409f9..000000000 --- a/extensions/warp-pd-flatfile/src/config.rs +++ /dev/null @@ -1,21 +0,0 @@ -use std::path::PathBuf; - -use serde::{Deserialize, Serialize}; - -#[derive(Default, Debug, Serialize, Deserialize)] -pub struct Config { - pub directory: PathBuf, - pub prefix: Option, - pub use_index_file: bool, - pub index_file: Option, -} - -impl Config { - pub fn development() -> Config { - Config { - directory: std::env::temp_dir(), - index_file: Some("cache-index".into()), - ..Default::default() - } - } -} diff --git a/extensions/warp-pd-flatfile/src/lib.rs b/extensions/warp-pd-flatfile/src/lib.rs deleted file mode 100644 index 22f9c90c2..000000000 --- a/extensions/warp-pd-flatfile/src/lib.rs +++ /dev/null @@ -1,734 +0,0 @@ -pub mod config; - -use anyhow::anyhow; -use std::{collections::HashMap, fs::create_dir_all}; -#[allow(unused_imports)] -use std::{ - fs::OpenOptions, - io::{ErrorKind, Read, Write}, - path::{Path, PathBuf}, - sync::Arc, -}; -use uuid::Uuid; -use warp::{error::Error, libipld::Ipld, SingleHandle}; -use warp::{ - libipld::Cid, - sata::{Sata, State}, - Extension, -}; - -use parking_lot::RwLock; - -use warp::data::DataType; -use warp::module::Module; -use warp::pocket_dimension::{ - query::{ComparatorFilter, QueryBuilder}, - DimensionData, PocketDimension, -}; - -type Result = std::result::Result; - -#[derive(Default, Debug, Clone)] -pub struct FlatfileStorage { - /// Path to caching directory - pub directory: PathBuf, - - /// Prefix for files for this instance - pub prefix: Option, - - /// Flag to enable the use of index files otherwise store - pub use_index_file: bool, - - /// Index file which stores an array of data object - pub index_file: Option, - - /// Handle for index - index: FlatfileIndex, -} - -impl From for FlatfileStorage { - fn from(config: config::Config) -> Self { - let config::Config { - directory, - prefix, - use_index_file, - index_file, - } = config; - - FlatfileStorage { - directory, - prefix, - use_index_file, - index_file, - ..Default::default() - } - } -} - -impl Extension for FlatfileStorage { - fn id(&self) -> String { - String::from("warp-pd-flatfile") - } - - fn name(&self) -> String { - String::from("Flatfile Cache Storage") - } - - fn module(&self) -> Module { - Module::Cache - } -} - -#[derive(Default, Debug, Clone)] -pub struct FlatfileIndex(pub Arc>>>); - -impl core::ops::Deref for FlatfileIndex { - type Target = Arc>>>; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl FlatfileIndex { - pub fn from_path>(path: P) -> Result { - let path = path.as_ref(); - let mut index = Self::default(); - - if path.is_dir() { - index.build_index()?; - } else { - let file = path.to_string_lossy().to_string(); - index.build_index_from_file(file)?; - } - - Ok(index) - } - - pub fn build_index_from_file>(&mut self, file: P) -> Result<()> { - let file = std::fs::File::open(file)?; - self.0 = serde_json::from_reader(file)?; - Ok(()) - } - - pub fn export_index_to_file>(&self, path: P) -> Result<()> { - let path = path.as_ref(); - if path.is_file() { - std::fs::copy(path, format!("{}_backup", path.display()))?; - std::fs::remove_file(path)?; - } - - let mut fs = std::fs::File::create(path)?; - let data = serde_json::to_string(&self.0)?; - fs.write_all(data.as_bytes())?; - if std::fs::remove_file(format!("{}_backup", path.display())).is_err() { - // - } - Ok(()) - } - - pub fn export_index_to_multifile>(&self, _: P) -> Result<()> { - Err(Error::Unimplemented) - } - - pub fn insert(&mut self, data_type: DataType, data: Sata) -> Result<()> { - let mut index = self.write(); - - if let Some(list) = index.get(&data_type) { - if list.contains(&data) { - return Err(Error::DataObjectExist); - } - } - index.entry(data_type).or_default().push(data); - Ok(()) - } - - pub fn remove_by_id(&mut self, data_type: DataType, id: Cid) -> Result { - let mut index = self.write(); - - if let Some(list) = index.get_mut(&data_type) { - let index = match list.iter().position(|data| data.id() == id) { - Some(index) => index, - None => return Err(Error::DataObjectNotFound), - }; - - let item = list.remove(index); - return Ok(item); - } - Err(Error::DataObjectNotFound) - } - - pub fn build_index(&mut self) -> Result<()> { - Ok(()) - } -} - -impl> From

for FlatfileStorage { - fn from(path: P) -> Self { - //TODO: Possibly perform a check to assure that the path is actually a directory - let directory = path.as_ref().to_path_buf(); - FlatfileStorage { - directory, - ..Default::default() - } - } -} - -impl FlatfileStorage { - pub fn from_config(config: config::Config) -> Result { - let mut storage = Self::from(config); - storage.initialize_internal()?; - Ok(storage) - } - - pub fn new>(path: P) -> Self { - Self::from(path) - } - - pub fn new_with_index_file>(path: P, file: P) -> Result { - let mut storage = Self::from(path); - - if !storage.is_valid() { - storage.create_directory(true)?; - } - - let file = file.as_ref(); - - storage.index_file = Some(file.to_string_lossy().to_string()); - storage.use_index_file = true; - - let mut cache_file_index = storage.directory.clone(); - cache_file_index.push(file); - - let mut index = FlatfileIndex::default(); - if !cache_file_index.is_file() { - index.export_index_to_file(&cache_file_index)?; - } - index.build_index_from_file(&cache_file_index)?; - - storage.index = index; - - Ok(storage) - } - - pub fn initialize_internal(&mut self) -> Result<()> { - let index_file = self.index_file.clone().ok_or(Error::InvalidFile)?; - self.initialize_index_file(index_file) - } - - pub fn initialize_index_file>(&mut self, file: P) -> Result<()> { - if !self.is_valid() { - self.create_directory(true)?; - } - - let file = file.as_ref(); - - self.index_file = Some(file.to_string_lossy().to_string()); - self.use_index_file = true; - - let mut cache_file_index = self.directory.clone(); - cache_file_index.push(file); - - let mut index = FlatfileIndex::default(); - if !cache_file_index.is_file() { - index.export_index_to_file(&cache_file_index)?; - } - index.build_index_from_file(&cache_file_index)?; - - self.index = index; - - Ok(()) - } - - pub fn create_directory(&self, all: bool) -> Result<()> { - match all { - true => std::fs::create_dir_all(&self.directory)?, - false => std::fs::create_dir(&self.directory)?, - } - Ok(()) - } - - pub fn is_valid(&self) -> bool { - self.directory.is_dir() - } - - pub fn use_index_file(&self) -> bool { - self.use_index_file - } - - //TODO: Set prefix onto files - pub fn set_prefix>(&mut self, prefix: S) { - self.prefix = Some(prefix.as_ref().to_string()); - } - - pub fn get_index_ref(&self) -> &FlatfileIndex { - &self.index - } - - pub fn get_index_mut(&mut self) -> &mut FlatfileIndex { - &mut self.index - } - - pub fn get_index_file(&self) -> Result { - if !self.use_index_file() { - return Err(Error::Any(anyhow!( - "'use_index_file' must be true to use this function" - ))); - } - - let mut path = self.directory.clone(); - path.push(self.index_file.as_ref().ok_or(Error::FileNotFound)?); - - Ok(path.to_string_lossy().to_string()) - } - - pub fn sync(&self) -> Result<()> { - let index_file = self.get_index_file()?; - - self.index.export_index_to_file(index_file)?; - - Ok(()) - } -} - -impl SingleHandle for FlatfileStorage {} - -impl PocketDimension for FlatfileStorage { - fn add_data(&mut self, dimension: DataType, data: &warp::sata::Sata) -> Result<()> { - let data = data.clone(); - - // let version = self - // .index - // .as_ref() - // .iter() - // .filter(|inner| inner.id() == data.id() && inner.data_type() == data.data_type()) - // .count(); - - // data.set_version(version as u32); - match dimension { - DataType::FileSystem => { - match data.decode::()? { - DimensionData { - name, - path: Some(path), - .. - } => { - let new_data = Sata::default(); - let old_path = path; - if !old_path.is_file() { - return Err(Error::ItemNotFile); - } - // let _size = std::fs::metadata(&old_path)?.len(); - - let filename = name.as_ref().ok_or(Error::FileNotFound)?; - - // data.set_size(size); - - let new_path = { - let mut path = self.directory.clone(); - path.push("FileSystem"); - create_dir_all(&path)?; - path.push(Uuid::new_v4().to_string()); - path - }; - - let data = new_data.encode( - warp::libipld::IpldCodec::DagCbor, - warp::sata::Kind::Reference, - DimensionData { - name: Some(filename.clone()), - path: Some(new_path.clone()), - ..Default::default() - }, - )?; - let mut writer = std::fs::File::create(&new_path)?; - - let mut reader = std::fs::File::open(old_path)?; - std::io::copy(&mut reader, &mut writer)?; - writer.flush()?; - - if let Err(e) = self.index.insert(DataType::FileSystem, data) { - std::fs::remove_file(new_path)?; - return Err(e); - } - } - DimensionData { - name: Some(name), - buffer: Some(buffer), - .. - } => { - let new_data = Sata::default(); - // - // let size = buffer.len(); - // data.set_size(size as u64); - - let new_path = { - let mut path = self.directory.clone(); - path.push("FileSystem"); - create_dir_all(&path)?; - path.push(Uuid::new_v4().to_string()); - path - }; - - let data = new_data.encode( - warp::libipld::IpldCodec::DagCbor, - warp::sata::Kind::Reference, - DimensionData { - name: Some(name), - path: Some(new_path.clone()), - ..Default::default() - }, - )?; - - let mut writer = std::fs::File::create(&new_path)?; - - let mut reader = std::io::Cursor::new(buffer); - std::io::copy(&mut reader, &mut writer)?; - writer.flush()?; - - if let Err(e) = self.index.insert(DataType::FileSystem, data) { - std::fs::remove_file(new_path)?; - return Err(e); - } - } - DimensionData { .. } => { - // let size = internal.len(); - // data.set_size(size as u64); - //let data_2 = data.clone(); - - match self.index.insert(DataType::FileSystem, data.clone()) { - Ok(()) => { - let new_path = { - let mut path = self.directory.clone(); - path.push("FileSystem"); - create_dir_all(&path)?; - path.push(Uuid::new_v4().to_string()); - path - }; - - let serialized_data = serde_json::to_string(&data).unwrap(); - let mut writer = std::fs::File::create(new_path)?; - - let mut reader = std::io::Cursor::new(serialized_data); - std::io::copy(&mut reader, &mut writer)?; - } - Err(e) => return Err(e), - } - } - } - } - DataType::Accounts => match self.index.insert(DataType::Accounts, data.clone()) { - Ok(()) => { - let new_path = { - let mut path = self.directory.clone(); - path.push("Account"); - create_dir_all(&path)?; - path.push(Uuid::new_v4().to_string()); - path - }; - - let serialized_data = serde_json::to_string(&data).unwrap(); - let mut writer = std::fs::File::create(new_path)?; - - let mut reader = std::io::Cursor::new(serialized_data); - std::io::copy(&mut reader, &mut writer)?; - } - Err(e) => return Err(e), - }, - - DataType::Messaging => match self.index.insert(DataType::Messaging, data.clone()) { - Ok(()) => { - let new_path = { - let mut path = self.directory.clone(); - path.push("Messaging"); - create_dir_all(&path)?; - path.push(Uuid::new_v4().to_string()); - path - }; - - let serialized_data = serde_json::to_string(&data).unwrap(); - let mut writer = std::fs::File::create(new_path)?; - - let mut reader = std::io::Cursor::new(serialized_data); - std::io::copy(&mut reader, &mut writer)?; - } - Err(e) => return Err(e), - }, - - DataType::DataExport => match self.index.insert(DataType::DataExport, data.clone()) { - Ok(()) => { - let new_path = { - let mut path = self.directory.clone(); - path.push("DataExport"); - create_dir_all(&path)?; - path.push(Uuid::new_v4().to_string()); - path - }; - - let serialized_data = serde_json::to_string(&data).unwrap(); - let mut writer = std::fs::File::create(new_path)?; - - let mut reader = std::io::Cursor::new(serialized_data); - std::io::copy(&mut reader, &mut writer)?; - } - Err(e) => return Err(e), - }, - DataType::Http => match self.index.insert(DataType::Http, data.clone()) { - Ok(()) => { - let new_path = { - let mut path = self.directory.clone(); - path.push("Http"); - create_dir_all(&path)?; - path.push(Uuid::new_v4().to_string()); - path - }; - - let serialized_data = serde_json::to_string(&data).unwrap(); - let mut writer = std::fs::File::create(new_path)?; - - let mut reader = std::io::Cursor::new(serialized_data); - std::io::copy(&mut reader, &mut writer)?; - } - Err(e) => return Err(e), - }, - _ => self.index.insert(dimension, data)?, - } - - self.sync() - } - - fn has_data( - &mut self, - dimension: DataType, - query: &warp::pocket_dimension::query::QueryBuilder, - ) -> Result<()> { - let list = self - .index - .read() - .get(&dimension) - .cloned() - .unwrap_or_default(); - - if execute(&list, query)?.is_empty() { - Err(Error::DataObjectNotFound) - } else { - Ok(()) - } - } - - fn get_data( - &self, - dimension: DataType, - query: Option<&warp::pocket_dimension::query::QueryBuilder>, - ) -> Result> { - let list = self - .index - .read() - .get(&dimension) - .cloned() - .unwrap_or_default(); - - match query { - Some(query) => execute(&list, query), - None => Ok(list), - } - } - - fn size( - &self, - dimension: DataType, - query: Option<&warp::pocket_dimension::query::QueryBuilder>, - ) -> Result { - let mut size = 0; - let list = self.get_data(dimension, query)?; - for item in list { - match dimension { - DataType::FileSystem => { - let data = match item.state() { - State::Encoded => match item.decode::() { - Ok(item) => item, - _ => continue, - }, - _ => continue, - }; - match data { - DimensionData { - path: Some(path), .. - } => { - if let Ok(meta) = std::fs::metadata(path) { - size += meta.len() as i64; - } - } - DimensionData { - internal: Some(internal), - .. - } => { - size += internal.len() as i64; - } - _ => {} - } - } - _ => { - size += item.data().len() as i64; - } - } - } - - Ok(size) - } - - fn count( - &self, - dimension: DataType, - query: Option<&warp::pocket_dimension::query::QueryBuilder>, - ) -> Result { - self.get_data(dimension, query) - .map(|list| list.len() as i64) - } - - fn empty(&mut self, dimension: DataType) -> Result<()> { - let mut index = self.index.write(); - - let list = index.remove(&dimension); - - if let Some(list) = list { - for item in list { - if dimension == DataType::FileSystem { - let data = match item.state() { - State::Encoded => match item.decode::() { - Ok(data) => data, - Err(_) => continue, - }, - _ => continue, - }; - - if let Ok(path) = data.path() { - std::fs::remove_file(path)?; - } - } else { - let mut directory = self.directory.clone(); - match dimension { - DataType::Accounts => directory.push("Account"), - DataType::Messaging => directory.push("Messaging"), - DataType::DataExport => directory.push("DataExport"), - DataType::Http => directory.push("Http"), - _ => {} - } - for item in std::fs::read_dir(directory)? { - let item = item?; - let path = item.path(); - std::fs::remove_file(path)?; - } - } - } - } - - self.sync() - } -} - -pub(crate) fn execute(data: &[Sata], query: &QueryBuilder) -> Result> { - let mut list = Vec::new(); - for data in data.iter() { - let object = match data.state() { - State::Encoded => data.decode::()?, - _ => continue, - }; - - match object { - Ipld::Map(_) => {} - _ => continue, - }; - - for (key, val) in query.get_where().iter() { - if let Ok(result) = object.get(key.as_str()) { - if *val == *result { - list.push(data.clone()); - } - } - } - - for comp in query.get_comparator().iter() { - match comp { - ComparatorFilter::Eq(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - if *result == *val { - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - } - ComparatorFilter::Ne(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - if *result != *val { - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - } - ComparatorFilter::Gte(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - match (result, val) { - (Ipld::Integer(res), Ipld::Integer(v)) if *res >= *v => {} - (Ipld::Float(res), Ipld::Float(v)) if *res >= *v => {} - _ => continue, - }; - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - ComparatorFilter::Gt(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - match (result, val) { - (Ipld::Integer(res), Ipld::Integer(v)) if *res > *v => {} - (Ipld::Float(res), Ipld::Float(v)) if *res > *v => {} - _ => continue, - }; - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - ComparatorFilter::Lte(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - match (result, val) { - (Ipld::Integer(res), Ipld::Integer(v)) if *res <= *v => {} - (Ipld::Float(res), Ipld::Float(v)) if *res <= *v => {} - _ => continue, - }; - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - ComparatorFilter::Lt(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - match (result, val) { - (Ipld::Integer(res), Ipld::Integer(v)) if *res < *v => {} - (Ipld::Float(res), Ipld::Float(v)) if *res < *v => {} - _ => continue, - }; - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - } - } - - if let Some(limit) = query.get_limit() { - if list.len() > limit { - list = list.drain(..limit).collect(); - } - } - } - Ok(list) -} diff --git a/extensions/warp-pd-memory/Cargo.toml b/extensions/warp-pd-memory/Cargo.toml deleted file mode 100644 index 47dd64eef..000000000 --- a/extensions/warp-pd-memory/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "warp-pd-memory" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib", "rlib", "staticlib"] - -[dependencies] -warp = { path = "../../warp" } - -serde = { version = "1.0", features = ["derive"] } -serde_json = { version = "1.0" } -anyhow = "1.0" -uuid = { version = "1.0", features = ["serde", "v4"] } - -thiserror = "1.0" - -[build-dependencies] -cbindgen = "0.23" - -[features] -build-header = [] \ No newline at end of file diff --git a/extensions/warp-pd-memory/examples/memory-cache.rs b/extensions/warp-pd-memory/examples/memory-cache.rs deleted file mode 100644 index 671130160..000000000 --- a/extensions/warp-pd-memory/examples/memory-cache.rs +++ /dev/null @@ -1,56 +0,0 @@ -use warp::data::DataType; -use warp::module::Module; -use warp::pocket_dimension::query::{Comparator, QueryBuilder}; -use warp::pocket_dimension::PocketDimension; -use warp::sata::Sata; -use warp_pd_memory::MemoryClient; - -#[derive(Debug, serde::Serialize, serde::Deserialize)] -pub struct Item { - pub port: u64, - pub data: String, -} - -fn main() -> anyhow::Result<()> { - let items = vec![ - Item { - port: 10000, - data: "Local".into(), - }, - Item { - port: 10001, - data: "Global".into(), - }, - Item { - port: 10002, - data: "All".into(), - }, - ]; - - let mut cache = MemoryClient::new(); - - for item in items { - let data = Sata::default().encode( - warp::sata::libipld::IpldCodec::DagJson, - warp::sata::Kind::Reference, - item, - )?; - cache.add_data(DataType::from(Module::Unknown), &data)?; - } - - let mut query = QueryBuilder::default(); - query.filter(Comparator::Eq, "port", 10001)?; - - let data_list = cache.get_data(DataType::from(Module::Unknown), Some(&query))?; - - for data in data_list { - let item = data.decode::()?; - println!("Item::port={}", item.port); - println!("Item::data={}", item.data); - } - - // let count = cache.count(DataType::from(Module::Unknown), None)?; - - // assert!(count == 0); - Ok(()) -} diff --git a/extensions/warp-pd-memory/src/lib.rs b/extensions/warp-pd-memory/src/lib.rs deleted file mode 100644 index b71edf24f..000000000 --- a/extensions/warp-pd-memory/src/lib.rs +++ /dev/null @@ -1,207 +0,0 @@ -use std::collections::HashMap; -use warp::error::Error; -use warp::libipld::Ipld; -use warp::pocket_dimension::query::{ComparatorFilter, QueryBuilder}; -use warp::pocket_dimension::PocketDimension; -use warp::{ - data::DataType, - module::Module, - sata::{Sata, State}, - Extension, SingleHandle, -}; - -pub type Result = std::result::Result; - -#[derive(Clone, Default)] -pub struct MemoryClient { - client: HashMap>, -} - -impl Extension for MemoryClient { - fn id(&self) -> String { - String::from("warp-pd-memory") - } - - fn name(&self) -> String { - String::from("In-Memory Caching System") - } - - fn description(&self) -> String { - String::from("") - } - - fn module(&self) -> Module { - Module::Cache - } -} - -impl MemoryClient { - pub fn new() -> Self { - let client = HashMap::new(); - Self { client } - } -} - -impl SingleHandle for MemoryClient {} - -impl PocketDimension for MemoryClient { - fn add_data(&mut self, dimension: DataType, data: &Sata) -> Result<()> { - let mut data = data.clone(); - if let Some(value) = self.client.get_mut(&dimension) { - let version = value.iter().filter(|item| item.id() == data.id()).count() as u32; - data.set_version(version); - value.push(data); - } else { - self.client.insert(dimension, vec![data]); - } - Ok(()) - } - - fn has_data(&mut self, dimension: DataType, query: &QueryBuilder) -> Result<()> { - self.client - .get(&dimension) - .ok_or(Error::DataObjectNotFound) - .and_then(|data| execute(data, query).map(|_| ())) - } - - fn get_data(&self, dimension: DataType, query: Option<&QueryBuilder>) -> Result> { - let data = self - .client - .get(&dimension) - .ok_or(Error::DataObjectNotFound)?; - - match query { - Some(query) => execute(data, query), - None => Ok(data.to_vec()), - } - } - - fn size(&self, dimension: DataType, query: Option<&QueryBuilder>) -> Result { - self.get_data(dimension, query) - .map(|data| data.iter().map(|i| i.data().len() as i64).sum()) - } - - fn count(&self, dimension: DataType, query: Option<&QueryBuilder>) -> Result { - self.get_data(dimension, query) - .map(|data| data.len() as i64) - } - - fn empty(&mut self, dimension: DataType) -> Result<()> { - self.client - .remove(&dimension) - .ok_or(Error::DataObjectNotFound) - .map(|_| ()) - } -} - -// #[cfg(not(target_arch = "wasm32"))] -// fn get_payload_as_value(data: &Sata) -> Result { -// data.decode().map_err(Error::from) -// } - -pub(crate) fn execute(data: &[Sata], query: &QueryBuilder) -> Result> { - let mut list = Vec::new(); - for data in data.iter() { - let object = match data.state() { - State::Encoded => data.decode::()?, - _ => continue, - }; - - match object { - Ipld::Map(_) => {} - _ => continue, - }; - - for (key, val) in query.get_where().iter() { - if let Ok(result) = object.get(key.as_str()) { - if *val == *result { - list.push(data.clone()); - } - } - } - - for comp in query.get_comparator().iter() { - match comp { - ComparatorFilter::Eq(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - if *result == *val { - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - } - ComparatorFilter::Ne(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - if *result != *val { - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - } - ComparatorFilter::Gte(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - match (result, val) { - (Ipld::Integer(res), Ipld::Integer(v)) if *res >= *v => {} - (Ipld::Float(res), Ipld::Float(v)) if *res >= *v => {} - _ => continue, - }; - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - ComparatorFilter::Gt(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - match (result, val) { - (Ipld::Integer(res), Ipld::Integer(v)) if *res > *v => {} - (Ipld::Float(res), Ipld::Float(v)) if *res > *v => {} - _ => continue, - }; - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - ComparatorFilter::Lte(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - match (result, val) { - (Ipld::Integer(res), Ipld::Integer(v)) if *res <= *v => {} - (Ipld::Float(res), Ipld::Float(v)) if *res <= *v => {} - _ => continue, - }; - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - ComparatorFilter::Lt(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - match (result, val) { - (Ipld::Integer(res), Ipld::Integer(v)) if *res < *v => {} - (Ipld::Float(res), Ipld::Float(v)) if *res < *v => {} - _ => continue, - }; - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - } - } - - if let Some(limit) = query.get_limit() { - if list.len() > limit { - list = list.drain(..limit).collect(); - } - } - } - Ok(list) -} diff --git a/extensions/warp-pd-stretto/Cargo.toml b/extensions/warp-pd-stretto/Cargo.toml deleted file mode 100644 index 03ad1a5df..000000000 --- a/extensions/warp-pd-stretto/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "warp-pd-stretto" -version = "0.1.0" -edition = "2021" - -[lib] -crate-type = ["cdylib", "rlib", "staticlib"] - -[dependencies] -warp = { path = "../../warp" } - -serde = { version = "1.0", features = ["derive"]} -serde_json = { version = "1.0", features = ["arbitrary_precision"] } -anyhow = "1.0" -uuid = { version = "1.0", features = ["serde", "v4"] } - -stretto = { version = "0.7", features = ["full"] } -thiserror = "1.0" - -[build-dependencies] -cbindgen = "0.23" - -[features] -build-header = [] \ No newline at end of file diff --git a/extensions/warp-pd-stretto/examples/stretto-cache.rs b/extensions/warp-pd-stretto/examples/stretto-cache.rs deleted file mode 100644 index b0e61a152..000000000 --- a/extensions/warp-pd-stretto/examples/stretto-cache.rs +++ /dev/null @@ -1,56 +0,0 @@ -use warp::data::DataType; -use warp::module::Module; -use warp::pocket_dimension::query::{Comparator, QueryBuilder}; -use warp::pocket_dimension::PocketDimension; -use warp::sata::Sata; -use warp_pd_stretto::StrettoClient; - -#[derive(Debug, serde::Serialize, serde::Deserialize)] -pub struct Item { - pub port: i64, - pub data: String, -} - -fn main() -> anyhow::Result<()> { - let items = vec![ - Item { - port: 10000, - data: "Local".into(), - }, - Item { - port: 10001, - data: "Global".into(), - }, - Item { - port: 10002, - data: "All".into(), - }, - ]; - - let mut cache = StrettoClient::new()?; - - for item in items { - let data = Sata::default().encode( - warp::sata::libipld::IpldCodec::DagCbor, - warp::sata::Kind::Reference, - item, - )?; - cache.add_data(DataType::from(Module::Unknown), &data)?; - } - - let mut query = QueryBuilder::default(); - query.filter(Comparator::Gte, "port", 10001)?; - let data_list = cache.get_data(DataType::from(Module::Unknown), Some(&query))?; - for data in data_list { - let item = data.decode::()?; - println!("Item::port={}", item.port); - println!("Item::data={}", item.data); - } - - cache.empty(DataType::from(Module::Unknown))?; - - let count = cache.count(DataType::from(Module::Unknown), None)?; - - assert!(count == 0); - Ok(()) -} diff --git a/extensions/warp-pd-stretto/src/lib.rs b/extensions/warp-pd-stretto/src/lib.rs deleted file mode 100644 index e89312574..000000000 --- a/extensions/warp-pd-stretto/src/lib.rs +++ /dev/null @@ -1,299 +0,0 @@ -use anyhow::anyhow; -use warp::{ - data::DataType, - libipld::Ipld, - module::Module, - sata::{Sata, State}, - Extension, SingleHandle, -}; - -use stretto::Cache; - -use warp::error::Error; - -use warp::pocket_dimension::query::{ComparatorFilter, QueryBuilder}; -use warp::pocket_dimension::PocketDimension; - -pub type Result = std::result::Result; - -#[derive(Clone)] -pub struct StrettoClient { - client: Cache>, -} - -impl Extension for StrettoClient { - fn id(&self) -> String { - String::from("warp-pd-stretto") - } - - fn name(&self) -> String { - String::from("Stretto Caching System") - } - - fn description(&self) -> String { - String::from("Pocket Dimension implementation with Stretto, a high performance thread-safe memory cache written in rust.") - } - - fn module(&self) -> Module { - Module::Cache - } -} - -impl StrettoClient { - pub fn new() -> anyhow::Result { - let client = Cache::new(12960, 1e6 as i64)?; - Ok(Self { client }) - } - - pub fn client(&self) -> &Cache> { - &self.client - } -} - -impl SingleHandle for StrettoClient {} - -impl PocketDimension for StrettoClient { - fn add_data(&mut self, dimension: DataType, data: &Sata) -> Result<()> { - let data = data.clone(); - // data.set_data_type(dimension); - - if let Some(mut value) = self.client.get_mut(&dimension) { - // let version = value - // .value() - // .iter() - // .filter(|item| item.id() == data.id()) - // .count() as u32; - // data.set_version(version); - (*value.value_mut()).push(data); - self.client - .wait() - .map_err(|_| warp::error::Error::DataObjectNotFound)?; - } else { - self.client.insert(dimension, vec![data], 1); - self.client - .wait() - .map_err(|_| warp::error::Error::DataObjectNotFound)?; - } - Ok(()) - } - - fn has_data(&mut self, dimension: DataType, query: &QueryBuilder) -> Result<()> { - self.client - .get(&dimension) - .ok_or(warp::error::Error::DataObjectNotFound) - .and_then(|data| execute(data.value(), query).map(|_| ())) - } - - fn get_data(&self, dimension: DataType, query: Option<&QueryBuilder>) -> Result> { - let data = self - .client - .get(&dimension) - .ok_or(warp::error::Error::DataObjectNotFound)?; - - let data = data.value(); - match query { - Some(query) => execute(data, query), - None => Ok(data.clone()), - } - } - - fn size(&self, dimension: DataType, query: Option<&QueryBuilder>) -> Result { - self.get_data(dimension, query) - .map(|data| data.iter().map(|i| i.data().len() as i64).sum()) - } - - fn count(&self, dimension: DataType, query: Option<&QueryBuilder>) -> Result { - self.get_data(dimension, query) - .map(|data| data.len() as i64) - } - - fn empty(&mut self, dimension: DataType) -> Result<()> { - if let Some(mut dim) = self.client.get_mut(&dimension) { - dim.value_mut().clear(); - } - - self.client - .wait() - .map_err(|e| anyhow!(e)) - .map_err(Error::from) - } -} - -//TODO: Rewrite -pub(crate) fn execute(data: &[Sata], query: &QueryBuilder) -> Result> { - let mut list = Vec::new(); - for data in data.iter() { - let object = match data.state() { - State::Encoded => data.decode::()?, - _ => continue, - }; - - match object { - Ipld::Map(_) => {} - _ => continue, - }; - - for (key, val) in query.get_where().iter() { - if let Ok(result) = object.get(key.as_str()) { - if *val == *result { - list.push(data.clone()); - } - } - } - - for comp in query.get_comparator().iter() { - match comp { - ComparatorFilter::Eq(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - if *result == *val { - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - } - ComparatorFilter::Ne(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - if *result != *val { - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - } - ComparatorFilter::Gte(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - match (result, val) { - (Ipld::Integer(res), Ipld::Integer(v)) if *res >= *v => {} - (Ipld::Float(res), Ipld::Float(v)) if *res >= *v => {} - _ => continue, - }; - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - ComparatorFilter::Gt(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - match (result, val) { - (Ipld::Integer(res), Ipld::Integer(v)) if *res > *v => {} - (Ipld::Float(res), Ipld::Float(v)) if *res > *v => {} - _ => continue, - }; - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - ComparatorFilter::Lte(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - match (result, val) { - (Ipld::Integer(res), Ipld::Integer(v)) if *res <= *v => {} - (Ipld::Float(res), Ipld::Float(v)) if *res <= *v => {} - _ => continue, - }; - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - ComparatorFilter::Lt(key, val) => { - if let Ok(result) = object.get(key.as_str()) { - match (result, val) { - (Ipld::Integer(res), Ipld::Integer(v)) if *res < *v => {} - (Ipld::Float(res), Ipld::Float(v)) if *res < *v => {} - _ => continue, - }; - if list.contains(data) { - continue; - } - list.push(data.clone()); - } - } - } - } - - if let Some(limit) = query.get_limit() { - if list.len() > limit { - list = list.drain(..limit).collect(); - } - } - } - Ok(list) -} - -#[cfg(test)] -mod test { - use crate::StrettoClient; - use serde::{Deserialize, Serialize}; - use warp::data::DataType; - use warp::error::Error; - use warp::module::Module; - use warp::pocket_dimension::query::{Comparator, QueryBuilder}; - use warp::pocket_dimension::PocketDimension; - use warp::sata::Sata; - - #[derive(Serialize, Deserialize, Debug, Clone)] - pub struct SomeData { - pub name: String, - pub age: i64, - } - - impl Default for SomeData { - fn default() -> Self { - Self { - name: String::from("John Doe"), - age: 21, - } - } - } - - impl SomeData { - pub fn set_name>(&mut self, name: S) { - self.name = name.as_ref().to_string(); - } - pub fn set_age(&mut self, age: i64) { - self.age = age - } - } - - fn generate_data(system: &mut StrettoClient, amount: i64) { - for i in 0..amount { - let mut data = SomeData::default(); - data.set_name(&format!("Test Subject {i}")); - data.set_age(18 + i); - - let object = Sata::default() - .encode( - warp::libipld::IpldCodec::DagJson, - warp::sata::Kind::Reference, - data, - ) - .unwrap(); - system - .add_data(DataType::from(Module::Accounts), &object) - .unwrap(); - } - } - - #[test] - fn if_count_eq_five() -> Result<(), Error> { - let mut memory = StrettoClient::new().map_err(|_| Error::Other)?; - - generate_data(&mut memory, 100); - - let mut query = QueryBuilder::default(); - query.filter(Comparator::Gte, "age", 19)?.limit(5); - - let count = memory.count(DataType::from(Module::Accounts), Some(&query))?; - - assert_eq!(count, 5); - - Ok(()) - } -} diff --git a/tools/relay-server/Cargo.toml b/tools/relay-server/Cargo.toml index feaf3928c..f91bb557d 100644 --- a/tools/relay-server/Cargo.toml +++ b/tools/relay-server/Cargo.toml @@ -21,9 +21,9 @@ serde.workspace = true serde_json.workspace = true void.workspace = true tracing.workspace = true -toml = "0.8" +toml.workspace = true tracing-subscriber = { version = "0.3", features = ["env-filter"] } clap = { version = "4.4", features = ["derive"] } -zeroize = "1" +zeroize.workspace = true dotenv = "0.15" base64 = "0.21" diff --git a/tools/shuttle/Cargo.toml b/tools/shuttle/Cargo.toml index 8878cedc1..a799e0296 100644 --- a/tools/shuttle/Cargo.toml +++ b/tools/shuttle/Cargo.toml @@ -38,8 +38,8 @@ chrono = { workspace = true } void.workspace = true clap = { version = "4.4", features = ["derive"] } -zeroize = "1" +zeroize.workspace = true dotenv = "0.15" base64 = "0.21" -bs58 = "0.4" +bs58.workspace = true diff --git a/warp/Cargo.toml b/warp/Cargo.toml index 5e1d20f5c..8b230f9e7 100644 --- a/warp/Cargo.toml +++ b/warp/Cargo.toml @@ -38,7 +38,6 @@ thiserror.workspace = true # Sync crates parking_lot = { workspace = true, features = ["send_guard", "serde"] } -once_cell.workspace = true # Time crate chrono = { workspace = true } @@ -47,24 +46,20 @@ chrono = { workspace = true } serde.workspace = true serde_cbor.workspace = true serde_json.workspace = true -serde_yaml.workspace = true bincode.workspace = true -toml.workspace = true bs58.workspace = true hex.workspace = true -libipld = { workspace = true } # Misc dyn-clone.workspace = true uuid.workspace = true derive_more.workspace = true paste.workspace = true -sata.workspace = true -tracing = { features = ["log"], workspace = true } +tracing = { workspace = true } mediatype.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -tokio = { workspace = true, features = ["full"] } +tokio = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dependencies] wasm-bindgen = { version = "0.2" } diff --git a/warp/src/constellation/mod.rs b/warp/src/constellation/mod.rs index 9e13d947f..c33b9f6f3 100644 --- a/warp/src/constellation/mod.rs +++ b/warp/src/constellation/mod.rs @@ -216,35 +216,6 @@ pub trait Constellation: async fn sync_ref(&mut self, _: &str) -> Result<(), Error> { Err(Error::Unimplemented) } - - /// Used to export the filesystem to a specific structure. Currently supports `Json`, `Toml`, and `Yaml` - fn export(&self, r#type: ConstellationDataType) -> Result { - match r#type { - ConstellationDataType::Json => { - serde_json::to_string(&self.root_directory()).map_err(Error::from) - } - ConstellationDataType::Yaml => { - serde_yaml::to_string(&self.root_directory()).map_err(Error::from) - } - ConstellationDataType::Toml => { - toml::to_string(&self.root_directory()).map_err(Error::from) - } - } - } - - /// Used to import data into the filesystem. This would override current contents. - /// TODO: Have the data argument accept either bytes or an reader field - fn import(&mut self, r#type: ConstellationDataType, data: String) -> Result<(), Error> { - let directory: Directory = match r#type { - ConstellationDataType::Json => serde_json::from_str(data.as_str())?, - ConstellationDataType::Yaml => serde_yaml::from_str(data.as_str())?, - ConstellationDataType::Toml => toml::from_str(data.as_str())?, - }; - //TODO: create a function to override directory items. - self.root_directory().set_items(directory.get_items()); - - Ok(()) - } } dyn_clone::clone_trait_object!(Constellation); diff --git a/warp/src/error.rs b/warp/src/error.rs index 43a7bd8d7..6fe652324 100644 --- a/warp/src/error.rs +++ b/warp/src/error.rs @@ -277,14 +277,6 @@ pub enum Error { UuidError(#[from] uuid::Error), #[error("{0}")] BincodeError(#[from] bincode::Error), - #[error("{0}")] - SerdeYamlError(#[from] serde_yaml::Error), - #[error("Cannot deserialize: {0}")] - TomlDeserializeError(#[from] toml::de::Error), - #[error("Cannot serialize: {0}")] - TomlSerializeError(#[from] toml::ser::Error), - #[error(transparent)] - SataError(#[from] sata::error::Error), #[error(transparent)] Any(#[from] anyhow::Error), #[error(transparent)] diff --git a/warp/src/lib.rs b/warp/src/lib.rs index 2ad28eed1..dd793ea45 100644 --- a/warp/src/lib.rs +++ b/warp/src/lib.rs @@ -7,13 +7,9 @@ pub mod data; pub mod error; pub mod module; pub mod multipass; -pub mod pocket_dimension; pub mod raygun; pub mod tesseract; -pub use libipld; -pub use sata; - /// Used to downcast a specific type from an extension to share to another pub trait SingleHandle { fn handle(&self) -> Result, error::Error> { diff --git a/warp/src/pocket_dimension/mod.rs b/warp/src/pocket_dimension/mod.rs deleted file mode 100644 index f77c25506..000000000 --- a/warp/src/pocket_dimension/mod.rs +++ /dev/null @@ -1,158 +0,0 @@ -#![allow(clippy::result_large_err)] -pub mod query; - -use crate::data::DataType; -use crate::error::Error; -use crate::{Extension, SingleHandle}; -use dyn_clone::DynClone; -use query::QueryBuilder; -use sata::Sata; -#[cfg(not(target_arch = "wasm32"))] -use std::io::Write; -use std::path::PathBuf; - -use serde::{Deserialize, Serialize}; - -#[cfg(not(target_arch = "wasm32"))] -#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct DimensionData { - pub name: Option, - pub path: Option, - pub buffer: Option>, - pub internal: Option>, -} - -impl> From

for DimensionData { - fn from(path: P) -> Self { - let path = path.as_ref().to_path_buf(); - let name = path.file_name().map(|s| s.to_string_lossy().to_string()); - DimensionData { - name, - path: Some(path), - ..Default::default() - } - } -} - -impl DimensionData { - pub fn from_path(name: &str, path: &str) -> Self { - DimensionData { - name: Some(name.to_string()), - path: Some(std::path::PathBuf::from(path.to_string())), - ..Default::default() - } - } - - pub fn from_buffer(name: &str, buffer: &[u8]) -> Self { - let name = name.to_string(); - let buffer = buffer.to_vec(); - DimensionData { - name: Some(name), - buffer: Some(buffer), - ..Default::default() - } - } - - pub fn from_buffer_nofile(name: &str, internal: &[u8]) -> Self { - let name = name.to_string(); - let internal = internal.to_vec(); - DimensionData { - name: Some(name), - internal: Some(internal), - ..Default::default() - } - } -} - -impl DimensionData { - pub fn name(&self) -> Result { - if let Self { - name: Some(name), .. - } = self - { - return Ok(name.clone()); - } - Err(Error::Other) - } -} - -impl DimensionData { - pub fn path(&self) -> Result { - if let Self { - path: Some(path), .. - } = self - { - return Ok(path.clone()); - } - Err(Error::Other) - } - - pub fn write_to_buffer(&self, buffer: &mut [u8]) -> Result<(), Error> { - if let Self { - internal: Some(internal), - .. - } = self - { - buffer.copy_from_slice(internal); - return Ok(()); - } - Err(Error::Other) - } -} - -#[cfg(not(target_arch = "wasm32"))] -impl DimensionData { - pub fn write_from_path(&self, writer: &mut W) -> Result<(), Error> { - match self { - Self { - name: Some(_), - path: Some(path), - .. - } => { - let mut file = std::fs::File::open(path)?; - std::io::copy(&mut file, writer)?; - return Ok(()); - } - Self { - internal: Some(internal), - .. - } => { - let mut cursor = std::io::Cursor::new(internal); - std::io::copy(&mut cursor, writer)?; - return Ok(()); - } - _ => {} - }; - Err(Error::Other) - } -} - -/// PocketDimension interface will allow [`Module`] to store data for quick indexing and searching later on. This would be useful -/// for caching frequently used data so that request can be made faster. This makes it easy by sorting the data per module, as well -/// as allowing querying by specific information stored inside the payload of the [`Sata`] for a quick turnaround for search -/// results. -pub trait PocketDimension: Extension + Send + Sync + SingleHandle + DynClone { - /// Used to add data to [`PocketDimension`] for [`Module`] - fn add_data(&mut self, dimension: DataType, data: &Sata) -> Result<(), Error>; - - /// Used to check to see if data exist within [`PocketDimension`] - fn has_data(&mut self, dimension: DataType, query: &QueryBuilder) -> Result<(), Error>; - - /// Used to obtain a list of [`Sata`] for [`Module`] - fn get_data( - &self, - dimension: DataType, - query: Option<&QueryBuilder>, - ) -> Result, Error>; - - /// Returns the total size within the [`Module`] - fn size(&self, dimension: DataType, query: Option<&QueryBuilder>) -> Result; - - /// Returns an total amount of [`Sata`] for [`Module`] - fn count(&self, dimension: DataType, query: Option<&QueryBuilder>) -> Result; - - /// Will flush out the data related to [`Module`]. - fn empty(&mut self, dimension: DataType) -> Result<(), Error>; -} - -dyn_clone::clone_trait_object!(PocketDimension); diff --git a/warp/src/pocket_dimension/query.rs b/warp/src/pocket_dimension/query.rs deleted file mode 100644 index 949c566cb..000000000 --- a/warp/src/pocket_dimension/query.rs +++ /dev/null @@ -1,103 +0,0 @@ -use libipld::{serde::to_ipld, Ipld}; - -use crate::error::Error; -use serde::{Deserialize, Serialize}; -use serde_json::{self}; - -#[derive(Debug, Serialize, Deserialize, Clone)] -#[repr(C)] -#[serde(rename_all = "lowercase")] -pub enum Comparator { - Eq, - Gt, - Gte, - Lt, - Lte, - Ne, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -#[serde(rename_all = "lowercase")] -pub enum ComparatorFilter { - Eq(String, Ipld), - Gt(String, Ipld), - Gte(String, Ipld), - Lt(String, Ipld), - Lte(String, Ipld), - Ne(String, Ipld), -} - -impl From<(Comparator, String, Ipld)> for ComparatorFilter { - fn from((comp, key, val): (Comparator, String, Ipld)) -> Self { - match comp { - Comparator::Eq => ComparatorFilter::Eq(key, val), - Comparator::Ne => ComparatorFilter::Ne(key, val), - Comparator::Gt => ComparatorFilter::Gt(key, val), - Comparator::Gte => ComparatorFilter::Gte(key, val), - Comparator::Lt => ComparatorFilter::Lt(key, val), - Comparator::Lte => ComparatorFilter::Lte(key, val), - } - } -} - -#[derive(Default, Serialize, Deserialize, Debug, Clone)] -pub struct QueryBuilder { - r#where: Vec<(String, Ipld)>, - comparator: Vec, - limit: Option, -} - -impl QueryBuilder { - pub fn import(data: &str) -> Result { - serde_json::from_str(data).map_err(Error::SerdeJsonError) - } -} - -impl QueryBuilder { - pub fn get_where(&self) -> Vec<(String, Ipld)> { - self.r#where.clone() - } - - pub fn get_comparator(&self) -> Vec { - self.comparator.clone() - } - - pub fn get_limit(&self) -> Option { - self.limit - } -} - -impl QueryBuilder { - pub fn r#where(&mut self, key: &str, value: I) -> Result<&mut Self, Error> - where - I: Serialize, - { - self.r#where.push(( - key.to_string(), - to_ipld(value).map_err(anyhow::Error::from)?, - )); - Ok(self) - } - - pub fn filter( - &mut self, - comparator: Comparator, - key: &str, - value: I, - ) -> Result<&mut Self, Error> - where - I: Serialize, - { - self.comparator.push(ComparatorFilter::from(( - comparator, - key.to_string(), - to_ipld(value).map_err(anyhow::Error::from)?, - ))); - Ok(self) - } - - pub fn limit(&mut self, limit: usize) -> &mut Self { - self.limit = Some(limit); - self - } -}