From 64fd3da00a5631b08ed6b858f943022402bf81dd Mon Sep 17 00:00:00 2001 From: Wesley Parr Date: Wed, 11 Oct 2023 22:28:34 -0600 Subject: [PATCH] backup_test --- .vscode/launch.json | 69 ++++++++++++++++++++++++++++++++++ Cargo.lock | 32 +--------------- Cargo.toml | 7 +++- src/backend/fs.rs | 88 +++++++++++++++++++++++++++++++++++++------- src/constants.rs | 4 +- src/errors.rs | 8 ++++ src/header.rs | 90 +++++++++++++++++++++++++++++++++++++-------- src/lib.rs | 1 + tests/file.rs | 12 +++--- 9 files changed, 243 insertions(+), 68 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 src/errors.rs diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..ad9d334 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,69 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in library 'carbonado-node'", + "cargo": { + "args": ["test", "--no-run", "--lib", "--package=carbonado-node"], + "filter": { + "name": "carbonado-node", + "kind": "lib" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'carbonadod'", + "cargo": { + "args": ["build", "--bin=carbonadod", "--package=carbonado-node"], + "filter": { + "name": "carbonadod", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in executable 'carbonadod'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=carbonadod", + "--package=carbonado-node" + ], + "filter": { + "name": "carbonadod", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug integration test 'file'", + "cargo": { + "args": ["test", "--no-run", "--test=file", "--package=carbonado-node"], + "filter": { + "name": "file", + "kind": "test" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + } + ] +} diff --git a/Cargo.lock b/Cargo.lock index 2044e3f..4b5b5ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -298,7 +298,7 @@ dependencies = [ "hex", "infer", "log", - "magic", + "nom 7.1.3", "once_cell", "par-stream", "percent-encoding 2.2.0 (git+https://github.com/cryptoquick/rust-url.git?branch=addl-percent-encode-sets)", @@ -308,6 +308,7 @@ dependencies = [ "serde", "serde_cbor", "syslog", + "thiserror", "tokio", "toml", "tower-http", @@ -1244,29 +1245,6 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" -[[package]] -name = "magic" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87142e3acb1f4daa62eaea96605421a534119d4777a9fb43fb2784798fd89665" -dependencies = [ - "bitflags", - "errno", - "libc", - "magic-sys", - "thiserror", -] - -[[package]] -name = "magic-sys" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff86ae08895140d628119d407d568f3b657145ee8c265878064f717534bb3bc" -dependencies = [ - "libc", - "vcpkg", -] - [[package]] name = "match_cfg" version = "0.1.0" @@ -2221,12 +2199,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.4" diff --git a/Cargo.toml b/Cargo.toml index 0281a03..cab0a7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,9 @@ name = "carbonado-node" version = "0.1.0" edition = "2021" +[profile.release] +debug = true + [[bin]] name = "carbonadod" @@ -30,7 +33,8 @@ hex = "0.4.3" # human_bytes = "0.4.1" infer = { version = "0.15.0", default-features = false } log = "0.4.17" -magic = "0.13.0" +#magic = "0.13.0" +nom = "7.1.3" once_cell = "1.17.1" par-stream = { version = "0.10.2", features = ["runtime-tokio"] } percent-encoding = { version = "2.2.0", git = "https://github.com/cryptoquick/rust-url.git", branch = "addl-percent-encode-sets" } @@ -45,6 +49,7 @@ secp256k1 = { version = "0.27.0", features = [ serde = { version = "1.0.152", features = ["derive"] } serde_cbor = "0.11" syslog = "6.0.1" +thiserror = "1.0" tokio = { version = "1.26.0", features = ["full"] } toml = "0.7.2" tower-http = { version = "0.4.0", features = ["cors"] } diff --git a/src/backend/fs.rs b/src/backend/fs.rs index 0f41ce1..43067f8 100644 --- a/src/backend/fs.rs +++ b/src/backend/fs.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Error, Result}; use axum::body::Bytes; use bytes::BytesMut; -use carbonado::{constants::Format, file::Header, structs::Encoded}; +use carbonado::{constants::Format, error::CarbonadoError, file::Header, structs::Encoded}; use chrono::{NaiveDateTime, TimeZone, Utc}; use futures_util::{stream, Stream, StreamExt, TryStreamExt}; use infer::{self}; @@ -13,6 +13,7 @@ use rayon::{ }; use secp256k1::{PublicKey, SecretKey}; use std::{ + fmt::DebugTuple, fs::{self, OpenOptions}, io::{self, Read, Seek, Write}, path::{Path, PathBuf}, @@ -27,6 +28,20 @@ use crate::{ prelude::*, }; +use bao::Hash; +// mod structs { +// pub struct Hash { +// // ... your fields here, probably: +// bytes: [u8; 32], +// } + +// impl From<[u8; 32]> for Hash { +// fn from(bytes: [u8; 32]) -> Self { +// Hash { bytes } +// } +// } +// } + pub type FileStream = Pin> + Send>>; pub async fn write_file<'a>( @@ -115,10 +130,17 @@ pub fn write_segment(sk: &[u8], pk: &[u8], encoded: &Encoded) -> Result metadata, )?; let header_bytes = header.try_to_vec()?; + // Helper function for naming a Carbonado file/Header.filename() archive file. + // file name is generated here + // but it is not stored as metadata let file_name = header.file_name(); let path = file_path(chunk_index, &hex::encode(pk), SEGMENT_DIR, &file_name)?; + // path tells it where to write segment but doesn't preserve it + // if we write it to sgement header, it won't be readable at the corrrect time. + // we need it to be wirtten to file? + trace!("Write segment at {path:?}"); let mut file = OpenOptions::new() .create_new(true) @@ -180,6 +202,10 @@ pub async fn write_catalog( let cbor_data = cbor_data.clone(); let metadata = Some(cbor_data.clone()); + // convert to + let my_array_option: Option<[u8; 8]> = metadata.and_then(vec_to_array); + let metadata = my_array_option; + move || { let header = CatHeader { cbor_len, metadata }; let header_bytes = header.try_to_vec()?; @@ -216,17 +242,18 @@ pub async fn write_catalog( } pub fn read_file(pk: &Secp256k1PubKey, lookup: &Lookup) -> Result { - debug!("Read file wiht lookup: {lookup}"); + debug!(">>>>>>>>>>>>>>>>>> Read file witn lookup: {lookup} <<<<<<<<<<<<<<<<<<<<<<<<<<,"); - trace!("Create a shared secret using ECDH"); + trace!(">>>>>>>>>>> SEGMENTS Create a shared secret using ECDH"); let ss = node_shared_secret(&pk.into_inner())?.secret_bytes(); let write_pk = PublicKey::from_secret_key_global(&SecretKey::from_slice(&ss)?); let write_pk_str = write_pk.to_string(); - trace!("Read catalog file bytes, parse out each hash, plus the segment Carbonado format"); + trace!(">>>>>>>>>>> Read catalog file bytes, parse out each hash, plus the segment Carbonado format"); let catalog_file = read_catalog(pk, lookup)?; - trace!("For each hash, read each chunk into a segment, then decode that segment"); + trace!(">>>>>>>>>>> For each hash, read each chunk into a segment, then decode that segment"); + let file_bytes: FileStream = stream::iter(catalog_file) .par_then(None, move |segment_hash| { let write_pk_str = write_pk_str.clone(); @@ -239,7 +266,20 @@ pub fn read_file(pk: &Secp256k1PubKey, lookup: &Lookup) -> Result { &format!("{}.c{}", segment_hash, NODE_FORMAT), )?; + // segment hash is wrong, causing to look for a filename that is not there. + debug!("243 > {}", segment_hash); + debug!("244 >"); + debug!("245 >"); + + println!("Message here {}", write_pk_str); + let mut chunk_file = OpenOptions::new().read(true).open(chunk_path)?; + + debug!( + ">>>> >>>>>>>>>>> ############ try to read in from path :: chunk_file >>>>>" + ); + + // HEADER VERSUS CAT HEADER let header = Header::try_from(&chunk_file)?; let segment: Vec = if SYS_CFG.drive_redundancy > 1 { @@ -250,7 +290,8 @@ pub fn read_file(pk: &Secp256k1PubKey, lookup: &Lookup) -> Result { let write_pk_str = write_pk_str.clone(); let segment_hash = segment_hash.clone(); move || { - trace!("Get catalogs directory path"); + debug!(" >>>>>>> ===== Get catalogs directory path"); + let path = file_path( volume_index, &write_pk_str, @@ -258,10 +299,6 @@ pub fn read_file(pk: &Secp256k1PubKey, lookup: &Lookup) -> Result { &segment_hash, )?; - // let cookie = Cookie::open(magic::flags::MIME_TYPE)?; - // let mime_type = cookie.file(path)?; - // println!("====>>>>>> LIKE MAGIC :: MIME Type: {}", mime_type); - trace!("Read segment file at {path:?}"); let mut file = OpenOptions::new().read(true).open(path)?; @@ -311,14 +348,19 @@ pub fn read_file(pk: &Secp256k1PubKey, lookup: &Lookup) -> Result { } pub fn read_catalog(pk: &Secp256k1PubKey, lookup: &Lookup) -> Result> { - trace!("Create a shared secret using ECDH"); + trace!("CATALOG Create a shared secret using ECDH"); let ss = node_shared_secret(&pk.into_inner())?.secret_bytes(); let write_pk = PublicKey::from_secret_key_global(&SecretKey::from_slice(&ss)?); let write_pk_str = write_pk.to_string(); - let path = file_path(0, &write_pk_str, CATALOG_DIR, &lookup.to_string())?; + let path = file_path( + 0, + &write_pk_str, + CATALOG_DIR, + &format!("{}.cat", &lookup.to_string()), + )?; - trace!("Read catalog at {path:?}"); + trace!("################# >>>> Read catalog at {path:?}"); let mut file = OpenOptions::new().read(true).open(path)?; let mut bytes = vec![]; @@ -411,3 +453,23 @@ fn remove_dir_catalogs(path: PathBuf, file: PathBuf) -> io::Result<()> { // } // Ok(()) // } + +fn vec_to_array(vec: Vec) -> Option<[u8; 8]> { + if vec.len() == 8 { + let mut array = [0u8; 8]; + array.copy_from_slice(&vec); + Some(array) + } else { + None + } +} + +/// Decodes a Bao hash from a hexadecimal string. +pub fn decode_bao_hash(hash: &[u8]) -> Result { + if hash.len() != bao::HASH_SIZE { + Err(CarbonadoError::HashDecodeError(bao::HASH_SIZE, hash.len())) + } else { + let hash_array: [u8; bao::HASH_SIZE] = hash[..].try_into()?; + Ok(hash_array.into()) + } +} diff --git a/src/constants.rs b/src/constants.rs index fffa65d..03df0d5 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -3,5 +3,5 @@ pub const SEGMENT_SIZE: usize = 1024 * 1024; pub const SEGMENT_DIR: &str = "segments"; pub const CATALOG_DIR: &str = "catalogs"; -/// "Magic number" used by the Carbonado file format. 12 bytes: "CARBONADO", and a version, 00, plus a newline character -pub const MAGICNO: &[u8; 16] = b"CARBONADONODE00\n"; +/// "Magic number" used by the Carbonado-node file format. 12 bytes: "CAT_MAGICNO", and a version, 00, plus a newline character +pub const CAT_MAGICNO: [u8; 16] = *b"CARBONADONODE00\n"; diff --git a/src/errors.rs b/src/errors.rs new file mode 100644 index 0000000..26f5cd6 --- /dev/null +++ b/src/errors.rs @@ -0,0 +1,8 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum CarbonadoNodeError { + /// Invalid magic number + #[error("File header lacks Carbonado magic number and may not be a proper Carbonado file. Magic number found was {0}.")] + CatInvalidMagicNumber(String), +} diff --git a/src/header.rs b/src/header.rs index 03dc981..f9af08d 100644 --- a/src/header.rs +++ b/src/header.rs @@ -1,23 +1,65 @@ -use carbonado::error::CarbonadoError; +use crate::errors::CarbonadoNodeError; use log::debug; use serde::{Deserialize, Serialize}; use serde_cbor; use serde_cbor::Error; -use crate::constants::MAGICNO; +use bytes::Bytes; +use nom::{ + bytes::complete::take, + number::complete::{le_u32, le_u8}, // if cbor length is actually u32 + IResult, +}; -// use crate::{ -// constants::{Format, MAGICNO}, -// error::CarbonadoError, -// }; +use crate::constants::CAT_MAGICNO; /// Contains deserialized copies of the data kept in the Catalog header. #[derive(Serialize, Deserialize, Clone, Debug)] pub struct CatHeader { /// Number of bytes added to pad to align zfec chunks and bao slices. - pub cbor_len: u8, + pub cbor_len: u8, // this appears to sbe u32 not u8 /// Bytes that are normally zero but can contain extra data if needed - pub metadata: Option>, + pub metadata: Option<[u8; 8]>, +} + +impl TryFrom<&[u8]> for CatHeader { + type Error = CarbonadoNodeError; + + /// Attempts to decode a cat_header from a file. + fn try_from(bytes: &[u8]) -> Result { + let mut cat_magic_no = [0_u8; 16]; + + // easily get cbor_length back from header to know how far to read + let (_, (cbor_len, metadata)) = CatHeader::parse_bytes(bytes).unwrap(); + + // create throw error for wrong magic number + if cat_magic_no != CAT_MAGICNO { + return Err(CarbonadoNodeError::CatInvalidMagicNumber(format!( + "{cat_magic_no:#?}" + ))); + } + + Ok(CatHeader { cbor_len, metadata }) + } +} + +impl TryFrom for CatHeader { + type Error = CarbonadoNodeError; + + /// Attempts to decode a header from a file. + fn try_from(bytes: Bytes) -> Result { + let mut cat_magic_no = [0_u8; 16]; + + let (_, (cbor_len, metadata)) = CatHeader::parse_bytes(&bytes).unwrap(); + + if cat_magic_no != CAT_MAGICNO { + return Err(CarbonadoNodeError::CatInvalidMagicNumber(format!( + "{cat_magic_no:#?}" + ))); + } + + Ok(CatHeader { cbor_len, metadata }) + } } impl CatHeader { @@ -28,17 +70,18 @@ impl CatHeader { /// Creates a new Catalog Header struct using the provided parameters. #[allow(clippy::too_many_arguments)] - pub fn new(metadata: Vec) -> Result { + pub fn new(metadata: [u8; 8]) -> Result { //type Error = CarbonadoError; let cbor_data = serde_cbor::to_vec(&metadata)?; println!("cbor data; {:?}", cbor_data); let cbor_len = cbor_data.len() as u8; + // cbor length is header length println!("cbor length ; {:?}", cbor_len); Ok(CatHeader { - cbor_len, + cbor_len: cbor_len.into(), metadata: if metadata.iter().any(|b| b != &0) { Some(metadata) } else { @@ -48,7 +91,7 @@ impl CatHeader { } /// Creates a header to be prepended to files. - pub fn try_to_vec(&self) -> Result, CarbonadoError> { + pub fn try_to_vec(&self) -> Result, CarbonadoNodeError> { let mut cbor_bytes = self.cbor_len.to_le_bytes().to_vec(); // 2 bytes let mut cbor_padding = if let Some(metadata) = &self.metadata { // debug!( @@ -64,17 +107,32 @@ impl CatHeader { debug!(">>>> CBORE BYTES LENGTH : {}", self.cbor_len); - let mut header = Vec::new(); + let mut cat_header = Vec::new(); - header.append(&mut MAGICNO.to_vec()); // 12 bytes - header.append(&mut cbor_bytes); - header.append(&mut cbor_padding); + cat_header.append(&mut CAT_MAGICNO.to_vec()); // 12 bytes + cat_header.append(&mut cbor_bytes); + cat_header.append(&mut cbor_padding); // if header.len() != CatHeader::len() { // return Err(CarbonadoError::InvalidHeaderLength); // } - Ok(header) + Ok(cat_header) + } + + #[allow(clippy::type_complexity)] + fn parse_bytes(b: &[u8]) -> IResult<&[u8], (u8, Option<[u8; 8]>)> { + let (b, cbor_len) = le_u8(b)?; + let (b, metadata) = take(8u8)(b)?; + + let metadata: [u8; 8] = metadata.try_into().expect("8 bytes = 8 bytes"); + let metadata = if metadata.iter().any(|b| b != &0) { + Some(metadata) + } else { + None + }; + + Ok((b, (cbor_len, metadata))) } } diff --git a/src/lib.rs b/src/lib.rs index c2acdea..a04f426 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,6 +7,7 @@ use crate::frontend::http; pub mod backend; pub mod config; pub mod constants; +pub mod errors; pub mod frontend; pub mod header; pub mod structs; diff --git a/tests/file.rs b/tests/file.rs index 10f9863..1e332b0 100644 --- a/tests/file.rs +++ b/tests/file.rs @@ -40,7 +40,7 @@ async fn write_read() -> Result<()> { info!("Writing file"); let (x_only, _) = write_pk.x_only_public_key(); - // let orig_hash = blake3::keyed_hash(&x_only.serialize(), &file_bytes); + let _orig_hash = blake3::keyed_hash(&x_only.serialize(), &file_bytes); let file_stream: FileStream = stream::iter(file_bytes) .chunks(SEGMENT_SIZE) @@ -62,11 +62,11 @@ async fn write_read() -> Result<()> { .await? .to_vec(); - // assert_eq!( - // decoded_file.len(), - // file_len, - // "Decoded file is same size as original" - // ); + assert_eq!( + decoded_file.len(), + file_len, + "Decoded file is same size as original" + ); // assert_eq!( // orig_hash.to_hex().to_string(),