diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..ad9d334 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,69 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in library 'carbonado-node'", + "cargo": { + "args": ["test", "--no-run", "--lib", "--package=carbonado-node"], + "filter": { + "name": "carbonado-node", + "kind": "lib" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'carbonadod'", + "cargo": { + "args": ["build", "--bin=carbonadod", "--package=carbonado-node"], + "filter": { + "name": "carbonadod", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in executable 'carbonadod'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=carbonadod", + "--package=carbonado-node" + ], + "filter": { + "name": "carbonadod", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug integration test 'file'", + "cargo": { + "args": ["test", "--no-run", "--test=file", "--package=carbonado-node"], + "filter": { + "name": "file", + "kind": "test" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + } + ] +} diff --git a/Cargo.lock b/Cargo.lock index 08d413d..5369bfb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,7 +18,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cipher", "cpufeatures", ] @@ -43,7 +43,7 @@ version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" dependencies = [ - "memchr", + "memchr 2.5.0", ] [[package]] @@ -84,17 +84,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -117,7 +106,7 @@ dependencies = [ "hyper", "itoa", "matchit", - "memchr", + "memchr 2.5.0", "mime", "percent-encoding 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "pin-project-lite", @@ -225,7 +214,7 @@ dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if", + "cfg-if 1.0.0", "constant_time_eq", "digest 0.10.6", "rayon", @@ -269,22 +258,23 @@ checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "carbonado" -version = "0.3.0-rc.8" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83df2d8d7f54171b22b0fa2d1dd3ca0527b21615ff188a3e4edb920afc450e79" +checksum = "de8605d0137ffe3e83f6aeac67011eb68740ca945204d091a2343aa351b06f86" dependencies = [ - "anyhow", "bao", "bech32", "bitmask-enum", + "bytes", "ecies", "hex", "log", - "nom", + "nom 7.1.3", "pretty_env_logger", "secp256k1", "serde", "snap", + "thiserror", "zfec-rs", ] @@ -306,7 +296,10 @@ dependencies = [ "flexi_syslog", "futures-util", "hex", + "infer", "log", + "mime", + "nom 7.1.3", "once_cell", "par-stream", "percent-encoding 2.2.0 (git+https://github.com/cryptoquick/rust-url.git?branch=addl-percent-encode-sets)", @@ -314,10 +307,13 @@ dependencies = [ "rayon", "secp256k1", "serde", + "serde_cbor", "syslog", + "thiserror", "tokio", "toml", "tower-http", + "tree_magic", ] [[package]] @@ -326,6 +322,12 @@ version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -394,6 +396,15 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -431,7 +442,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", @@ -445,7 +456,7 @@ version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crossbeam-utils", ] @@ -455,7 +466,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crossbeam-epoch", "crossbeam-utils", ] @@ -467,7 +478,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 1.0.0", "crossbeam-utils", "memoffset", "scopeguard", @@ -479,7 +490,7 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "crossbeam-utils", ] @@ -489,7 +500,7 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -578,11 +589,11 @@ version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "hashbrown", - "lock_api", + "lock_api 0.4.9", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.7", ] [[package]] @@ -660,12 +671,12 @@ checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "env_logger" -version = "0.7.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" dependencies = [ - "atty", "humantime", + "is-terminal", "log", "regex", "termcolor", @@ -701,6 +712,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "fixedbitset" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" + [[package]] name = "flexi_logger" version = "0.25.3" @@ -840,7 +857,7 @@ dependencies = [ "futures-macro", "futures-sink", "futures-task", - "memchr", + "memchr 2.5.0", "pin-project-lite", "pin-utils", "slab", @@ -862,7 +879,7 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -885,6 +902,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "hashbrown" version = "0.12.3" @@ -897,15 +920,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.2.6" @@ -1019,12 +1033,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "humantime" -version = "1.3.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error", -] +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" @@ -1083,6 +1094,12 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "infer" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb33622da908807a06f9513c19b3c1ad50fab3e4137d82a78107d502075aa199" + [[package]] name = "inout" version = "0.1.3" @@ -1204,6 +1221,15 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + [[package]] name = "lock_api" version = "0.4.9" @@ -1216,12 +1242,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "match_cfg" @@ -1235,6 +1258,15 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" +[[package]] +name = "memchr" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" +dependencies = [ + "libc", +] + [[package]] name = "memchr" version = "2.5.0" @@ -1283,13 +1315,22 @@ dependencies = [ "getrandom", ] +[[package]] +name = "nom" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b" +dependencies = [ + "memchr 1.0.2", +] + [[package]] name = "nom" version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ - "memchr", + "memchr 2.5.0", "minimal-lexical", ] @@ -1385,19 +1426,43 @@ dependencies = [ "futures", "num_cpus", "once_cell", - "parking_lot", + "parking_lot 0.12.1", "pin-project", "tokio", ] +[[package]] +name = "parking_lot" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" +dependencies = [ + "lock_api 0.3.4", + "parking_lot_core 0.7.3", +] + [[package]] name = "parking_lot" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ - "lock_api", - "parking_lot_core", + "lock_api 0.4.9", + "parking_lot_core 0.9.7", +] + +[[package]] +name = "parking_lot_core" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93f386bb233083c799e6e642a9d73db98c24a5deeb95ffc85bf281255dffc98" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall 0.1.57", + "smallvec", + "winapi", ] [[package]] @@ -1406,9 +1471,9 @@ version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", "windows-sys 0.45.0", ] @@ -1424,6 +1489,16 @@ name = "percent-encoding" version = "2.2.0" source = "git+https://github.com/cryptoquick/rust-url.git?branch=addl-percent-encode-sets#c85328f48c4aec76cba87e9562ee571777ade5d8" +[[package]] +name = "petgraph" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "pin-project" version = "1.0.12" @@ -1462,7 +1537,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef234e08c11dfcb2e56f79fd70f6f2eb7f025c0ce2333e82f4f0518ecad30c6" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "opaque-debug", "universal-hash", @@ -1476,9 +1551,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "pretty_env_logger" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" +checksum = "865724d4dbe39d9f3dd3b52b88d859d66bcb2d6a0acfd5ea68a65fb66d4bdc1c" dependencies = [ "env_logger", "log", @@ -1517,12 +1592,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - [[package]] name = "quote" version = "1.0.26" @@ -1584,6 +1653,12 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + [[package]] name = "redox_syscall" version = "0.2.16" @@ -1600,7 +1675,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", - "redox_syscall", + "redox_syscall 0.2.16", "thiserror", ] @@ -1611,7 +1686,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", - "memchr", + "memchr 2.5.0", "regex-syntax", ] @@ -1682,18 +1757,28 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.160" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" dependencies = [ "serde_derive", ] +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half", + "serde", +] + [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", @@ -1748,7 +1833,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.9.0", "opaque-debug", @@ -1760,7 +1845,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest 0.10.6", ] @@ -1811,7 +1896,7 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" dependencies = [ - "lock_api", + "lock_api 0.4.9", ] [[package]] @@ -1945,10 +2030,10 @@ dependencies = [ "autocfg", "bytes", "libc", - "memchr", + "memchr 2.5.0", "mio", "num_cpus", - "parking_lot", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2", @@ -2053,7 +2138,7 @@ version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-core", @@ -2068,6 +2153,19 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tree_magic" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d99367ce3e553a84738f73bd626ccca541ef90ae757fdcdc4cbe728e6cb629" +dependencies = [ + "fnv", + "lazy_static", + "nom 3.2.1", + "parking_lot 0.10.2", + "petgraph", +] + [[package]] name = "try-lock" version = "0.2.4" @@ -2136,7 +2234,7 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen-macro", ] @@ -2353,7 +2451,7 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23d020b441f92996c80d94ae9166e8501e59c7bb56121189dc9eab3bd8216966" dependencies = [ - "memchr", + "memchr 2.5.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml old mode 100644 new mode 100755 index 72de521..0fd3a45 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ rayon = ["blake3/rayon"] [dependencies] anyhow = "1.0.69" axum = "0.6.9" -axum-macros = "0.3.4" +axum-macros = "0.3.3" bao = "0.12.1" blake3 = "1.3.3" bytes = "1.4.0" @@ -22,13 +22,18 @@ carbonado = "0.3.0-rc.8" chrono = "0.4.23" clap = { version = "4.1.8", features = ["derive"] } directories = "5.0.1" +file-format = { version = "0.21", features = ["reader-xml"] } flexi_logger = { version = "0.25.2", features = ["colors"] } flexi_syslog = "0.5.2" futures-util = "0.3.28" hex = "0.4.3" +http = "0.2.9" # human_bytes = "0.4.1" -# infer = "0.13.0" +infer = { version = "0.15.0", default-features = true } log = "0.4.17" +mime = "0.3" +mime_guess = "2.0" +nom = "7.1.3" once_cell = "1.17.1" par-stream = { version = "0.10.2", features = ["runtime-tokio"] } percent-encoding = { version = "2.2.0", git = "https://github.com/cryptoquick/rust-url.git", branch = "addl-percent-encode-sets" } @@ -41,10 +46,13 @@ secp256k1 = { version = "0.27.0", features = [ "serde", ] } serde = { version = "1.0.152", features = ["derive"] } +serde_cbor = "0.11" syslog = "6.0.1" +thiserror = "1.0" tokio = { version = "1.26.0", features = ["full"] } toml = "0.7.2" tower-http = { version = "0.4.0", features = ["cors"] } +tree_magic = { version = "0.2.1" } # [dev-dependencies] # serial_test = "2.0.0" diff --git a/src/backend/fs.rs b/src/backend/fs.rs old mode 100644 new mode 100755 index 1ef0d94..263b058 --- a/src/backend/fs.rs +++ b/src/backend/fs.rs @@ -6,10 +6,13 @@ use std::{ sync::Arc, }; +use tokio::sync::watch; + use anyhow::{anyhow, Error, Result}; use axum::body::Bytes; use bytes::BytesMut; use carbonado::{constants::Format, file::Header, structs::Encoded}; +use chrono::{NaiveDateTime, TimeZone, Utc}; use futures_util::{stream, Stream, StreamExt, TryStreamExt}; use log::{debug, trace}; use par_stream::{ParStreamExt, TryParStreamExt}; @@ -22,6 +25,7 @@ use tokio::sync::Mutex; use crate::{ config::{ensure_pk_dirs_exist, file_path, node_shared_secret, SYS_CFG}, + header::CatHeader, prelude::*, }; @@ -31,6 +35,7 @@ pub async fn write_file<'a>( pk: &Secp256k1PubKey, file_stream: FileStream, name: Option, + mime_type_receiver: watch::Receiver, ) -> Result { trace!("write_file, create a shared secret using ECDH"); let ss = node_shared_secret(&pk.into_inner())?.secret_bytes(); @@ -50,12 +55,32 @@ pub async fn write_file<'a>( trace!("Iterate through file body stream"); let thread_file_hasher = file_hasher.clone(); + let current_mime_type = Arc::new(Mutex::new(mime_type_receiver.borrow().clone())); + + // Clone the receiver to avoid borrowing issues + let mime_type_receiver_clone = mime_type_receiver.clone(); + + // Clone the Arc outside the closure + let current_mime_type_clone = Arc::clone(¤t_mime_type); + let segment_hashes: Vec = file_stream .try_par_then(None, move |segment: Bytes| { - trace!("Process segment"); + let current_mime_type = Arc::clone(¤t_mime_type_clone); + let mut mime_type_receiver = mime_type_receiver_clone.clone(); + + debug!("Process segment"); let thread_file_hasher = thread_file_hasher.clone(); + async move { + if mime_type_receiver.changed().await.is_ok() { + let new_mime_type = mime_type_receiver.borrow().clone(); + if new_mime_type != "application/octet-stream" { + *current_mime_type.lock().await = new_mime_type; + } + } + thread_file_hasher.lock().await.update(&segment); + trace!("Encoding segment"); let encoded_segment = carbonado::encode(&pk_bytes, &segment, NODE_FORMAT)?; trace!("Writing segment"); @@ -79,8 +104,19 @@ pub async fn write_file<'a>( return Err(anyhow!("This file already exists for this public key.")); } + // Access the updated MIME type after the loop + let final_mime_type_str = current_mime_type.lock().await.clone(); + debug!(">>>>>>>>>> Mime_Type has Changed {final_mime_type_str:?}"); + trace!("Append each hash to its catalog"); - write_catalog(&write_pk_str, &file_hash, &segment_hashes, name).await?; + write_catalog( + &write_pk_str, + &file_hash, + name, + &final_mime_type_str, + &segment_hashes, + ) + .await?; debug!("Finished write_file"); Ok(file_hash) @@ -93,6 +129,8 @@ pub fn write_segment(sk: &[u8], pk: &[u8], encoded: &Encoded) -> Result let encoded_chunk_size = encode_info.bytes_verifiable as usize / SYS_CFG.volumes.len(); trace!("Encoded chunk size: {}", encoded_chunk_size); + let metadata = std::option::Option::Some([0_u8; 8]); + encoded_bytes .par_chunks_exact(encoded_chunk_size) .enumerate() @@ -106,7 +144,9 @@ pub fn write_segment(sk: &[u8], pk: &[u8], encoded: &Encoded) -> Result chunk_index as u8, encode_info.output_len, encode_info.padding_len, + metadata, )?; + let header_bytes = header.try_to_vec()?; let file_name = header.file_name(); @@ -132,8 +172,9 @@ pub fn write_segment(sk: &[u8], pk: &[u8], encoded: &Encoded) -> Result pub async fn write_catalog( write_pk_str: &str, file_hash: &Blake3Hash, - segment_hashes: &[BaoHash], name: Option, + mime_type: &str, + segment_hashes: &[BaoHash], ) -> Result<()> { debug!("Write catalog"); let contents: Vec = segment_hashes @@ -145,21 +186,59 @@ pub async fn write_catalog( let file_hash = file_hash.to_string(); let name = name.unwrap_or(file_hash); + let date_utc = Utc.from_utc_datetime(&NaiveDateTime::from_timestamp_opt(61, 0).unwrap()); + let date = date_utc.to_string(); + let mime_type = mime_type.to_string(); + + debug!("Write catalog mime_type to meda-data {}", mime_type); + + // HEADER METADATA + let cat_data = CborData { + name: name.to_string(), + date, + mime_type, + }; + + // Serialize to CBOR + let cbor_data = serde_cbor::to_vec(&cat_data)?; + let length = cbor_data.len(); + stream::iter(0..SYS_CFG.volumes.len()) .par_map(None, move |volume_index| { let write_pk_str = write_pk_str.clone(); - let name = name.clone(); + let contents = contents.clone(); + let name = name.clone(); + let cbor_len = length as u8; + let cbor_data = cbor_data.clone(); + let metadata = Some(cbor_data.clone()); + + // convert to + let my_array_option: Option<[u8; 8]> = metadata.and_then(vec_to_array); + let metadata = my_array_option; + move || { + let cat_header = CatHeader { cbor_len, metadata }; + let cat_header_bytes = cat_header.try_to_vec()?; + trace!("Get catalogs directory path"); let path = file_path(volume_index, &write_pk_str, CATALOG_DIR, &name)?; + // MAKE THE ENDPOINT AVAILABLE + trace!(">>>> ENDPOINTS <<<<<"); + debug!("Open catalog file at {path:?}"); + // JSON FORMATTED CBOR DATA ENDPOINT + let decoded_cbor_data: CborData = serde_cbor::from_slice(&cbor_data)?; + debug!(">>> JSON DATA decoded_cbpr_data; {:?}", decoded_cbor_data); + trace!("Open catalog file at {path:?}"); let mut file = OpenOptions::new() .create_new(true) .write(true) .open(&path)?; + file.write_all(&cat_header_bytes)?; + trace!("Write file contents"); file.write_all(&contents)?; file.flush()?; @@ -175,18 +254,20 @@ pub async fn write_catalog( } pub fn read_file(pk: &Secp256k1PubKey, lookup: &Lookup) -> Result { - debug!("Read file wiht lookup: {lookup}"); - trace!("Create a shared secret using ECDH"); let ss = node_shared_secret(&pk.into_inner())?.secret_bytes(); let write_pk = PublicKey::from_secret_key_global(&SecretKey::from_slice(&ss)?); let write_pk_str = write_pk.to_string(); - trace!("Read catalog file bytes, parse out each hash, plus the segment Carbonado format"); + trace!( + ">>>>>> Read catalog file bytes, parse out each hash, plus the segment Carbonado format" + ); let catalog_file = read_catalog(&write_pk_str, lookup)?; + let catalog_file_clone = catalog_file.clone(); + trace!("For each hash, read each chunk into a segment, then decode that segment"); - let file_bytes: FileStream = stream::iter(catalog_file) + let file_bytes: FileStream = stream::iter(catalog_file_clone) .par_then(None, move |segment_hash| { let write_pk_str = write_pk_str.clone(); @@ -209,15 +290,13 @@ pub fn read_file(pk: &Secp256k1PubKey, lookup: &Lookup) -> Result { let write_pk_str = write_pk_str.clone(); let segment_hash = segment_hash.clone(); move || { - trace!("Get catalogs directory path"); let path = file_path( volume_index, &write_pk_str, SEGMENT_DIR, - &segment_hash, + &format!("{}.c{}", segment_hash, NODE_FORMAT), )?; - trace!("Read segment file at {path:?}"); let mut file = OpenOptions::new().read(true).open(path)?; let mut bytes = vec![]; @@ -272,9 +351,13 @@ pub fn read_catalog(write_pk_str: &str, lookup: &Lookup) -> Result> let mut file = OpenOptions::new().read(true).open(path)?; let mut bytes = vec![]; + file.read_to_end(&mut bytes)?; - let bao_hashes = bytes + // Split the CatHeader bytes from the content bytes + let (_cat_header, content_bytes) = bytes.split_at(25); + + let bao_hashes = content_bytes .chunks_exact(bao::HASH_SIZE) .map(BaoHash::try_from) .collect::>>()?; @@ -323,32 +406,12 @@ fn remove_dir_catalogs(path: PathBuf, file: PathBuf) -> io::Result<()> { Ok(()) } -// fn remove_dir_segements>(path: P, seg_file: PathBuf) -> io::Result<()> { -// trace!(">>> remove_Segment_contents"); -// for entry in fs::read_dir(path)? { -// let entry = entry?; -// trace!("ENTRY Delete SEGMENT File at {:?}", entry); - -// match &entry { -// seg_file => { -// fs::remove_file(seg_file.path())?; -// trace!("Delete Segment File at {:?}", seg_file); -// } -// } -// } -// Ok(()) -// } - -// fn remove_dir_catalogs(path: PathBuf, file: PathBuf) -> io::Result<()> { -// for entry in fs::read_dir(path)? { -// let entry = entry?; -// trace!("ENTRY Delete CATALOG File at {:?}", entry); -// match &entry { -// file => { -// fs::remove_file(file.path())?; -// trace!("FILE MATCH Delete CATALOG File at {:?}", file); -// } -// } -// } -// Ok(()) -// } +fn vec_to_array(vec: Vec) -> Option<[u8; 8]> { + if vec.len() == 8 { + let mut array = [0u8; 8]; + array.copy_from_slice(&vec); + Some(array) + } else { + None + } +} diff --git a/src/constants.rs b/src/constants.rs index e27fefc..d6beaf9 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -1,4 +1,8 @@ pub const NODE_FORMAT: u8 = 15; pub const SEGMENT_SIZE: usize = 1024 * 1024; + pub const SEGMENT_DIR: &str = "segments"; pub const CATALOG_DIR: &str = "catalogs"; + +/// "Magic number" used by the Carbonado-node file format. 12 bytes: "CAT_MAGICNO", and a version, 00, plus a newline character +pub const CAT_MAGICNO: [u8; 16] = *b"CARBONADONODE00\n"; diff --git a/src/errors.rs b/src/errors.rs new file mode 100644 index 0000000..26f5cd6 --- /dev/null +++ b/src/errors.rs @@ -0,0 +1,8 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum CarbonadoNodeError { + /// Invalid magic number + #[error("File header lacks Carbonado magic number and may not be a proper Carbonado file. Magic number found was {0}.")] + CatInvalidMagicNumber(String), +} diff --git a/src/frontend/http.rs b/src/frontend/http.rs old mode 100644 new mode 100755 index 18bf05f..d1411f0 --- a/src/frontend/http.rs +++ b/src/frontend/http.rs @@ -1,4 +1,6 @@ +use std::sync::Arc; use std::{net::SocketAddr, str::FromStr}; +use tokio::sync::watch; use anyhow::{anyhow, Result}; use axum::{ @@ -10,8 +12,10 @@ use axum::{ Router, }; use bytes::BytesMut; +use file_format::FileFormat; + use futures_util::{stream, StreamExt}; -use log::{debug, info, trace}; +use log::{debug, info}; use percent_encoding::{percent_decode_str, utf8_percent_encode, FORM}; use secp256k1::PublicKey; use tower_http::cors::CorsLayer; @@ -25,31 +29,64 @@ use crate::{ async fn write_file_handler(pk: &str, body: BodyStream, name: Option) -> Result { let pk = &Secp256k1PubKey::try_from(pk)?; + let name_clone = name.clone().unwrap(); + let extension = name_clone.split('.').last().unwrap_or_default(); + if extension.is_empty() { + debug!(">>>>>> NO EXETENSION FROM NAME {}", extension); + } else { + debug!(">>>>>> EXETENSION FROM NAME {}", extension); + } + + // Create a watch channel for MIME type updates + let (mime_type_sender, mime_type_receiver) = watch::channel("init_mime_type".to_string()); + + // Wrap the sender in an Arc + let mime_type_sender = Arc::new(mime_type_sender); + + // Process the file stream and determine the MIME type let file_stream: FileStream = stream::try_unfold( (body, BytesMut::with_capacity(SEGMENT_SIZE * 2)), - |(mut body_stream, mut remainder)| async { - while remainder.len() < SEGMENT_SIZE { - if let Some(chunk) = body_stream.next().await { + move |(mut body_stream, mut remainder)| { + let mime_type_sender = mime_type_sender.clone(); + async move { + // Stream processing logic... + while let Some(chunk) = body_stream.next().await { let bytes = chunk?; - remainder.extend(bytes); + remainder.extend_from_slice(&bytes); + + let format = FileFormat::from_bytes(&remainder); + let stage_mime_type = format.media_type().to_string(); + + //let mime_type = stage_mime_type; + let _ = mime_type_sender.send(stage_mime_type.clone()); + + // Determine MIME type and store in shared state if remainder.len() >= SEGMENT_SIZE { let segment = remainder.split_to(SEGMENT_SIZE); - trace!("Stream 1MB segment"); return Ok(Some((segment.freeze(), (body_stream, remainder)))); } - } else { - trace!("No more segments in Body Stream"); - return Ok(None); } + // Handle the case where the stream has ended but there's unprocessed data + if !remainder.is_empty() { + let final_chunk = remainder.split_to(remainder.len()); + + // Ensure MIME type is sent for the final chunk + let format = FileFormat::from_bytes(&final_chunk); + let final_mime_type = format.media_type().to_string(); + let _ = mime_type_sender.send(final_mime_type); + + return Ok(Some((final_chunk.freeze(), (body_stream, BytesMut::new())))); + } + + Ok(None) } - trace!("Unexpected while short-circuit"); - Ok(None) }, ) .boxed(); - let Blake3Hash(hash) = write_file(pk, file_stream, name).await?; + // Call write_file with the receiver part of the channel + let Blake3Hash(hash) = write_file(pk, file_stream, name, mime_type_receiver).await?; Ok(hash.to_hex().to_string()) } diff --git a/src/header.rs b/src/header.rs new file mode 100644 index 0000000..5bc7e61 --- /dev/null +++ b/src/header.rs @@ -0,0 +1,144 @@ +use crate::errors::CarbonadoNodeError; +use log::debug; +use serde::{Deserialize, Serialize}; +use serde_cbor; +use serde_cbor::Error; + +use bytes::Bytes; +use nom::{ + bytes::complete::take, + number::complete::le_u8, // if cbor length is actually u32 + IResult, +}; + +use crate::constants::CAT_MAGICNO; + +/// Contains deserialized copies of the data kept in the Catalog header. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct CatHeader { + /// Number of bytes added to pad to align zfec chunks and bao slices. + pub cbor_len: u8, // this appears to sbe u32 not u8 + /// Bytes that are normally zero but can contain extra data if needed + pub metadata: Option<[u8; 8]>, +} + +impl TryFrom<&[u8]> for CatHeader { + type Error = CarbonadoNodeError; + + /// Attempts to decode a cat_header from a file. + fn try_from(bytes: &[u8]) -> Result { + let cat_magic_no = [0_u8; 16]; + + // easily get cbor_length back from header to know how far to read + let (_, (cbor_len, metadata)) = CatHeader::parse_bytes(bytes).unwrap(); + + // create throw error for wrong magic number + if cat_magic_no != CAT_MAGICNO { + return Err(CarbonadoNodeError::CatInvalidMagicNumber(format!( + "{cat_magic_no:#?}" + ))); + } + + Ok(CatHeader { cbor_len, metadata }) + } +} + +impl TryFrom for CatHeader { + type Error = CarbonadoNodeError; + + /// Attempts to decode a header from a file. + fn try_from(bytes: Bytes) -> Result { + let cat_magic_no = [0_u8; 16]; + + let (_, (cbor_len, metadata)) = CatHeader::parse_bytes(&bytes).unwrap(); + + if cat_magic_no != CAT_MAGICNO { + return Err(CarbonadoNodeError::CatInvalidMagicNumber(format!( + "{cat_magic_no:#?}" + ))); + } + + Ok(CatHeader { cbor_len, metadata }) + } +} + +impl CatHeader { + /// 160 bytes should be added for Carbonado headers. + pub fn len() -> usize { + 12 + 33 + 32 + 64 + 1 + 1 + 4 + 4 + 8 + 1 + } + + /// Creates a new Catalog Header struct using the provided parameters. + #[allow(clippy::too_many_arguments)] + pub fn new(metadata: [u8; 8]) -> Result { + //type Error = CarbonadoError; + let cbor_data = serde_cbor::to_vec(&metadata)?; + println!("cbor data; {:?}", cbor_data); + + let cbor_leng = cbor_data.len() as u8; + + // cbor length is header length + println!("cbor length ; {:?}", cbor_leng); + + Ok(CatHeader { + cbor_len: cbor_leng, + metadata: if metadata.iter().any(|b| b != &0) { + Some(metadata) + } else { + None + }, + }) + } + + /// Creates a header to be prepended to files. + pub fn try_to_vec(&self) -> Result, CarbonadoNodeError> { + let mut cbor_bytes = self.cbor_len.to_le_bytes().to_vec(); // 2 bytes + let mut cbor_padding = if let Some(metadata) = &self.metadata { + // debug!( + // ">>> SELF METADATA IF STATEMENT : {:?}", + // &metadata.to_vec() + // ); + + metadata.to_vec() + } else { + debug!("### ERROR ELSE STATEMENT [0_u8; 8]"); + vec![0_u8; 8] + }; + + debug!(">>>> CBORE BYTES LENGTH : {}", self.cbor_len); + + let mut cat_header = Vec::new(); + + cat_header.append(&mut CAT_MAGICNO.to_vec()); // 12 bytes + cat_header.append(&mut cbor_bytes); + cat_header.append(&mut cbor_padding); + + // if header.len() != CatHeader::len() { + // return Err(CarbonadoError::InvalidHeaderLength); + // } + + Ok(cat_header) + } + + #[allow(clippy::type_complexity)] + fn parse_bytes(b: &[u8]) -> IResult<&[u8], (u8, Option<[u8; 8]>)> { + let (b, cbor_len) = le_u8(b)?; + let (b, metadata) = take(8u8)(b)?; + + let metadata: [u8; 8] = metadata.try_into().expect("8 bytes = 8 bytes"); + let metadata = if metadata.iter().any(|b| b != &0) { + Some(metadata) + } else { + None + }; + + Ok((b, (cbor_len, metadata))) + } +} + +pub fn encode_cbor(cat_header: Option<[u8; 8]>) -> Result<(), Error> { + // Serialize to CBOR + let cbor_data = serde_cbor::to_vec(&cat_header)?; + println!("cbor data; {:?}", cbor_data); + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 9b2732d..a04f426 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,7 +7,9 @@ use crate::frontend::http; pub mod backend; pub mod config; pub mod constants; +pub mod errors; pub mod frontend; +pub mod header; pub mod structs; pub mod prelude { diff --git a/src/structs.rs b/src/structs.rs index f386d91..8293289 100644 --- a/src/structs.rs +++ b/src/structs.rs @@ -3,6 +3,8 @@ use std::{ str::FromStr, }; +use serde::{Deserialize, Serialize}; + use anyhow::{Error, Result}; #[derive(Clone, Debug)] @@ -76,7 +78,7 @@ impl Display for Lookup { Hash::Blake3Bytes(_) => todo!(), Hash::BaoBytes(_) => todo!(), Hash::Blake3(blake3_hash) => f.write_str(&blake3_hash.to_string()), - Hash::Bao(_) => todo!(), + Hash::Bao(hash) => f.write_str(&hash.to_string()), }, Lookup::Name(name) => f.write_str(name), } @@ -121,3 +123,13 @@ pub enum PubKey { Secp256k1Bytes(Box<[u8]>), Secp256k1(secp256k1::PublicKey), } + +#[derive(Serialize, Deserialize, Debug, PartialEq)] + +pub struct CborData { + /// Number of bytes added to pad to align zfec chunks and bao slices. + pub name: String, + pub date: String, + pub mime_type: String, + // pub metadata: Option>, +} diff --git a/start.sh b/start.sh new file mode 100755 index 0000000..537a1b0 --- /dev/null +++ b/start.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +echo "Start Carbonado-Node" +cargo run start + diff --git a/tests/file.rs b/tests/file.rs old mode 100644 new mode 100755 index ce76168..11a6c19 --- a/tests/file.rs +++ b/tests/file.rs @@ -1,4 +1,6 @@ +use file_format::FileFormat; use std::fs; +use tokio::sync::watch; use anyhow::Result; use axum::body::Bytes; @@ -9,6 +11,7 @@ use carbonado_node::{ prelude::SEGMENT_SIZE, structs::{Hash, Lookup, Secp256k1PubKey}, }; + use futures_util::{stream, StreamExt, TryStreamExt}; use log::{debug, info}; use rand::thread_rng; @@ -20,6 +23,8 @@ const RUST_LOG: &str = "carbonado_node=trace,carbonado=trace,file=trace"; async fn write_read() -> Result<()> { carbonado::utils::init_logging(RUST_LOG); + let (mime_type_sender, mime_type_receiver) = watch::channel("init_mime_type".to_string()); + let (_sk, pk) = generate_keypair(&mut thread_rng()); // TODO: Use after remove_file is finished: // let pk = @@ -33,6 +38,12 @@ async fn write_read() -> Result<()> { let file_len = file_bytes.len(); debug!("{} Write Delete:: bytes read", file_len); + let format = FileFormat::from_bytes(&file_bytes); + let stage_mime_type = format.media_type().to_string(); + + //let mime_type = stage_mime_type; + let _ = mime_type_sender.send(stage_mime_type.clone()); + info!("Writing file"); let (x_only, _) = write_pk.x_only_public_key(); @@ -43,9 +54,10 @@ async fn write_read() -> Result<()> { .map(|chunk| Ok(Bytes::from(chunk))) .boxed(); - let blake3_hash = write_file(&Secp256k1PubKey(pk), file_stream, None).await?; + let blake3_hash = + write_file(&Secp256k1PubKey(pk), file_stream, None, mime_type_receiver).await?; - info!("Reading file"); + info!("Reading file, {}", blake3_hash); let decoded_file: Vec = read_file( &Secp256k1PubKey(pk), @@ -83,10 +95,18 @@ async fn write_read() -> Result<()> { async fn read_write_delete_file() -> Result<()> { carbonado::utils::init_logging(RUST_LOG); + let (mime_type_sender, mime_type_receiver) = watch::channel("init_mime_type".to_string()); + info!("Write Delete:: Reading file bytes"); let file_bytes = fs::read("tests/samples/cat.gif")?; debug!("{} Write Delete:: bytes read", file_bytes.len()); + let format = FileFormat::from_bytes(&file_bytes); + let stage_mime_type = format.media_type().to_string(); + + //let mime_type = stage_mime_type; + let _ = mime_type_sender.send(stage_mime_type.clone()); + let file_stream = stream::iter(file_bytes.clone()) .chunks(1024 * 1024) .map(|chunk| Ok(Bytes::from(chunk))) @@ -96,9 +116,14 @@ async fn read_write_delete_file() -> Result<()> { let (_sk, pk) = generate_keypair(&mut thread_rng()); // info!("Write Delete:: Writing file if not exists in order to test delete"); - let file_did_write = write_file(&Secp256k1PubKey(pk), file_stream, None) - .await - .is_ok(); + let file_did_write = write_file( + &Secp256k1PubKey(pk), + file_stream, + None, + mime_type_receiver.clone(), + ) + .await + .is_ok(); if file_did_write { info!( @@ -116,7 +141,7 @@ async fn read_write_delete_file() -> Result<()> { .boxed(); info!("Write Delete:: Writing file if not exists in order to test delete"); - let blake3_hash = write_file(&Secp256k1PubKey(pk), file_stream, None) + let blake3_hash = write_file(&Secp256k1PubKey(pk), file_stream, None, mime_type_receiver) .await .is_ok();