From 6d7869355e96db3c49d9ac3e19138b51c80b53b0 Mon Sep 17 00:00:00 2001 From: Pasha Date: Thu, 15 Feb 2024 11:27:07 +0300 Subject: [PATCH] [feat] Development --- Cargo.lock | 238 ++++++++++++++++------------------- Cargo.toml | 2 +- bin/main.rs | 12 ++ src/config.rs | 5 + src/iroh_node.rs | 21 +++- src/storages/fs_storage.rs | 142 ++++++++++++--------- src/storages/iroh_storage.rs | 22 +++- src/storages/mirroring.rs | 17 ++- src/storages/mod.rs | 27 ++-- trident-py/pyproject.toml | 2 +- trident-py/trident/client.py | 5 +- 11 files changed, 270 insertions(+), 223 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b8f1a46..068544e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,9 +92,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" +checksum = "2faccea4cc4ab4a667ce676a30e8ec13922a692c99bb8f5b11f1502c72e04220" [[package]] name = "anstyle-parse" @@ -234,12 +234,11 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91bb1df3e5f0e47475498570cdd10ab0370bc1d7af3151aa0161d5f5876b6908" +checksum = "8b30c39ebe61f75d1b3785362b1586b41991873c9ab3e317a9181c246fb71d82" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-sdk-sso", "aws-sdk-ssooidc", @@ -265,9 +264,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d67c6836a1009b23e3f4cd1457c83e0aa49a490d9c3033b53c3f7b8cf2facc0f" +checksum = "33cc49dcdd31c8b6e79850a179af4c367669150c7ac0135f176c61bec81a70f7" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -275,30 +274,13 @@ dependencies = [ "zeroize", ] -[[package]] -name = "aws-http" -version = "0.60.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a081f0c4576f7549dd255987d2d23920eccaa90cdd21d6440f91e0d7537f0e0d" -dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "bytes", - "http 0.2.11", - "http-body 0.4.6", - "pin-project-lite", - "tracing", -] - [[package]] name = "aws-runtime" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab7bf4b9b083e6dc86e2bb4fb09a4daca97e12cc0bc174a6f51fe624c23aa87f" +checksum = "eb031bff99877c26c28895766f7bb8484a05e24547e370768d6cc9db514662aa" dependencies = [ "aws-credential-types", - "aws-http", "aws-sigv4", "aws-smithy-async", "aws-smithy-eventstream", @@ -306,21 +288,23 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", + "bytes", "fastrand", "http 0.2.11", + "http-body 0.4.6", "percent-encoding", + "pin-project-lite", "tracing", "uuid", ] [[package]] name = "aws-sdk-s3" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f94d9842a304e066d1eddea61c703fb217ce5a1063cc087d07c8de1db4535f7" +checksum = "951f7730f51a2155c711c85c79f337fbc02a577fa99d2a0a8059acfce5392113" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-sigv4", "aws-smithy-async", @@ -345,12 +329,11 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "865b1b85249196e4868a48a4b6683d60e19b82371b869297bc9982bc9f47de4b" +checksum = "f486420a66caad72635bc2ce0ff6581646e0d32df02aa39dc983bfe794955a5b" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -368,12 +351,11 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beadaf5c48f5d923329ec87b7db6eea5e4ce2af4cc9f96d9f85d520f34573f40" +checksum = "39ddccf01d82fce9b4a15c8ae8608211ee7db8ed13a70b514bbfe41df3d24841" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -391,12 +373,11 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262a6d40c8e11eea633ba541a39745cebfcee9c1683020cee63413048b5c6188" +checksum = "1a591f8c7e6a621a501b2b5d2e88e1697fcb6274264523a6ad4d5959889a41ce" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", @@ -415,9 +396,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "511879249616f30e30fd2fa81edb4833784f65dd5d56053b7de2e2bcb583dda7" +checksum = "c371c6b0ac54d4605eb6f016624fb5c7c2925d315fdf600ac1bf21b19d5f1742" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -444,9 +425,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eac0bb78e9e2765699999a02d7bfb4e6ad8f13e0962ebb9f5202b1d8cd76006" +checksum = "72ee2d09cce0ef3ae526679b522835d63e75fb427aca5413cd371e490d52dcc6" dependencies = [ "futures-util", "pin-project-lite", @@ -455,9 +436,9 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.3" +version = "0.60.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "535a2d5f1e459bc7709580a77152c8d493982db083236c2b1d1c51dc6217e8a3" +checksum = "be2acd1b9c6ae5859999250ed5a62423aedc5cf69045b844432de15fa2f31f2b" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -476,9 +457,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.3" +version = "0.60.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "682371561562d08ab437766903c6bc28f4f95d7ab2ecfb389bda7849dd98aefe" +checksum = "e6363078f927f612b970edf9d1903ef5cef9a64d1e8423525ebb1f0a1633c858" dependencies = [ "aws-smithy-types", "bytes", @@ -487,9 +468,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.3" +version = "0.60.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "365ca49744b2bda2f1e2dc03b856da3fa5a28ca5b0a41e41d7ff5305a8fae190" +checksum = "dab56aea3cd9e1101a0a999447fb346afb680ab1406cebc44b32346e25b4117d" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -508,18 +489,18 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.60.3" +version = "0.60.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733ccdb727ac63370836aa3b3c483d75ad2ef7bc6507db3efe1d01e8d2e50367" +checksum = "fd3898ca6518f9215f62678870064398f00031912390efd03f1f6ef56d83aa8e" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-query" -version = "0.60.3" +version = "0.60.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aff02ae2ee7968bbce2983ffb5ce529d24f4848532300f398347bde8c2196974" +checksum = "bda4b1dfc9810e35fba8a620e900522cd1bd4f9578c446e82f49d1ce41d2e9f9" dependencies = [ "aws-smithy-types", "urlencoding", @@ -527,9 +508,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ab9cb6fee50680af8ceaa293ae79eba32095ca117161cb323f9ee30dd87d139" +checksum = "fafdab38f40ad7816e7da5dec279400dd505160780083759f01441af1bbb10ea" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -552,9 +533,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02ca2da7619517310bfead6d18abcdde90f1439224d887d608503cfacff46dff" +checksum = "c18276dd28852f34b3bf501f4f3719781f4999a51c7bff1a5c6dc8c4529adc29" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -568,9 +549,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4bb944488536cd2fef43212d829bc7e9a8bfc4afa079d21170441e7be8d2d0" +checksum = "bb3e134004170d3303718baa2a4eb4ca64ee0a1c0a7041dca31b38be0fb414f3" dependencies = [ "base64-simd", "bytes", @@ -591,18 +572,18 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.3" +version = "0.60.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef796feaf894d7fd03869235237aeffe73ed1b29a3927cceeee2eecadf876eba" +checksum = "8604a11b25e9ecaf32f9aa56b9fe253c5e2f606a3477f0071e96d3155a5ed218" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2739d97d47f47cdf0d27982019a405dcc736df25925d1a75049f1faa79df88" +checksum = "789bbe008e65636fe1b6dbbb374c40c8960d1232b96af5ff4aec349f9c4accf4" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -696,14 +677,14 @@ dependencies = [ [[package]] name = "bao-tree" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "155e7e0c896695a9049badd7bf2b915d29230e24dc82a7c7ef065eded072404f" +checksum = "96c194b6b40b5abe645e9fc54a62b8247dbd86815443c0d0e10a89faa1218c10" dependencies = [ "bytes", "futures", "iroh-blake3", - "iroh-io", + "iroh-io 0.4.0", "positioned-io", "range-collections", "self_cell", @@ -729,15 +710,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" -[[package]] -name = "base64" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" -dependencies = [ - "byteorder", -] - [[package]] name = "base64" version = "0.13.1" @@ -873,9 +845,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41daef31d7a747c5c847246f36de49ced6f7403b4cdabc807a97b5cc184cda7a" +checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1988,7 +1960,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.11", - "indexmap 2.1.0", + "indexmap 2.2.1", "slab", "tokio", "tokio-util", @@ -2007,7 +1979,7 @@ dependencies = [ "futures-sink", "futures-util", "http 1.0.0", - "indexmap 2.1.0", + "indexmap 2.2.1", "slab", "tokio", "tokio-util", @@ -2346,9 +2318,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "433de089bd45971eecf4668ee0ee8f4cec17db4f8bd8f7bc3197a6ce37aa7d9b" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -2416,7 +2388,7 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "iroh" version = "0.12.0" -source = "git+https://github.com/n0-computer/iroh?branch=perf-startup#4fbb3bf064ea51ef4127147aa6d70427da4ad4eb" +source = "git+https://github.com/n0-computer/iroh?branch=main#4615915f9cb17ca95c39d51e720c74996ce63427" dependencies = [ "anyhow", "bao-tree", @@ -2440,7 +2412,7 @@ dependencies = [ "iroh-base", "iroh-bytes", "iroh-gossip", - "iroh-io", + "iroh-io 0.4.0", "iroh-metrics", "iroh-net", "iroh-sync", @@ -2475,7 +2447,7 @@ dependencies = [ [[package]] name = "iroh-base" version = "0.12.0" -source = "git+https://github.com/n0-computer/iroh?branch=perf-startup#4fbb3bf064ea51ef4127147aa6d70427da4ad4eb" +source = "git+https://github.com/n0-computer/iroh?branch=main#4615915f9cb17ca95c39d51e720c74996ce63427" dependencies = [ "anyhow", "bao-tree", @@ -2483,7 +2455,6 @@ dependencies = [ "hex", "multibase", "postcard", - "redb", "serde", "serde-error", "thiserror", @@ -2505,7 +2476,7 @@ dependencies = [ [[package]] name = "iroh-bytes" version = "0.12.0" -source = "git+https://github.com/n0-computer/iroh?branch=perf-startup#4fbb3bf064ea51ef4127147aa6d70427da4ad4eb" +source = "git+https://github.com/n0-computer/iroh?branch=main#4615915f9cb17ca95c39d51e720c74996ce63427" dependencies = [ "anyhow", "bao-tree", @@ -2518,15 +2489,16 @@ dependencies = [ "genawaiter", "hex", "iroh-base", - "iroh-io", + "iroh-io 0.4.0", + "iroh-metrics", + "iroh-net", "num_cpus", "once_cell", - "paths-as-strings", + "parking_lot", "postcard", "quinn", "rand", "range-collections", - "redb", "reflink-copy", "self_cell", "serde", @@ -2542,7 +2514,7 @@ dependencies = [ [[package]] name = "iroh-gossip" version = "0.12.0" -source = "git+https://github.com/n0-computer/iroh?branch=perf-startup#4fbb3bf064ea51ef4127147aa6d70427da4ad4eb" +source = "git+https://github.com/n0-computer/iroh?branch=main#4615915f9cb17ca95c39d51e720c74996ce63427" dependencies = [ "anyhow", "bytes", @@ -2551,7 +2523,7 @@ dependencies = [ "ed25519-dalek", "futures", "genawaiter", - "indexmap 2.1.0", + "indexmap 2.2.1", "iroh-base", "iroh-blake3", "iroh-metrics", @@ -2580,10 +2552,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "iroh-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd67e386f948a6f09e71057b48fff51b6414f0080997495b5bdf2d1bdcdbe46" +dependencies = [ + "bytes", + "futures", + "pin-project", + "smallvec", + "tokio", +] + [[package]] name = "iroh-metrics" version = "0.12.0" -source = "git+https://github.com/n0-computer/iroh?branch=perf-startup#4fbb3bf064ea51ef4127147aa6d70427da4ad4eb" +source = "git+https://github.com/n0-computer/iroh?branch=main#4615915f9cb17ca95c39d51e720c74996ce63427" dependencies = [ "anyhow", "erased_set", @@ -2603,7 +2588,7 @@ dependencies = [ [[package]] name = "iroh-net" version = "0.12.0" -source = "git+https://github.com/n0-computer/iroh?branch=perf-startup#4fbb3bf064ea51ef4127147aa6d70427da4ad4eb" +source = "git+https://github.com/n0-computer/iroh?branch=main#4615915f9cb17ca95c39d51e720c74996ce63427" dependencies = [ "aead", "anyhow", @@ -2678,7 +2663,7 @@ dependencies = [ [[package]] name = "iroh-sync" version = "0.12.0" -source = "git+https://github.com/n0-computer/iroh?branch=perf-startup#4fbb3bf064ea51ef4127147aa6d70427da4ad4eb" +source = "git+https://github.com/n0-computer/iroh?branch=main#4615915f9cb17ca95c39d51e720c74996ce63427" dependencies = [ "anyhow", "bytes", @@ -2714,9 +2699,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -2798,9 +2783,9 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "lru" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2994eeba8ed550fd9b47a0b38f0242bc3344e496483c6180b69139cc2fa5d1d7" +checksum = "db2c024b41519440580066ba82aab04092b333e09066a5eb86c7c4890df31f22" dependencies = [ "hashbrown 0.14.3", ] @@ -2963,9 +2948,9 @@ dependencies = [ [[package]] name = "netlink-proto" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "842c6770fc4bb33dd902f41829c61ef872b8e38de1405aa0b938b27b8fba12c3" +checksum = "86b33524dc0968bfad349684447bfce6db937a9ac3332a1fe60c0c5a5ce63f21" dependencies = [ "bytes", "futures", @@ -3322,15 +3307,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" -[[package]] -name = "paths-as-strings" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7963201bed34799ddd0f54a2e1e4c5082313ca260db118822109f344f13b285c" -dependencies = [ - "base64 0.10.1", -] - [[package]] name = "pem" version = "2.0.1" @@ -3413,18 +3389,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", @@ -3693,9 +3669,9 @@ dependencies = [ [[package]] name = "prometheus-client" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "510c4f1c9d81d556458f94c98f857748130ea9737bbd6053da497503b26ea63c" +checksum = "6f87c10af16e0af74010d2a123d202e8363c04db5acfa91d8747f64a8524da3a" dependencies = [ "dtoa", "itoa", @@ -3963,7 +3939,7 @@ checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.4", + "regex-automata 0.4.5", "regex-syntax 0.8.2", ] @@ -3978,9 +3954,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", @@ -4385,9 +4361,9 @@ checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" [[package]] name = "serde" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" dependencies = [ "serde_derive", ] @@ -4412,9 +4388,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", @@ -4423,11 +4399,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.1", "itoa", "ryu", "serde", @@ -4466,11 +4442,11 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.30" +version = "0.9.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1bf28c79a99f70ee1f1d83d10c875d2e70618417fda01ad1785e027579d9d38" +checksum = "adf8a49373e98a4c5f0ceb5d05aa7c648d75f63774981ed95b7c7443bbd50c6e" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.1", "itoa", "ryu", "serde", @@ -5111,7 +5087,7 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.1", "serde", "serde_spanned", "toml_datetime", @@ -5249,7 +5225,7 @@ dependencies = [ "clap", "futures", "iroh", - "iroh-io", + "iroh-io 0.3.0", "lru", "md5", "percent-encoding", @@ -5831,9 +5807,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.34" +version = "0.5.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cf47b659b318dccbd69cc4797a39ae128f533dce7902a1096044d1967b9c16" +checksum = "1931d78a9c73861da0134f453bb1f790ce49b2e30eba8410b4b79bac72b46a2d" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index b73a8e3..76d04d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ aws-credential-types = { version = "1.0", features = ["hardcoded-credentials"] } axum = "0.7" bisection = "0.1" clap = { version = "4.4", features = ["derive"] } -iroh = { version = "0.12.0", branch = "perf-startup", git = "https://github.com/n0-computer/iroh", features = [ "flat-db", "metrics" ] } +iroh = { version = "0.12.0", branch = "main", git = "https://github.com/n0-computer/iroh", features = [ "flat-db", "metrics" ] } iroh-io = { version = "0.3.0" } md5 = "0.7" percent-encoding = "2.3.1" diff --git a/bin/main.rs b/bin/main.rs index 35094e9..a4ebb7f 100644 --- a/bin/main.rs +++ b/bin/main.rs @@ -10,6 +10,7 @@ use axum::{ }; use clap::Parser; use futures::TryStreamExt; +use iroh::sync::store::DownloadPolicy; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; @@ -18,6 +19,7 @@ use tokio::sync::RwLock; use tokio_util::io::{ReaderStream, StreamReader}; use tower_http::trace::{self, TraceLayer}; use tracing::Level; +use tracing_subscriber::EnvFilter; use trident_storage::config::{ load_config, save_config, Config, MirroringConfig, SinkConfig, TableConfig, }; @@ -43,6 +45,7 @@ struct TablesCreateRequest { #[derive(Deserialize)] struct TablesImportRequest { ticket: String, + download_policy: DownloadPolicy, storage: Option, mirroring: Option, } @@ -132,8 +135,16 @@ fn main() -> Result<(), Error> { // initialize tracing tracing_subscriber::fmt() .with_target(false) + .with_env_filter(EnvFilter::from_default_env()) .compact() .init(); + /*console_subscriber::ConsoleLayer::builder() + // set how long the console will retain data from completed tasks + .retention(Duration::from_secs(120)) + // set the address the server is bound to + .server_addr(([0, 0, 0, 0], 6669)) + // ... other configurations ... + .init();*/ Builder::new_multi_thread() .enable_all() .build() @@ -205,6 +216,7 @@ async fn tables_import( .tables_import( &table, &tables_import_request.ticket, + tables_import_request.download_policy, tables_import_request.storage.as_deref(), tables_import_request.mirroring, ) diff --git a/src/config.rs b/src/config.rs index 41d219e..7d39ddf 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,4 +1,5 @@ use crate::error::Error; +use iroh::sync::store::DownloadPolicy; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::path::PathBuf; @@ -17,6 +18,8 @@ pub enum StorageEngineConfig { pub struct FSStorageEngineConfig { pub replicas: u8, pub fs_shards: Vec, + #[serde(default = "return_false")] + pub is_import_missing_enabled: bool, } #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] @@ -57,6 +60,8 @@ pub struct MirroringConfig { #[derive(Clone, Debug, Serialize, Deserialize)] pub struct TableConfig { pub id: String, + #[serde(default = "DownloadPolicy::default")] + pub download_policy: DownloadPolicy, #[serde(skip_serializing_if = "Option::is_none")] pub mirroring: Option, pub storage_engine: StorageEngineConfig, diff --git a/src/iroh_node.rs b/src/iroh_node.rs index b0a87a0..d2aa120 100644 --- a/src/iroh_node.rs +++ b/src/iroh_node.rs @@ -13,9 +13,9 @@ use async_stream::stream; use futures::StreamExt; use iroh::bytes::Hash; use iroh::client::quic::RPC_ALPN; -use iroh::net::derp::DerpMode; use iroh::node::{GcPolicy, Node}; use iroh::rpc_protocol::{ProviderRequest, ProviderResponse, ShareMode}; +use iroh::sync::store::DownloadPolicy; use iroh::sync::{AuthorId, NamespaceId}; use iroh::ticket::DocTicket; use quic_rpc::transport::quinn::QuinnServerEndpoint; @@ -87,7 +87,6 @@ impl IrohNode { let mut node_builder = Node::builder(db, docs) .secret_key(secret_key) .peers_data_path(peer_data_path) - .derp_mode(DerpMode::Default) .bind_port(config_lock.iroh.bind_port) .rpc_endpoint(rpc_endpoint); @@ -138,7 +137,11 @@ impl IrohNode { .await .map_err(Error::table)? .unwrap(); - iroh_doc.start_sync(vec![]).await.unwrap(); + iroh_doc + .set_download_policy(table_config.download_policy.clone()) + .await + .map_err(Error::doc)?; + iroh_doc.start_sync(vec![]).await.map_err(Error::doc)?; let storage_engine = match &table_config.storage_engine { StorageEngineConfig::Iroh => { StorageEngine::Iroh(IrohStorageEngine::new(author_id, iroh_doc.clone())) @@ -222,7 +225,6 @@ impl IrohNode { Entry::Occupied(_) => Err(Error::existing_table(table_name)), Entry::Vacant(entry) => { let iroh_doc = self.sync_client.docs.create().await.map_err(Error::table)?; - let (storage_engine, storage_engine_config) = match storage_name { None => ( StorageEngine::Iroh(IrohStorageEngine::new( @@ -266,6 +268,7 @@ impl IrohNode { table_name.to_string(), TableConfig { id: iroh_doc.id().to_string(), + download_policy: DownloadPolicy::default(), mirroring: mirroring_config, storage_engine: storage_engine_config, }, @@ -280,6 +283,7 @@ impl IrohNode { &mut self, table_name: &str, table_ticket: &str, + download_policy: DownloadPolicy, storage_name: Option<&str>, mirroring_config: Option, ) -> Result { @@ -289,9 +293,13 @@ impl IrohNode { let iroh_doc = self .sync_client .docs - .import(DocTicket::from_str(table_ticket).unwrap()) + .import(DocTicket::from_str(table_ticket).map_err(Error::doc)?) .await .map_err(Error::table)?; + iroh_doc + .set_download_policy(download_policy.clone()) + .await + .map_err(Error::doc)?; let (storage_engine, storage_engine_config) = match storage_name { None => ( StorageEngine::Iroh(IrohStorageEngine::new( @@ -336,6 +344,7 @@ impl IrohNode { table_name.to_string(), TableConfig { id: iroh_doc.id().to_string(), + download_policy: download_policy, mirroring: mirroring_config, storage_engine: storage_engine_config, }, @@ -427,7 +436,7 @@ impl IrohNode { pub async fn table_exists(&self, table_name: &str, key: &str) -> Result { match self.table_storages.get(table_name) { - Some(table_storage) => Ok(table_storage.get_hash(key).await?.is_some()), + Some(table_storage) => table_storage.exists(key).await, None => Err(Error::missing_table(table_name)), } } diff --git a/src/storages/fs_storage.rs b/src/storages/fs_storage.rs index cd5f3d0..6160cb1 100644 --- a/src/storages/fs_storage.rs +++ b/src/storages/fs_storage.rs @@ -11,14 +11,13 @@ use iroh::sync::store::Query; use iroh::sync::{AuthorId, ContentStatus}; use lru::LruCache; use std::collections::{HashMap, HashSet}; -use std::io; use std::num::NonZeroUsize; use std::path::PathBuf; use std::sync::Arc; use tokio::io::AsyncRead; use tokio_task_pool::Pool; use tokio_util::bytes; -use tracing::{info, info_span, warn, Instrument}; +use tracing::{error, info, info_span, warn, Instrument}; #[derive(Clone)] pub struct FSStorageEngine { @@ -95,65 +94,69 @@ impl FSStorageEngine { let fs_storage_clone = fs_storage.clone(); - tokio::spawn(async move { - let all_keys: Arc> = Arc::new( - fs_storage_clone - .iroh_doc() - .get_many(Query::all()) - .await - .map_err(Error::doc)? - .map(|x| bytes::Bytes::copy_from_slice(x.unwrap().key())) - .collect() - .await, - ); - let pool = Arc::new(Pool::bounded(16)); - - for fs_shard in &fs_storage_clone.fs_shards.values() { - let fs_storage_clone = fs_storage_clone.clone(); - let fs_shard = fs_shard.clone(); - let all_keys = all_keys.clone(); - let pool = pool.clone(); - tokio::spawn(async move { - let base_path = fs_shard.path().to_path_buf(); - let mut read_dir_stream = tokio::fs::read_dir(&base_path) - .await - .map_err(Error::io_error)?; - while let Some(entry) = read_dir_stream - .next_entry() + if fs_storage_config.is_import_missing_enabled { + tokio::spawn(async move { + let all_keys: Arc> = Arc::new( + fs_storage_clone + .iroh_doc() + .get_many(Query::all()) .await - .map_err(Error::io_error)? - { - let key = key_to_bytes(&entry.file_name().to_string_lossy()); - if all_keys.contains(&key) || key.starts_with(&[b'~']) { - info!(action = "skipped", key = ?entry.file_name()); - continue; - } - pool.spawn({ - let iroh_doc = fs_storage_clone.iroh_doc().clone(); - async move { - let import_progress = iroh_doc - .import_file( - fs_storage_clone.author_id, - key, - &entry.path(), - true, - ) - .await - .map_err(Error::doc) - .unwrap(); - import_progress.finish().await.map_err(Error::hash).unwrap(); - info!(action = "imported", key = ?entry.file_name()) + .map_err(Error::doc)? + .map(|x| bytes::Bytes::copy_from_slice(x.unwrap().key())) + .collect() + .await, + ); + let pool = Arc::new(Pool::bounded(16)); + + for fs_shard in fs_storage_clone.fs_shards.values() { + let fs_storage_clone = fs_storage_clone.clone(); + let fs_shard = fs_shard.clone(); + let all_keys = all_keys.clone(); + let pool = pool.clone(); + tokio::spawn(async move { + let base_path = fs_shard.path().to_path_buf(); + let mut read_dir_stream = tokio::fs::read_dir(&base_path) + .await + .map_err(Error::io_error)?; + while let Some(entry) = read_dir_stream + .next_entry() + .await + .map_err(Error::io_error)? + { + let key = key_to_bytes(&entry.file_name().to_string_lossy()); + if all_keys.contains(&key) || key.starts_with(&[b'~']) { + continue; } - .instrument(info_span!(parent: None, "restore")) - }) - .await - .unwrap(); - } - Ok::<(), Error>(()) - }); - } - Ok::<(), Error>(()) - }); + pool.spawn({ + let iroh_doc = fs_storage_clone.iroh_doc().clone(); + async move { + let import_progress = iroh_doc + .import_file( + fs_storage_clone.author_id, + key, + &entry.path(), + true, + ) + .await + .map_err(Error::doc) + .unwrap(); + match import_progress.finish().await.map_err(Error::hash) { + Err(error) => error!(error = ?error, path = ?entry.path(), key = ?entry.file_name(), "import_progress_error"), + _ => {} + }; + info!(action = "imported", key = ?entry.file_name()) + } + .instrument(info_span!(parent: None, "restore")) + }) + .await + .unwrap(); + } + Ok::<(), Error>(()) + }); + } + Ok::<(), Error>(()) + }); + } Ok(fs_storage) } @@ -223,10 +226,15 @@ impl FSStorageEngine { }) } - pub async fn exists(&self, key: &str) -> io::Result> { + pub async fn exists(&self, key: &str) -> Result> { for file_shard_config in self.hash_ring.range(key, 1) { let file_shard = &self.fs_shards[&file_shard_config.name]; - if file_shard.exists(key).await?.is_some() { + if file_shard + .exists(key) + .await + .map_err(Error::io_error)? + .is_some() + { return Ok(Some(file_shard.get_path_for(key))); } } @@ -236,4 +244,16 @@ impl FSStorageEngine { pub fn iroh_doc(&self) -> &IrohDoc { &self.iroh_doc } + + pub async fn get(&self, key: &str) -> Result> { + for file_shard_config in self.hash_ring.range(key, 1) { + let file_shard = &self.fs_shards[&file_shard_config.name]; + match file_shard.open_store(key).await { + Ok(Some(file)) => return Ok(Box::new(file)), + Ok(None) => return Err(Error::io_error("missing file")), + Err(e) => return Err(Error::io_error(e)), + } + } + Err(Error::io_error("missing shard")) + } } diff --git a/src/storages/iroh_storage.rs b/src/storages/iroh_storage.rs index 32e6bb5..1e2ab65 100644 --- a/src/storages/iroh_storage.rs +++ b/src/storages/iroh_storage.rs @@ -2,7 +2,7 @@ use crate::error::{Error, Result}; use crate::utils::key_to_bytes; use crate::IrohDoc; use iroh::bytes::Hash; -use iroh::sync::store::Query; +use iroh::sync::store::{Query, SortBy, SortDirection}; use iroh::sync::AuthorId; use tokio::io::{AsyncRead, AsyncReadExt}; @@ -49,7 +49,9 @@ impl IrohStorageEngine { pub async fn exists(&self, key: &str) -> Result { Ok(self .iroh_doc - .get_one(Query::key_exact(key_to_bytes(key))) + .get_one( + Query::key_exact(key_to_bytes(key)).sort_by(SortBy::KeyAuthor, SortDirection::Asc), + ) .await .map_err(Error::doc)? .is_some()) @@ -58,4 +60,20 @@ impl IrohStorageEngine { pub const fn iroh_doc(&self) -> &IrohDoc { &self.iroh_doc } + + pub async fn get(&self, key: &str) -> Result> { + Ok(Box::new( + self.iroh_doc() + .get_one( + Query::key_exact(key_to_bytes(key)) + .sort_by(SortBy::KeyAuthor, SortDirection::Asc), + ) + .await + .map_err(Error::doc)? + .ok_or_else(|| Error::missing_key(key))? + .content_reader(self.iroh_doc()) + .await + .map_err(Error::doc)?, + )) + } } diff --git a/src/storages/mirroring.rs b/src/storages/mirroring.rs index dac3d6e..4f9ea4b 100644 --- a/src/storages/mirroring.rs +++ b/src/storages/mirroring.rs @@ -6,6 +6,7 @@ use iroh::client::LiveEvent; use lru::LruCache; use std::num::NonZeroUsize; use std::sync::Arc; +use std::time::Duration; use tokio::task::JoinHandle; use tracing::{info, info_span, warn, Instrument}; @@ -87,12 +88,16 @@ impl Mirroring { info!(action = "sent", sink = sink.name(), key = ?std::str::from_utf8(key)); } if delete_after_mirroring { - if let Err(error) = - sync_client.blobs.delete_blob(*hash).await - { - warn!(error = ?error); - continue; - } + let sync_client = sync_client.clone(); + let hash = hash.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(120)).await; + if let Err(error) = + sync_client.blobs.delete_blob(hash).await + { + warn!(error = ?error); + } + }); } } Err(error) => warn!(error = ?error), diff --git a/src/storages/mod.rs b/src/storages/mod.rs index 4a517b7..400baf6 100644 --- a/src/storages/mod.rs +++ b/src/storages/mod.rs @@ -4,7 +4,7 @@ use futures::Stream; use iroh::bytes::Hash; use iroh::client::Entry; use iroh::rpc_protocol::ShareMode; -use iroh::sync::store::Query; +use iroh::sync::store::{Query, SortBy, SortDirection}; use iroh::ticket::DocTicket; use tokio::io::AsyncRead; @@ -36,16 +36,10 @@ impl Storage { } pub async fn get(&self, key: &str) -> Result> { - Ok(Box::new( - self.iroh_doc() - .get_one(Query::key_exact(key_to_bytes(key))) - .await - .map_err(Error::doc)? - .ok_or_else(|| Error::missing_key(key))? - .content_reader(self.iroh_doc()) - .await - .map_err(Error::doc)?, - )) + match &self.engine { + StorageEngine::FS(storage) => storage.get(key).await, + StorageEngine::Iroh(storage) => storage.get(key).await, + } } pub async fn insert(&self, key: &str, value: S) -> Result { @@ -69,6 +63,13 @@ impl Storage { } } + pub async fn exists(&self, key: &str) -> Result { + match &self.engine { + StorageEngine::FS(storage) => Ok(storage.exists(key).await?.is_some()), + StorageEngine::Iroh(storage) => storage.exists(key).await, + } + } + pub fn iroh_doc(&self) -> &IrohDoc { match &self.engine { StorageEngine::FS(storage) => storage.iroh_doc(), @@ -79,7 +80,9 @@ impl Storage { pub async fn get_hash(&self, key: &str) -> Result> { Ok(self .iroh_doc() - .get_one(Query::key_exact(key_to_bytes(key))) + .get_one( + Query::key_exact(key_to_bytes(key)).sort_by(SortBy::KeyAuthor, SortDirection::Asc), + ) .await .map_err(Error::missing_key)? .map(|entry| (entry.content_hash(), entry.content_len()))) diff --git a/trident-py/pyproject.toml b/trident-py/pyproject.toml index 28c0add..f5e2481 100644 --- a/trident-py/pyproject.toml +++ b/trident-py/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "izihawa-trident-client" -version = "1.0.5" +version = "1.0.7" description = "" authors = ["Pasha Podolsky "] readme = "README.md" diff --git a/trident-py/trident/client.py b/trident-py/trident/client.py index d751722..6562c83 100644 --- a/trident-py/trident/client.py +++ b/trident-py/trident/client.py @@ -57,10 +57,9 @@ async def table_share(self, table: str) -> dict: response = await self.get(url) return await response.json() - async def table_delete(self, table: str, key: str) -> dict: + async def table_delete(self, table: str, key: str): url = f"/tables/{table}/{key}/" - response = await self.delete(url) - return await response.json() + await self.delete(url) async def table_foreign_insert(self, from_table: str, from_key: str, to_table: str, to_key: str) -> bytes: url = f"/tables/foreign_insert/"