From bbd98752ee53709b88e9d84c3db1956440f0f666 Mon Sep 17 00:00:00 2001 From: Franz Heinzmann Date: Tue, 9 Apr 2024 11:19:41 +0200 Subject: [PATCH] feat: update redb to v2 (#2120) ## Description `redb` 2.0.0 was released: https://github.com/cberner/redb/releases/tag/v2.0.0 most relevant for us: > 2.0.0 uses a new file format that optimizes len() to be constant time. This means that it is not backwards compatible with 1.x. To upgrade, consider using a pattern like that shown in the [upgrade_v1_to_v2](https://github.com/cberner/redb/blob/222a37f4600588261b0983eebcd074bb69d6e5a0/tests/backward_compatibility.rs#L282-L299) test. and some nice API changes that will simplify the iroh-sync codebase (no more ouroboros for self-refential structs needed!) > * Remove lifetimes from read-only tables > * Remove lifetime from WriteTransaction and ReadTransaction and some further API changes, see the notes. --- * [x] Update iroh-bytes to redb v2 * Did some crude search&replaces to remove now unneeded lifetime bounds. Compiles now, let's see what test says. * [x] Migration for iroh-bytes db * Code adapted from here https://github.com/n0-computer/migrate-bao-store-redb/blob/main/src/main.rs * [x] Update iroh-sync to redb v2 * This will be a bit more involved as iroh-bytes because we hit more API changes, but also great because we can remove `ouroborous` because iterators can now own a range * [x] Migration for iroh-sync db The migrations likely need a dependency to redb v1 to read the old database, so at least for one release cycle we'll have to depend on both redb v2 and v1. ## Notes & open questions This currently includes an extra-safeguard by preserving the old databases for sync and blobs as `docs.redb.backup-redb-v1` and `blobs.db-backup-redb-v1` - do we want this or delete the backups after successfull migration? ## Change checklist - [x] Self-review. - [ ] Documentation updates if relevant. - [ ] Tests if relevant. --------- Co-authored-by: Friedel Ziegelmayer --- Cargo.lock | 85 ++++++++++-------------------------------------------- 1 file changed, 15 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 91c92d5ea6..c3b2ec4fa8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,12 +49,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "aliasable" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" - [[package]] name = "allocator-api2" version = "0.2.16" @@ -2363,7 +2357,7 @@ dependencies = [ "proptest", "rand", "rand_core", - "redb", + "redb 2.0.0", "serde", "serde-error", "serde_json", @@ -2418,7 +2412,8 @@ dependencies = [ "rand", "range-collections", "rcgen 0.12.1", - "redb", + "redb 1.5.1", + "redb 2.0.0", "reflink-copy", "rustls", "self_cell", @@ -2672,7 +2667,6 @@ dependencies = [ "lru", "num_enum", "once_cell", - "ouroboros", "parking_lot", "postcard", "proptest", @@ -2680,7 +2674,8 @@ dependencies = [ "rand", "rand_chacha", "rand_core", - "redb", + "redb 1.5.1", + "redb 2.0.0", "serde", "strum 0.25.0", "tempfile", @@ -2732,15 +2727,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.10" @@ -3269,31 +3255,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "ouroboros" -version = "0.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b7be5a8a3462b752f4be3ff2b2bf2f7f1d00834902e46be2a4d68b87b0573c" -dependencies = [ - "aliasable", - "ouroboros_macro", - "static_assertions", -] - -[[package]] -name = "ouroboros_macro" -version = "0.18.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b645dcde5f119c2c454a92d0dfa271a2a3b205da92e4292a68ead4bdbfde1f33" -dependencies = [ - "heck 0.4.1", - "itertools 0.12.1", - "proc-macro2", - "proc-macro2-diagnostics", - "quote", - "syn 2.0.53", -] - [[package]] name = "overload" version = "0.1.1" @@ -3693,7 +3654,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" dependencies = [ "diff", - "yansi 0.5.1", + "yansi", ] [[package]] @@ -3755,19 +3716,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "proc-macro2-diagnostics" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.53", - "version_check", - "yansi 1.0.1", -] - [[package]] name = "prometheus-client" version = "0.22.2" @@ -4073,6 +4021,15 @@ dependencies = [ "libc", ] +[[package]] +name = "redb" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1100a056c5dcdd4e5513d5333385223b26ef1bf92f31eb38f407e8c20549256" +dependencies = [ + "libc", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -4849,12 +4806,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - [[package]] name = "str-buf" version = "1.0.6" @@ -6162,12 +6113,6 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" - [[package]] name = "yasna" version = "0.5.2"