From 0b2dfb7b4d9efa1f0c44473890d44800b2027bc0 Mon Sep 17 00:00:00 2001 From: Luke Iannucci Date: Thu, 3 Oct 2024 14:02:08 -0400 Subject: [PATCH 01/12] start --- Cargo.lock | 223 ++++++------------- Cargo.toml | 38 ++-- builder/src/lib.rs | 2 +- builder/src/permissioned.rs | 6 +- marketplace-builder/src/lib.rs | 2 +- node-metrics/src/service/client_state/mod.rs | 4 +- sequencer/src/context.rs | 6 +- sequencer/src/lib.rs | 12 +- sequencer/src/message_compat_tests.rs | 4 +- types/src/v0/mod.rs | 4 +- types/src/v0/traits.rs | 10 +- 11 files changed, 109 insertions(+), 202 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b7b5259c..5ccb7a59f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1318,11 +1318,11 @@ dependencies = [ "hotshot-orchestrator", "hotshot-stake-table", "hotshot-state-prover", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "jf-merkle-tree", "jf-signature", "libp2p", - "libp2p-networking 0.5.75 (git+https://github.com/EspressoSystems//HotShot.git?tag=0.5.75-patch1)", + "libp2p-networking", "portpicker", "rand 0.8.5", "sequencer", @@ -2978,7 +2978,7 @@ dependencies = [ "hotshot-orchestrator", "hotshot-query-service", "hotshot-testing", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "itertools 0.12.1", "jf-merkle-tree", "jf-utils", @@ -4098,8 +4098,8 @@ dependencies = [ [[package]] name = "hotshot" -version = "0.5.75" -source = "git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1#652042033d7944985a2361a74a7f3024acf77404" +version = "0.5.77" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" dependencies = [ "anyhow", "async-broadcast", @@ -4124,10 +4124,10 @@ dependencies = [ "hotshot-orchestrator", "hotshot-task", "hotshot-task-impls", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "jf-signature", "libp2p-identity", - "libp2p-networking 0.5.75 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "libp2p-networking", "lru 0.12.4", "num_enum", "parking_lot", @@ -4147,14 +4147,14 @@ dependencies = [ [[package]] name = "hotshot-builder-api" version = "0.1.7" -source = "git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1#652042033d7944985a2361a74a7f3024acf77404" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" dependencies = [ "async-trait", "clap", "committable", "derive_more 1.0.0", "futures", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "serde", "snafu 0.8.4", "tagged-base64", @@ -4165,8 +4165,8 @@ dependencies = [ [[package]] name = "hotshot-builder-core" -version = "0.1.50" -source = "git+https://github.com/EspressoSystems/marketplace-builder-core?tag=0.1.50#c7fa32faf1a7664469543453760e7adc54376f10" +version = "0.1.51" +source = "git+https://github.com/EspressoSystems/marketplace-builder-core?branch=hotshot/rc-0.5.77#08a9c61d3ae17348e3043a517aef46bab445c971" dependencies = [ "anyhow", "async-broadcast", @@ -4183,7 +4183,7 @@ dependencies = [ "hotshot", "hotshot-builder-api", "hotshot-events-service", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "lru 0.12.4", "serde", "sha2 0.10.8", @@ -4211,7 +4211,7 @@ dependencies = [ "contract-bindings", "diff-test-bn254", "ethers", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "jf-pcs", "jf-plonk", "jf-utils", @@ -4221,8 +4221,8 @@ dependencies = [ [[package]] name = "hotshot-events-service" -version = "0.1.48" -source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?tag=0.1.48#6f93923374891e973ff7dec773e4339c7add049e" +version = "0.1.49" +source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?branch=hotshot/rc-0.5.77#3b81fc059818fe273bc2cf7aea575514d233ecd4" dependencies = [ "async-broadcast", "async-compatibility-layer", @@ -4234,7 +4234,7 @@ dependencies = [ "derive_more 0.99.18", "either", "futures", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "rand 0.8.5", "serde", "snafu 0.8.4", @@ -4247,8 +4247,8 @@ dependencies = [ [[package]] name = "hotshot-example-types" -version = "0.5.75" -source = "git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1#652042033d7944985a2361a74a7f3024acf77404" +version = "0.5.77" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" dependencies = [ "anyhow", "async-broadcast", @@ -4264,9 +4264,9 @@ dependencies = [ "hotshot", "hotshot-task", "hotshot-task-impls", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "rand 0.8.5", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "sha2 0.10.8", "sha3", @@ -4280,15 +4280,15 @@ dependencies = [ [[package]] name = "hotshot-fakeapi" -version = "0.5.75" -source = "git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1#652042033d7944985a2361a74a7f3024acf77404" +version = "0.5.77" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" dependencies = [ "anyhow", "async-lock 2.8.0", "async-trait", "futures", "hotshot-example-types", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "rand 0.8.5", "serde", "tide-disco", @@ -4299,8 +4299,8 @@ dependencies = [ [[package]] name = "hotshot-macros" -version = "0.5.75" -source = "git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1#652042033d7944985a2361a74a7f3024acf77404" +version = "0.5.77" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" dependencies = [ "derive_builder", "proc-macro2", @@ -4310,8 +4310,8 @@ dependencies = [ [[package]] name = "hotshot-orchestrator" -version = "0.5.75" -source = "git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1#652042033d7944985a2361a74a7f3024acf77404" +version = "0.5.77" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" dependencies = [ "anyhow", "async-compatibility-layer", @@ -4322,7 +4322,7 @@ dependencies = [ "clap", "csv", "futures", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "libp2p", "multiaddr", "serde", @@ -4340,8 +4340,8 @@ dependencies = [ [[package]] name = "hotshot-query-service" -version = "0.1.61" -source = "git+https://github.com/EspressoSystems/hotshot-query-service?tag=0.1.61#2fe7e46cb174168b16171f1fb885e624f1e8da7c" +version = "0.1.62" +source = "git+https://github.com/EspressoSystems/hotshot-query-service?branch=hotshot/rc-0.5.77#3003620bfac88e14a76bee9998929a606b586585" dependencies = [ "anyhow", "ark-serialize", @@ -4363,7 +4363,7 @@ dependencies = [ "hotshot", "hotshot-example-types", "hotshot-testing", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "include_dir", "itertools 0.12.1", "jf-merkle-tree", @@ -4396,8 +4396,8 @@ dependencies = [ [[package]] name = "hotshot-stake-table" -version = "0.5.75" -source = "git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1#652042033d7944985a2361a74a7f3024acf77404" +version = "0.5.77" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" dependencies = [ "ark-bn254", "ark-ed-on-bn254", @@ -4406,7 +4406,7 @@ dependencies = [ "ark-std", "digest 0.10.7", "ethereum-types", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "jf-crhf", "jf-rescue", "jf-signature", @@ -4435,7 +4435,7 @@ dependencies = [ "futures", "hotshot-contract-adapter", "hotshot-stake-table", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "itertools 0.12.1", "jf-crhf", "jf-pcs", @@ -4444,7 +4444,7 @@ dependencies = [ "jf-rescue", "jf-signature", "jf-utils", - "reqwest 0.12.7", + "reqwest 0.12.8", "sequencer-utils", "serde", "surf-disco", @@ -4458,8 +4458,8 @@ dependencies = [ [[package]] name = "hotshot-task" -version = "0.5.75" -source = "git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1#652042033d7944985a2361a74a7f3024acf77404" +version = "0.5.77" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" dependencies = [ "anyhow", "async-broadcast", @@ -4473,8 +4473,8 @@ dependencies = [ [[package]] name = "hotshot-task-impls" -version = "0.5.75" -source = "git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1#652042033d7944985a2361a74a7f3024acf77404" +version = "0.5.77" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" dependencies = [ "anyhow", "async-broadcast", @@ -4491,7 +4491,7 @@ dependencies = [ "futures", "hotshot-builder-api", "hotshot-task", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "jf-signature", "jf-vid", "rand 0.8.5", @@ -4510,8 +4510,8 @@ dependencies = [ [[package]] name = "hotshot-testing" -version = "0.5.75" -source = "git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1#652042033d7944985a2361a74a7f3024acf77404" +version = "0.5.77" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" dependencies = [ "anyhow", "async-broadcast", @@ -4533,14 +4533,14 @@ dependencies = [ "hotshot-orchestrator", "hotshot-task", "hotshot-task-impls", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "itertools 0.13.0", "jf-signature", "jf-vid", "lru 0.12.4", "portpicker", "rand 0.8.5", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "sha2 0.10.8", "sha3", @@ -4557,7 +4557,7 @@ dependencies = [ [[package]] name = "hotshot-types" version = "0.1.11" -source = "git+https://github.com/EspressoSystems//HotShot.git?tag=0.5.75-patch1#652042033d7944985a2361a74a7f3024acf77404" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" dependencies = [ "anyhow", "ark-bn254", @@ -4584,62 +4584,6 @@ dependencies = [ "espresso-systems-common 0.4.1", "ethereum-types", "futures", - "generic-array", - "jf-pcs", - "jf-signature", - "jf-utils", - "jf-vid", - "lazy_static", - "libp2p", - "memoize", - "rand 0.8.5", - "rand_chacha 0.3.1", - "reqwest 0.12.7", - "serde", - "serde_bytes", - "sha2 0.10.8", - "snafu 0.8.4", - "tagged-base64", - "time 0.3.36", - "tokio", - "tracing", - "typenum", - "url", - "vbs", - "vec1", -] - -[[package]] -name = "hotshot-types" -version = "0.1.11" -source = "git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1#652042033d7944985a2361a74a7f3024acf77404" -dependencies = [ - "anyhow", - "ark-bn254", - "ark-ed-on-bn254", - "ark-ff", - "ark-serialize", - "ark-srs", - "ark-std", - "async-compatibility-layer", - "async-lock 2.8.0", - "async-std", - "async-trait", - "bincode", - "bitvec", - "blake3", - "cdn-proto 0.4.0 (git+https://github.com/EspressoSystems/Push-CDN?tag=0.4.7)", - "committable", - "custom_debug 0.5.1", - "derivative", - "digest 0.10.7", - "displaydoc", - "dyn-clone 1.0.17 (git+https://github.com/dtolnay/dyn-clone?tag=1.0.17)", - "either", - "espresso-systems-common 0.4.1", - "ethereum-types", - "futures", - "generic-array", "jf-pcs", "jf-signature", "jf-utils", @@ -4649,7 +4593,7 @@ dependencies = [ "memoize", "rand 0.8.5", "rand_chacha 0.3.1", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "serde_bytes", "sha2 0.10.8", @@ -5868,43 +5812,8 @@ dependencies = [ [[package]] name = "libp2p-networking" -version = "0.5.75" -source = "git+https://github.com/EspressoSystems//HotShot.git?tag=0.5.75-patch1#652042033d7944985a2361a74a7f3024acf77404" -dependencies = [ - "anyhow", - "async-compatibility-layer", - "async-lock 2.8.0", - "async-std", - "async-trait", - "bincode", - "blake3", - "custom_debug 0.5.1", - "delegate", - "derive_builder", - "either", - "futures", - "hotshot-types 0.1.11 (git+https://github.com/EspressoSystems//HotShot.git?tag=0.5.75-patch1)", - "lazy_static", - "libp2p", - "libp2p-identity", - "libp2p-swarm-derive", - "pin-project", - "portpicker", - "rand 0.8.5", - "serde", - "serde_bytes", - "serde_json", - "snafu 0.8.4", - "tokio", - "tokio-stream", - "tracing", - "void", -] - -[[package]] -name = "libp2p-networking" -version = "0.5.75" -source = "git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1#652042033d7944985a2361a74a7f3024acf77404" +version = "0.5.77" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" dependencies = [ "anyhow", "async-compatibility-layer", @@ -5918,7 +5827,7 @@ dependencies = [ "derive_builder", "either", "futures", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "lazy_static", "libp2p", "libp2p-identity", @@ -6272,7 +6181,7 @@ dependencies = [ "hotshot-orchestrator", "hotshot-query-service", "hotshot-stake-table", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "jf-merkle-tree", "jf-signature", "marketplace-builder-core", @@ -6291,8 +6200,8 @@ dependencies = [ [[package]] name = "marketplace-builder-core" -version = "0.1.50" -source = "git+https://github.com/EspressoSystems/marketplace-builder-core?tag=0.1.50#c7fa32faf1a7664469543453760e7adc54376f10" +version = "0.1.51" +source = "git+https://github.com/EspressoSystems/marketplace-builder-core?branch=hotshot/rc-0.5.77#08a9c61d3ae17348e3043a517aef46bab445c971" dependencies = [ "anyhow", "async-broadcast", @@ -6310,7 +6219,7 @@ dependencies = [ "hotshot-builder-api", "hotshot-events-service", "hotshot-task-impls", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "lru 0.12.4", "multimap", "rkyv", @@ -6345,7 +6254,7 @@ dependencies = [ "hotshot", "hotshot-events-service", "hotshot-query-service", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "jf-signature", "portpicker", "rand 0.8.5", @@ -6728,9 +6637,9 @@ dependencies = [ "hotshot", "hotshot-query-service", "hotshot-stake-table", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "prometheus-parse", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "serde_json", "surf-disco", @@ -7646,7 +7555,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.77", @@ -8136,9 +8045,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.7" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ "base64 0.22.1", "bytes 1.7.1", @@ -8756,7 +8665,7 @@ dependencies = [ "hotshot-stake-table", "hotshot-state-prover", "hotshot-testing", - "hotshot-types 0.1.11 (git+https://www.github.com/EspressoSystems/HotShot.git?tag=0.5.75-patch1)", + "hotshot-types", "include_dir", "itertools 0.12.1", "jf-crhf", @@ -8765,7 +8674,7 @@ dependencies = [ "jf-signature", "jf-vid", "libp2p", - "libp2p-networking 0.5.75 (git+https://github.com/EspressoSystems//HotShot.git?tag=0.5.75-patch1)", + "libp2p-networking", "marketplace-builder-core", "marketplace-solver", "num_enum", @@ -8774,7 +8683,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rand_distr", - "reqwest 0.12.7", + "reqwest 0.12.8", "sequencer-utils", "serde", "serde_json", @@ -9663,7 +9572,7 @@ dependencies = [ "derivative", "futures", "hex", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "serde_json", "tide-disco", @@ -9938,7 +9847,7 @@ dependencies = [ "espresso-types", "ethers", "futures", - "reqwest 0.12.7", + "reqwest 0.12.8", "surf-disco", ] @@ -10032,7 +9941,7 @@ dependencies = [ "pin-project", "prometheus", "rand 0.8.5", - "reqwest 0.12.7", + "reqwest 0.12.8", "routefinder", "semver 1.0.23", "serde", diff --git a/Cargo.toml b/Cargo.toml index 49fe56049..73295c033 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,23 +57,23 @@ dyn-clone = "1.0" ethers = { version = "2.0", features = ["solc"] } futures = "0.3" -hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76", features = ["dependency-tasks"] } +hotshot = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77", features = ["dependency-tasks"] } # Hotshot imports -hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.76" } -hotshot-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.1.50" } -marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.1.50" } -hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.48" } -hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76" } -hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.61" } -hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76" } +hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", branch = "bump/rc-0.5.77" } +hotshot-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", branch = "hotshot/rc-0.5.77" } +marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", branch = "hotshot/rc-0.5.77" } +hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", branch = "hotshot/rc-0.5.77" } +hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77" } +hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", branch = "hotshot/rc-0.5.77" } +hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77" } hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" } -hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76" } -hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76", features = ["dependency-tasks"] } -hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76" } -libp2p-networking = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76" } +hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77" } +hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77", features = ["dependency-tasks"] } +hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77" } +libp2p-networking = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77" } hotshot-contract-adapter = { version = "0.1.0", path = "contracts/rust/adapter" } # Temporary, used to pull in the mock auction results provider -hotshot-example-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.76" } +hotshot-example-types = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77" } # Push CDN imports cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", features = [ @@ -144,15 +144,3 @@ paste = "1.0" rand = "0.8.5" time = "0.3" trait-set = "0.3.0" - -[patch."https://github.com/EspressoSystems/HotShot.git"] -hotshot = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" } -hotshot-builder-api = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" } -hotshot-orchestrator = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" } -hotshot-stake-table = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" } -hotshot-task ={ git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" } -hotshot-task-impls ={ git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" } -hotshot-testing ={ git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" } -hotshot-types = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" } -libp2p-networking = { git = "https://github.com/EspressoSystems//HotShot.git", tag = "0.5.75-patch1" } -hotshot-example-types = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.75-patch1" } diff --git a/builder/src/lib.rs b/builder/src/lib.rs index a44c2fbdb..d72d9518b 100755 --- a/builder/src/lib.rs +++ b/builder/src/lib.rs @@ -28,7 +28,7 @@ use futures::{ stream::{Stream, StreamExt}, }; use hotshot::{ - traits::election::static_committee::GeneralStaticCommittee, + traits::election::static_committee::StaticCommittee, types::{SignatureKey, SystemContextHandle}, HotShotInitializer, Memberships, SystemContext, }; diff --git a/builder/src/permissioned.rs b/builder/src/permissioned.rs index db23cfa4c..aa03f827b 100644 --- a/builder/src/permissioned.rs +++ b/builder/src/permissioned.rs @@ -41,7 +41,7 @@ use futures::{ }; use hotshot::{ traits::{ - election::static_committee::GeneralStaticCommittee, + election::static_committee::StaticCommittee, implementations::{ derive_libp2p_peer_id, CdnMetricsValue, CdnTopic, CombinedNetworks, GossipConfig, KeyPair, Libp2pNetwork, PushCdnNetwork, WrappedSignatureKey, @@ -342,12 +342,12 @@ pub async fn init_hotshot, P: SequencerPersistence, None => config.known_nodes_with_stake.clone(), }; - let quorum_membership = GeneralStaticCommittee::new( + let quorum_membership = StaticCommittee::new( combined_known_nodes_with_stake.clone(), combined_known_nodes_with_stake.clone(), Topic::Global, ); - let da_membership = GeneralStaticCommittee::new( + let da_membership = StaticCommittee::new( combined_known_nodes_with_stake.clone(), combined_known_nodes_with_stake, Topic::Da, diff --git a/marketplace-builder/src/lib.rs b/marketplace-builder/src/lib.rs index 42916015a..7a2336d19 100755 --- a/marketplace-builder/src/lib.rs +++ b/marketplace-builder/src/lib.rs @@ -31,7 +31,7 @@ use futures::{ stream::{Stream, StreamExt}, }; use hotshot::{ - traits::election::static_committee::GeneralStaticCommittee, + traits::election::static_committee::StaticCommittee, types::{SignatureKey, SystemContextHandle}, HotShotInitializer, Memberships, SystemContext, }; diff --git a/node-metrics/src/service/client_state/mod.rs b/node-metrics/src/service/client_state/mod.rs index 54c61a3ac..54dbf8692 100644 --- a/node-metrics/src/service/client_state/mod.rs +++ b/node-metrics/src/service/client_state/mod.rs @@ -406,14 +406,14 @@ where block_size: data_state_read_lock_guard .latest_blocks() .skip(1) - .map(|block| block.size) + .map(|block| Some(block.size)) .collect(), block_time: data_state_read_lock_guard .latest_blocks() .skip(1) .zip(data_state_read_lock_guard.latest_blocks()) .map(|(block_i, block_i_sub_1)| { - (block_i.time.0 - block_i_sub_1.time.0).whole_seconds() as u64 + Some((block_i.time.0 - block_i_sub_1.time.0).whole_seconds() as u64) }) .collect(), block_transactions: data_state_read_lock_guard diff --git a/sequencer/src/context.rs b/sequencer/src/context.rs index 8d272b406..ed4fc55e3 100644 --- a/sequencer/src/context.rs +++ b/sequencer/src/context.rs @@ -15,7 +15,7 @@ use futures::{ stream::{Stream, StreamExt}, }; use hotshot::{ - traits::election::static_committee::GeneralStaticCommittee, + traits::election::static_committee::StaticCommittee, types::{Event, EventType, SystemContextHandle}, MarketplaceConfig, Memberships, SystemContext, }; @@ -103,13 +103,13 @@ impl, P: SequencerPersistence, V: Versions> Sequence .load_consensus_state::(instance_state.clone()) .await?; - let committee_membership = GeneralStaticCommittee::new( + let committee_membership = StaticCommittee::new( config.known_nodes_with_stake.clone(), config.known_nodes_with_stake.clone(), Topic::Global, ); - let da_membership = GeneralStaticCommittee::new( + let da_membership = StaticCommittee::new( config.known_nodes_with_stake.clone(), config.known_da_nodes.clone(), Topic::Da, diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 63c57ceb3..28160fe38 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -39,7 +39,7 @@ use derivative::Derivative; use espresso_types::v0::traits::{PersistenceOptions, SequencerPersistence}; pub use genesis::Genesis; #[cfg(feature = "libp2p")] -use hotshot::traits::implementations::{CombinedNetworks, GossipConfig, Libp2pNetwork}; +use hotshot::traits::implementations::{CombinedNetworks, GossipConfig, Libp2pNetwork, derive_libp2p_multiaddr}; use hotshot::{ traits::implementations::{ derive_libp2p_peer_id, CdnMetricsValue, CdnTopic, KeyPair, MemoryNetwork, PushCdnNetwork, @@ -205,11 +205,11 @@ pub async fn init_node( // Orchestrator client let validator_args = ValidatorArgs { url: network_params.orchestrator_url, - advertise_address: Some(network_params.libp2p_advertise_address), + advertise_address: Some(network_params.libp2p_advertise_address.to_string()), builder_address: None, network_config_file: None, }; - let orchestrator_client = OrchestratorClient::new(validator_args); + let orchestrator_client = OrchestratorClient::new(validator_args.url); let state_key_pair = StateKeyPair::from_sign_key(network_params.private_state_key); let my_config = ValidatorConfig { public_key: pub_key, @@ -254,12 +254,14 @@ pub async fn init_node( tracing::error!( "waiting for other nodes to connect, DO NOT RESTART until fully connected" ); + let multiaddr = + derive_libp2p_multiaddr(&network_params.libp2p_advertise_address.to_string()).expect("Failed to derive valid multiaddr, {}"); let config = NetworkConfig::get_complete_config( &orchestrator_client, my_config.clone(), // Register in our Libp2p advertise address and public key so other nodes // can contact us on startup - Some(network_params.libp2p_advertise_address), + Some(multiaddr), Some(libp2p_public_key), ) .await? @@ -328,7 +330,7 @@ pub async fn init_node( let p2p_network = Libp2pNetwork::from_config::( config.clone(), GossipConfig::default(), - network_params.libp2p_bind_address, + network_params.libp2p_bind_address.to_string(), &my_config.public_key, // We need the private key so we can derive our Libp2p keypair // (using https://docs.rs/blake3/latest/blake3/fn.derive_key.html) diff --git a/sequencer/src/message_compat_tests.rs b/sequencer/src/message_compat_tests.rs index 136967e97..c0323486b 100755 --- a/sequencer/src/message_compat_tests.rs +++ b/sequencer/src/message_compat_tests.rs @@ -18,7 +18,7 @@ use std::path::Path; use committable::Committable; use espresso_types::{Leaf, NodeState, PubKey, ValidatedState}; -use hotshot::traits::election::static_committee::GeneralStaticCommittee; +use hotshot::traits::election::static_committee::StaticCommittee; use hotshot_types::{ data::{ DaProposal, QuorumProposal, UpgradeProposal, VidDisperse, VidDisperseShare, @@ -59,7 +59,7 @@ async fn test_message_compat(_ver: Ver) { let (sender, priv_key) = PubKey::generated_from_seed_indexed(Default::default(), 0); let signature = PubKey::sign(&priv_key, &[]).unwrap(); - let membership = GeneralStaticCommittee::new( + let membership = StaticCommittee::new( vec![], /* no elligible leaders */ vec![PeerConfig::default()], /* one committee member, necessary to generate a VID share */ Topic::Global, diff --git a/types/src/v0/mod.rs b/types/src/v0/mod.rs index 6be73b8f5..97d1e8a94 100644 --- a/types/src/v0/mod.rs +++ b/types/src/v0/mod.rs @@ -1,6 +1,6 @@ use std::marker::PhantomData; -use hotshot::traits::election::static_committee::GeneralStaticCommittee; +use hotshot::traits::election::static_committee::StaticCommittee; use hotshot_types::{ data::ViewNumber, signature_key::BLSPubKey, @@ -133,7 +133,7 @@ impl NodeType for SeqTypes { type Transaction = Transaction; type InstanceState = NodeState; type ValidatedState = ValidatedState; - type Membership = GeneralStaticCommittee; + type Membership = StaticCommittee; type BuilderSignatureKey = FeeAccount; type AuctionResult = SolverAuctionResults; } diff --git a/types/src/v0/traits.rs b/types/src/v0/traits.rs index 7622a944c..7b278406d 100644 --- a/types/src/v0/traits.rs +++ b/types/src/v0/traits.rs @@ -13,7 +13,7 @@ use hotshot_types::{ data::{DaProposal, QuorumProposal, VidDisperseShare, ViewNumber}, event::{HotShotAction, LeafInfo}, message::Proposal, - simple_certificate::QuorumCertificate, + simple_certificate::{QuorumCertificate, UpgradeCertificate}, traits::{ node_implementation::{ConsensusTime, Versions}, storage::Storage, @@ -461,6 +461,7 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static { highest_voted_view, saved_proposals, high_qc, + None, undecided_leaves.into_values().collect(), undecided_state, ), @@ -615,6 +616,13 @@ impl Storage for Arc

{ ) -> anyhow::Result<()> { (**self).append_quorum_proposal(proposal).await } + + async fn update_decided_upgrade_certificate( + &self, + _decided_upgrade_certificate: Option>, + ) -> anyhow::Result<()> { + Ok(()) + } } /// Data that can be deserialized from a subslice of namespace payload bytes. From 2c0dd9f750e306dde164f92dc96fe231e46c01e3 Mon Sep 17 00:00:00 2001 From: Rob Date: Thu, 3 Oct 2024 14:20:58 -0400 Subject: [PATCH 02/12] libp2p --- builder/src/bin/permissioned-builder.rs | 20 +--------- builder/src/permissioned.rs | 37 +++++++++++++------ sequencer/src/bin/cdn-whitelist.rs | 9 ++--- sequencer/src/lib.rs | 49 +++++++++++++++---------- sequencer/src/main.rs | 22 ++--------- 5 files changed, 64 insertions(+), 73 deletions(-) diff --git a/builder/src/bin/permissioned-builder.rs b/builder/src/bin/permissioned-builder.rs index b036a3ca9..be7aa9a1d 100644 --- a/builder/src/bin/permissioned-builder.rs +++ b/builder/src/bin/permissioned-builder.rs @@ -235,26 +235,10 @@ async fn run( let builder_key_pair = EthKeyPair::from_mnemonic(&opt.eth_mnemonic, opt.eth_account_index)?; - // Parse supplied Libp2p addresses to their socket form - // We expect all nodes to be reachable via IPv4, so we filter out any IPv6 addresses. - // Downstream in HotShot we pin the IP address to v4, but this can be fixed in the future. - let libp2p_advertise_address = opt - .libp2p_advertise_address - .to_socket_addrs()? - .find(|x| x.is_ipv4()) - .ok_or(anyhow::anyhow!( - "Failed to resolve Libp2p advertise address" - ))?; - let libp2p_bind_address = opt - .libp2p_bind_address - .to_socket_addrs()? - .find(|x| x.is_ipv4()) - .ok_or(anyhow::anyhow!("Failed to resolve Libp2p bind address"))?; - let network_params = NetworkParams { cdn_endpoint: opt.cdn_endpoint, - libp2p_advertise_address, - libp2p_bind_address, + libp2p_advertise_address: opt.libp2p_advertise_address, + libp2p_bind_address: opt.libp2p_bind_address, libp2p_bootstrap_nodes: opt.libp2p_bootstrap_nodes, orchestrator_url: opt.orchestrator_url, state_relay_server_url: opt.state_relay_server_url, diff --git a/builder/src/permissioned.rs b/builder/src/permissioned.rs index aa03f827b..374426072 100644 --- a/builder/src/permissioned.rs +++ b/builder/src/permissioned.rs @@ -43,8 +43,9 @@ use hotshot::{ traits::{ election::static_committee::StaticCommittee, implementations::{ - derive_libp2p_peer_id, CdnMetricsValue, CdnTopic, CombinedNetworks, GossipConfig, - KeyPair, Libp2pNetwork, PushCdnNetwork, WrappedSignatureKey, + derive_libp2p_multiaddr, derive_libp2p_peer_id, CdnMetricsValue, CdnTopic, + CombinedNetworks, GossipConfig, KeyPair, Libp2pNetwork, PushCdnNetwork, + WrappedSignatureKey, }, BlockPayload, }, @@ -105,6 +106,7 @@ use sequencer::{ }; use surf_disco::Client; use tide_disco::{app, method::ReadState, App, Url}; +use tracing::info; use vbs::version::StaticVersionType; use crate::run_builder_api_service; @@ -148,13 +150,7 @@ pub async fn init_node( maximize_txns_count_timeout_duration: Duration, ) -> anyhow::Result> { // Orchestrator client - let validator_args = ValidatorArgs { - url: network_params.orchestrator_url, - advertise_address: Some(network_params.libp2p_advertise_address), - builder_address: None, - network_config_file: None, - }; - let orchestrator_client = OrchestratorClient::new(validator_args); + let orchestrator_client = OrchestratorClient::new(network_params.orchestrator_url); let state_key_pair = StateKeyPair::from_sign_key(network_params.private_state_key); let my_config = ValidatorConfig { public_key: BLSPubKey::from_private(&network_params.private_staking_key), @@ -164,6 +160,25 @@ pub async fn init_node( is_da, }; + // Parse the Libp2p bind and advertise addresses to multiaddresses + let libp2p_bind_address = derive_libp2p_multiaddr(&network_params.libp2p_bind_address) + .with_context(|| { + format!( + "Failed to derive Libp2p bind address of {}", + &network_params.libp2p_bind_address + ) + })?; + let libp2p_advertise_address = + derive_libp2p_multiaddr(&network_params.libp2p_advertise_address).with_context(|| { + format!( + "Failed to derive Libp2p advertise address of {}", + &network_params.libp2p_advertise_address + ) + })?; + + info!("Libp2p bind address: {}", libp2p_bind_address); + info!("Libp2p advertise address: {}", libp2p_advertise_address); + // Derive our Libp2p public key from our private key let libp2p_public_key = derive_libp2p_peer_id::<::SignatureKey>(&my_config.private_key) @@ -174,7 +189,7 @@ pub async fn init_node( my_config.clone(), // Register in our Libp2p advertise address and public key so other nodes // can contact us on startup - Some(network_params.libp2p_advertise_address), + Some(libp2p_advertise_address), Some(libp2p_public_key), ) .await? @@ -224,7 +239,7 @@ pub async fn init_node( let p2p_network = Libp2pNetwork::from_config::( config.clone(), GossipConfig::default(), - network_params.libp2p_bind_address, + libp2p_bind_address, &my_config.public_key, // We need the private key so we can derive our Libp2p keypair // (using https://docs.rs/blake3/latest/blake3/fn.derive_key.html) diff --git a/sequencer/src/bin/cdn-whitelist.rs b/sequencer/src/bin/cdn-whitelist.rs index 44fd97ff3..c59f14bba 100644 --- a/sequencer/src/bin/cdn-whitelist.rs +++ b/sequencer/src/bin/cdn-whitelist.rs @@ -44,12 +44,9 @@ async fn main() -> Result<()> { tracing_subscriber::fmt::init(); // Create a new `OrchestratorClient` from the supplied URL - let orchestrator_client = OrchestratorClient::new(ValidatorArgs { - url: Url::from_str(&args.orchestrator_url).with_context(|| "Invalid URL")?, - advertise_address: None, - builder_address: None, - network_config_file: None, - }); + let orchestrator_client = OrchestratorClient::new( + Url::from_str(&args.orchestrator_url).with_context(|| "Invalid URL")?, + ); tracing::info!( "Waiting for config from orchestrator on {}", diff --git a/sequencer/src/lib.rs b/sequencer/src/lib.rs index 28160fe38..df3715361 100644 --- a/sequencer/src/lib.rs +++ b/sequencer/src/lib.rs @@ -27,19 +27,22 @@ use libp2p::Multiaddr; use network::libp2p::split_off_peer_id; use options::Identity; use state_signature::static_stake_table_commitment; +use tracing::info; use url::Url; pub mod persistence; pub mod state; #[cfg(feature = "libp2p")] use std::time::Duration; -use std::{collections::BTreeMap, fmt::Debug, marker::PhantomData, net::SocketAddr, sync::Arc}; +use std::{collections::BTreeMap, fmt::Debug, marker::PhantomData, sync::Arc}; use derivative::Derivative; use espresso_types::v0::traits::{PersistenceOptions, SequencerPersistence}; pub use genesis::Genesis; #[cfg(feature = "libp2p")] -use hotshot::traits::implementations::{CombinedNetworks, GossipConfig, Libp2pNetwork, derive_libp2p_multiaddr}; +use hotshot::traits::implementations::{ + derive_libp2p_multiaddr, CombinedNetworks, GossipConfig, Libp2pNetwork, +}; use hotshot::{ traits::implementations::{ derive_libp2p_peer_id, CdnMetricsValue, CdnTopic, KeyPair, MemoryNetwork, PushCdnNetwork, @@ -48,10 +51,7 @@ use hotshot::{ types::SignatureKey, MarketplaceConfig, }; -use hotshot_orchestrator::{ - client::{OrchestratorClient, ValidatorArgs}, - config::NetworkConfig, -}; +use hotshot_orchestrator::{client::OrchestratorClient, config::NetworkConfig}; use hotshot_types::{ data::ViewNumber, light_client::{StateKeyPair, StateSignKey}, @@ -115,9 +115,9 @@ pub struct NetworkParams { pub public_api_url: Option, /// The address to send to other Libp2p nodes to contact us - pub libp2p_advertise_address: SocketAddr, + pub libp2p_advertise_address: String, /// The address to bind to for Libp2p - pub libp2p_bind_address: SocketAddr, + pub libp2p_bind_address: String, /// The (optional) bootstrap node addresses for Libp2p. If supplied, these will /// override the bootstrap nodes specified in the config file. pub libp2p_bootstrap_nodes: Option>, @@ -202,14 +202,27 @@ pub async fn init_node( .text_family("node".into(), vec!["key".into()]) .create(vec![pub_key.to_string()]); + // Parse the Libp2p bind and advertise addresses to multiaddresses + let libp2p_bind_address = derive_libp2p_multiaddr(&network_params.libp2p_bind_address) + .with_context(|| { + format!( + "Failed to derive Libp2p bind address of {}", + &network_params.libp2p_bind_address + ) + })?; + let libp2p_advertise_address = + derive_libp2p_multiaddr(&network_params.libp2p_advertise_address).with_context(|| { + format!( + "Failed to derive Libp2p advertise address of {}", + &network_params.libp2p_advertise_address + ) + })?; + + info!("Libp2p bind address: {}", libp2p_bind_address); + info!("Libp2p advertise address: {}", libp2p_advertise_address); + // Orchestrator client - let validator_args = ValidatorArgs { - url: network_params.orchestrator_url, - advertise_address: Some(network_params.libp2p_advertise_address.to_string()), - builder_address: None, - network_config_file: None, - }; - let orchestrator_client = OrchestratorClient::new(validator_args.url); + let orchestrator_client = OrchestratorClient::new(network_params.orchestrator_url); let state_key_pair = StateKeyPair::from_sign_key(network_params.private_state_key); let my_config = ValidatorConfig { public_key: pub_key, @@ -254,14 +267,12 @@ pub async fn init_node( tracing::error!( "waiting for other nodes to connect, DO NOT RESTART until fully connected" ); - let multiaddr = - derive_libp2p_multiaddr(&network_params.libp2p_advertise_address.to_string()).expect("Failed to derive valid multiaddr, {}"); let config = NetworkConfig::get_complete_config( &orchestrator_client, my_config.clone(), // Register in our Libp2p advertise address and public key so other nodes // can contact us on startup - Some(multiaddr), + Some(libp2p_advertise_address), Some(libp2p_public_key), ) .await? @@ -330,7 +341,7 @@ pub async fn init_node( let p2p_network = Libp2pNetwork::from_config::( config.clone(), GossipConfig::default(), - network_params.libp2p_bind_address.to_string(), + libp2p_bind_address, &my_config.public_key, // We need the private key so we can derive our Libp2p keypair // (using https://docs.rs/blake3/latest/blake3/fn.derive_key.html) diff --git a/sequencer/src/main.rs b/sequencer/src/main.rs index d0896db50..c95b2c205 100644 --- a/sequencer/src/main.rs +++ b/sequencer/src/main.rs @@ -1,4 +1,4 @@ -use std::{net::ToSocketAddrs, sync::Arc}; +use std::sync::Arc; use clap::Parser; use espresso_types::{ @@ -153,26 +153,10 @@ where events_max_block_range: opt.l1_events_max_block_range, }; - // Parse supplied Libp2p addresses to their socket form - // We expect all nodes to be reachable via IPv4, so we filter out any IPv6 addresses. - // Downstream in HotShot we pin the IP address to v4, but this can be fixed in the future. - let libp2p_advertise_address = opt - .libp2p_advertise_address - .to_socket_addrs()? - .find(|x| x.is_ipv4()) - .ok_or(anyhow::anyhow!( - "Failed to resolve Libp2p advertise address" - ))?; - let libp2p_bind_address = opt - .libp2p_bind_address - .to_socket_addrs()? - .find(|x| x.is_ipv4()) - .ok_or(anyhow::anyhow!("Failed to resolve Libp2p bind address"))?; - let network_params = NetworkParams { cdn_endpoint: opt.cdn_endpoint, - libp2p_advertise_address, - libp2p_bind_address, + libp2p_advertise_address: opt.libp2p_advertise_address, + libp2p_bind_address: opt.libp2p_bind_address, libp2p_bootstrap_nodes: opt.libp2p_bootstrap_nodes, orchestrator_url: opt.orchestrator_url, state_relay_server_url: opt.state_relay_server_url, From ba42decdc864e853887e2e35757c097eff0e64d9 Mon Sep 17 00:00:00 2001 From: Luke Iannucci Date: Thu, 3 Oct 2024 16:16:46 -0400 Subject: [PATCH 03/12] clippy --- builder/src/bin/permissioned-builder.rs | 4 +--- builder/src/non_permissioned.rs | 4 ++-- sequencer/src/bin/cdn-whitelist.rs | 5 +---- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/builder/src/bin/permissioned-builder.rs b/builder/src/bin/permissioned-builder.rs index be7aa9a1d..afba07ee9 100644 --- a/builder/src/bin/permissioned-builder.rs +++ b/builder/src/bin/permissioned-builder.rs @@ -1,6 +1,4 @@ -use std::{ - collections::HashMap, net::ToSocketAddrs, num::NonZeroUsize, path::PathBuf, time::Duration, -}; +use std::{collections::HashMap, num::NonZeroUsize, path::PathBuf, time::Duration}; use anyhow::{bail, Context}; use builder::permissioned::init_node; diff --git a/builder/src/non_permissioned.rs b/builder/src/non_permissioned.rs index ef0004645..1d35abb5c 100644 --- a/builder/src/non_permissioned.rs +++ b/builder/src/non_permissioned.rs @@ -47,7 +47,7 @@ use sequencer::{catchup::StatePeers, L1Params, NetworkParams, SequencerApiVersio use surf::http::headers::ACCEPT; use surf_disco::Client; use tide_disco::{app, method::ReadState, App, Url}; -use vbs::version::{StaticVersionType, Version}; +use vbs::version::{StaticVersion, StaticVersionType, Version}; use crate::run_builder_api_service; @@ -195,7 +195,7 @@ impl BuilderConfig { let global_state_clone = global_state.clone(); tracing::info!("Running permissionless builder against hotshot events API at {events_url}",); async_spawn(async move { - let res = run_non_permissioned_standalone_builder_service::<_, V>( + let res = run_non_permissioned_standalone_builder_service::<_, StaticVersion<0, 1>>( da_sender, qc_sender, decide_sender, diff --git a/sequencer/src/bin/cdn-whitelist.rs b/sequencer/src/bin/cdn-whitelist.rs index c59f14bba..ec6ff63d2 100644 --- a/sequencer/src/bin/cdn-whitelist.rs +++ b/sequencer/src/bin/cdn-whitelist.rs @@ -8,10 +8,7 @@ use anyhow::{Context, Result}; use cdn_broker::reexports::discovery::{DiscoveryClient, Embedded, Redis}; use clap::Parser; use espresso_types::SeqTypes; -use hotshot_orchestrator::{ - client::{OrchestratorClient, ValidatorArgs}, - config::NetworkConfig, -}; +use hotshot_orchestrator::{client::OrchestratorClient, config::NetworkConfig}; use hotshot_types::traits::{node_implementation::NodeType, signature_key::SignatureKey}; use surf_disco::Url; From 8c8a7b57ab60a473734dc505357f3fbbe19707e8 Mon Sep 17 00:00:00 2001 From: Luke Iannucci Date: Fri, 4 Oct 2024 11:43:00 -0400 Subject: [PATCH 04/12] update hotshot --- Cargo.lock | 26 +++++++++++++------------- data/v1/messages.bin | Bin 7396 -> 7396 bytes data/v1/messages.json | 2 +- data/v2/messages.bin | Bin 7404 -> 7404 bytes data/v2/messages.json | 2 +- data/v3/messages.bin | Bin 7444 -> 7444 bytes data/v3/messages.json | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5ccb7a59f..d0b80e557 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4099,7 +4099,7 @@ dependencies = [ [[package]] name = "hotshot" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" dependencies = [ "anyhow", "async-broadcast", @@ -4147,7 +4147,7 @@ dependencies = [ [[package]] name = "hotshot-builder-api" version = "0.1.7" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" dependencies = [ "async-trait", "clap", @@ -4248,7 +4248,7 @@ dependencies = [ [[package]] name = "hotshot-example-types" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" dependencies = [ "anyhow", "async-broadcast", @@ -4281,7 +4281,7 @@ dependencies = [ [[package]] name = "hotshot-fakeapi" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" dependencies = [ "anyhow", "async-lock 2.8.0", @@ -4300,7 +4300,7 @@ dependencies = [ [[package]] name = "hotshot-macros" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" dependencies = [ "derive_builder", "proc-macro2", @@ -4311,7 +4311,7 @@ dependencies = [ [[package]] name = "hotshot-orchestrator" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" dependencies = [ "anyhow", "async-compatibility-layer", @@ -4397,7 +4397,7 @@ dependencies = [ [[package]] name = "hotshot-stake-table" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" dependencies = [ "ark-bn254", "ark-ed-on-bn254", @@ -4459,7 +4459,7 @@ dependencies = [ [[package]] name = "hotshot-task" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" dependencies = [ "anyhow", "async-broadcast", @@ -4474,7 +4474,7 @@ dependencies = [ [[package]] name = "hotshot-task-impls" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" dependencies = [ "anyhow", "async-broadcast", @@ -4511,7 +4511,7 @@ dependencies = [ [[package]] name = "hotshot-testing" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" dependencies = [ "anyhow", "async-broadcast", @@ -4557,7 +4557,7 @@ dependencies = [ [[package]] name = "hotshot-types" version = "0.1.11" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" dependencies = [ "anyhow", "ark-bn254", @@ -5813,7 +5813,7 @@ dependencies = [ [[package]] name = "libp2p-networking" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#35a2a447b6593b6f2cd8bf4c53b36116d1c43c07" +source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" dependencies = [ "anyhow", "async-compatibility-layer", @@ -7555,7 +7555,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.77", diff --git a/data/v1/messages.bin b/data/v1/messages.bin index bb61729181d384fb10407aa7d2447b51461a01cd..d3549aecff3c2c081eec7ef20ea7eca8e637786f 100644 GIT binary patch delta 46 zcmV+}0MY;CIpjI83O+ E8Qhr^3&$FSn{;V(2T5AY^`h}HWnoiz+_JzPglvn>O+ E8R04usQ>@~ diff --git a/data/v1/messages.json b/data/v1/messages.json index 85e80a7e3..0b394cb58 100644 --- a/data/v1/messages.json +++ b/data/v1/messages.json @@ -42,7 +42,7 @@ }, "signatures": null, "view_number": 0, - "vote_commitment": "COMMIT~tzMwZg2x5QJTx7Di9eEvNC0P8CZ_iNX7LJ01DHA9XEdO" + "vote_commitment": "COMMIT~f11eqw2KFTy-i-EaXUfEC1v0CPo8gPcd2My10Y5zSNok" }, "proposal_certificate": { "Timeout": { diff --git a/data/v2/messages.bin b/data/v2/messages.bin index c08c76fbf8ec080bb305cc8faf45a9a7dde40f82..e3549e2752c60e991da995493772ed4531608495 100644 GIT binary patch delta 46 zcmV+}0MY;KIqW&G6ayiDU0$mViWNM*i{TnwN5l(T^a%PqfcG8P%(c;ub4c1Gvo!;^ E8Q{?s1ONa4 delta 46 zcmV+}0MY;KIqW&G6ayi*GcaZivE>3&$FSn{;V(2T5AY^`h}HWnoiz+_JzPglvo!;^ E8RcRV$p8QV diff --git a/data/v2/messages.json b/data/v2/messages.json index bfdd2ddb4..7791e5025 100644 --- a/data/v2/messages.json +++ b/data/v2/messages.json @@ -50,7 +50,7 @@ }, "signatures": null, "view_number": 0, - "vote_commitment": "COMMIT~tzMwZg2x5QJTx7Di9eEvNC0P8CZ_iNX7LJ01DHA9XEdO" + "vote_commitment": "COMMIT~f11eqw2KFTy-i-EaXUfEC1v0CPo8gPcd2My10Y5zSNok" }, "proposal_certificate": { "Timeout": { diff --git a/data/v3/messages.bin b/data/v3/messages.bin index 034e2422cd0babf7606783e1c9f842a67e0ea0e8..2795dea9d84928eef045eaed5632377159ffe6e8 100644 GIT binary patch delta 46 zcmV+}0MY-HI+QxFJOd$rU0$mViWNM*i{TnwN5l(T^a%PqfcG8P%(c;ub4c1Gvt0wX E8OcEurT_o{ delta 46 zcmV+}0MY-HI+QxFJOd%OGcaZivE>3&$FSn{;V(2T5AY^`h}HWnoiz+_JzPglvt0wX E8O_oYYXATM diff --git a/data/v3/messages.json b/data/v3/messages.json index bc22398eb..ccf60b3da 100644 --- a/data/v3/messages.json +++ b/data/v3/messages.json @@ -58,7 +58,7 @@ }, "signatures": null, "view_number": 0, - "vote_commitment": "COMMIT~tzMwZg2x5QJTx7Di9eEvNC0P8CZ_iNX7LJ01DHA9XEdO" + "vote_commitment": "COMMIT~f11eqw2KFTy-i-EaXUfEC1v0CPo8gPcd2My10Y5zSNok" }, "proposal_certificate": { "Timeout": { From 8f0242bd61ac714f77b29cd8e33e3e096d06aaa1 Mon Sep 17 00:00:00 2001 From: Luke Iannucci Date: Fri, 4 Oct 2024 14:37:55 -0400 Subject: [PATCH 05/12] store upgrade certificate and load --- .../migrations/V38__upgrade_certificate.sql | 6 +++ sequencer/src/persistence/fs.rs | 41 ++++++++++++++++++- sequencer/src/persistence/no_storage.rs | 13 +++++- sequencer/src/persistence/sql.rs | 41 ++++++++++++++++++- types/src/v0/traits.rs | 21 ++++++++-- 5 files changed, 116 insertions(+), 6 deletions(-) create mode 100644 sequencer/api/migrations/V38__upgrade_certificate.sql diff --git a/sequencer/api/migrations/V38__upgrade_certificate.sql b/sequencer/api/migrations/V38__upgrade_certificate.sql new file mode 100644 index 000000000..27b1d37ec --- /dev/null +++ b/sequencer/api/migrations/V38__upgrade_certificate.sql @@ -0,0 +1,6 @@ +CREATE TABLE upgrade_certificate ( + -- The ID is always set to 0. Setting it explicitly allows us to enforce with every insert or + -- update that there is only a single entry in this table: the latest known state. + id INT PRIMARY KEY, + data BYTEA +); diff --git a/sequencer/src/persistence/fs.rs b/sequencer/src/persistence/fs.rs index ef3594757..b0d571030 100644 --- a/sequencer/src/persistence/fs.rs +++ b/sequencer/src/persistence/fs.rs @@ -11,7 +11,7 @@ use hotshot_types::{ data::{DaProposal, QuorumProposal, VidDisperseShare}, event::{Event, EventType, HotShotAction, LeafInfo}, message::Proposal, - simple_certificate::QuorumCertificate, + simple_certificate::{QuorumCertificate, UpgradeCertificate}, traits::{block_contents::BlockPayload, node_implementation::ConsensusTime}, utils::View, vote::HasViewNumber, @@ -122,6 +122,10 @@ impl Inner { self.path.join("quorum_proposals") } + fn upgrade_certificate_dir_path(&self) -> PathBuf { + self.path.join("upgrade_certificate") + } + /// Overwrite a file if a condition is met. /// /// The file at `path`, if it exists, is opened in read mode and passed to `pred`. If `pred` @@ -698,6 +702,41 @@ impl SequencerPersistence for Persistence { Ok(map) } + + async fn load_upgrade_certificate( + &self, + ) -> anyhow::Result>> { + let inner = self.inner.read().await; + let path = inner.upgrade_certificate_dir_path(); + if !path.is_file() { + return Ok(None); + } + let bytes = fs::read(&path).context("read")?; + Ok(Some( + bincode::deserialize(&bytes).context("deserialize upgrade certificate")?, + )) + } + + async fn store_upgrade_certificate( + &self, + decided_upgrade_certificate: Option>, + ) -> anyhow::Result<()> { + let mut inner = self.inner.write().await; + let path = &inner.upgrade_certificate_dir_path(); + inner.replace( + path, + |_| { + // Always overwrite the previous file. + Ok(true) + }, + |mut file| { + let bytes = bincode::serialize(&decided_upgrade_certificate) + .context("serializing upgrade certificate")?; + file.write_all(&bytes)?; + Ok(()) + }, + ) + } } /// Update a `NetworkConfig` that may have originally been persisted with an old version. diff --git a/sequencer/src/persistence/no_storage.rs b/sequencer/src/persistence/no_storage.rs index a211390b4..63193e606 100644 --- a/sequencer/src/persistence/no_storage.rs +++ b/sequencer/src/persistence/no_storage.rs @@ -12,7 +12,7 @@ use hotshot_types::{ data::{DaProposal, QuorumProposal, VidDisperseShare}, event::{Event, EventType, HotShotAction, LeafInfo}, message::Proposal, - simple_certificate::QuorumCertificate, + simple_certificate::{QuorumCertificate, UpgradeCertificate}, utils::View, }; use std::collections::BTreeMap; @@ -113,6 +113,11 @@ impl SequencerPersistence for NoStorage { ) -> anyhow::Result>>> { Ok(Default::default()) } + async fn load_upgrade_certificate( + &self, + ) -> anyhow::Result>> { + Ok(None) + } async fn append_vid( &self, @@ -142,4 +147,10 @@ impl SequencerPersistence for NoStorage { ) -> anyhow::Result<()> { Ok(()) } + async fn store_upgrade_certificate( + &self, + _decided_upgrade_certificate: Option>, + ) -> anyhow::Result<()> { + Ok(()) + } } diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index 74099af09..89fe58f30 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -20,7 +20,7 @@ use hotshot_types::{ data::{DaProposal, QuorumProposal, VidDisperseShare}, event::{Event, EventType, HotShotAction, LeafInfo}, message::Proposal, - simple_certificate::QuorumCertificate, + simple_certificate::{QuorumCertificate, UpgradeCertificate}, traits::{node_implementation::ConsensusTime, BlockPayload}, utils::View, vote::HasViewNumber, @@ -546,6 +546,41 @@ impl SequencerPersistence for Persistence { .await?; tx.commit().await } + + async fn load_upgrade_certificate( + &self, + ) -> anyhow::Result>> { + let result = self + .db + .read() + .await? + .query_opt("SELECT * FROM upgrade_certificate where id = 0", [&(0i32)]) + .await?; + + result + .map(|row| { + let bytes: Vec = row.get("data"); + anyhow::Result::<_>::Ok(bincode::deserialize(&bytes)?) + }) + .transpose() + } + + async fn store_upgrade_certificate( + &self, + decided_upgrade_certificate: Option>, + ) -> anyhow::Result<()> { + let upgrade_certificate_bytes = + bincode::serialize(&decided_upgrade_certificate).context("serializing proposal")?; + let mut tx = self.db.write().await?; + tx.upsert( + "upgrade_certificate", + ["id", "data"], + ["id"], + [[sql_param(&0i32), sql_param(&upgrade_certificate_bytes)]], + ) + .await?; + tx.commit().await + } } async fn collect_garbage( @@ -624,6 +659,10 @@ async fn collect_garbage( ) .await?; + // Cleanup upgrade certificate, should only be 1 + tx.execute("DELETE FROM upgrade_certificate where id <= $1", [&(0i32)]) + .await?; + // Exclude from the decide event any leaves which have definitely already been processed. We may // have selected an already-processed leaf because the oldest leaf -- the last leaf processed in // the previous decide event -- remained in the database to serve as the anchor leaf, so our diff --git a/types/src/v0/traits.rs b/types/src/v0/traits.rs index 7b278406d..772e995b5 100644 --- a/types/src/v0/traits.rs +++ b/types/src/v0/traits.rs @@ -361,6 +361,9 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static { &self, view: ViewNumber, ) -> anyhow::Result>>>; + async fn load_upgrade_certificate( + &self, + ) -> anyhow::Result>>; /// Load the latest known consensus state. /// @@ -441,6 +444,11 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static { .await .context("loading saved proposals")?; + let upgrade_certificate = self + .load_upgrade_certificate() + .await + .context("loading upgrade certificate")?; + tracing::info!( ?leaf, ?view, @@ -449,6 +457,7 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static { ?undecided_leaves, ?undecided_state, ?saved_proposals, + ?upgrade_certificate, "loaded consensus state" ); @@ -461,7 +470,7 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static { highest_voted_view, saved_proposals, high_qc, - None, + upgrade_certificate, undecided_leaves.into_values().collect(), undecided_state, ), @@ -550,6 +559,10 @@ pub trait SequencerPersistence: Sized + Send + Sync + 'static { &self, proposal: &Proposal>, ) -> anyhow::Result<()>; + async fn store_upgrade_certificate( + &self, + decided_upgrade_certificate: Option>, + ) -> anyhow::Result<()>; } #[async_trait] @@ -619,9 +632,11 @@ impl Storage for Arc

{ async fn update_decided_upgrade_certificate( &self, - _decided_upgrade_certificate: Option>, + decided_upgrade_certificate: Option>, ) -> anyhow::Result<()> { - Ok(()) + (**self) + .store_upgrade_certificate(decided_upgrade_certificate) + .await } } From 49a6eaa90ac5f1271e8b3257d9f42afaa8af50aa Mon Sep 17 00:00:00 2001 From: Luke Iannucci Date: Fri, 4 Oct 2024 14:54:03 -0400 Subject: [PATCH 06/12] remove unnecessary param --- sequencer/src/persistence/sql.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index 89fe58f30..b6ccd66d2 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -554,7 +554,7 @@ impl SequencerPersistence for Persistence { .db .read() .await? - .query_opt("SELECT * FROM upgrade_certificate where id = 0", [&(0i32)]) + .query_opt_static("SELECT * FROM upgrade_certificate where id = 0") .await?; result From 6c5dfda1f72dc9a702b7d569fdc2a229e430595d Mon Sep 17 00:00:00 2001 From: lukeiannucci Date: Mon, 7 Oct 2024 15:07:30 -0600 Subject: [PATCH 07/12] update tags --- Cargo.lock | 32 ++++++++++++++++---------------- Cargo.toml | 26 +++++++++++++------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d0b80e557..850d3b33c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4099,7 +4099,7 @@ dependencies = [ [[package]] name = "hotshot" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" +source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.77#e8c3e75775e2124f11a60ec4d6a6090d012dbb35" dependencies = [ "anyhow", "async-broadcast", @@ -4147,7 +4147,7 @@ dependencies = [ [[package]] name = "hotshot-builder-api" version = "0.1.7" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" +source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.77#e8c3e75775e2124f11a60ec4d6a6090d012dbb35" dependencies = [ "async-trait", "clap", @@ -4166,7 +4166,7 @@ dependencies = [ [[package]] name = "hotshot-builder-core" version = "0.1.51" -source = "git+https://github.com/EspressoSystems/marketplace-builder-core?branch=hotshot/rc-0.5.77#08a9c61d3ae17348e3043a517aef46bab445c971" +source = "git+https://github.com/EspressoSystems/marketplace-builder-core?tag=0.1.51#da0ac05c1c3dd8708a6cebc74334194e6816337e" dependencies = [ "anyhow", "async-broadcast", @@ -4222,7 +4222,7 @@ dependencies = [ [[package]] name = "hotshot-events-service" version = "0.1.49" -source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?branch=hotshot/rc-0.5.77#3b81fc059818fe273bc2cf7aea575514d233ecd4" +source = "git+https://github.com/EspressoSystems/hotshot-events-service.git?tag=0.1.49#06700117ec9a5f73f042906b6b3eec4a7b28d0de" dependencies = [ "async-broadcast", "async-compatibility-layer", @@ -4248,7 +4248,7 @@ dependencies = [ [[package]] name = "hotshot-example-types" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" +source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.77#e8c3e75775e2124f11a60ec4d6a6090d012dbb35" dependencies = [ "anyhow", "async-broadcast", @@ -4281,7 +4281,7 @@ dependencies = [ [[package]] name = "hotshot-fakeapi" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" +source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.77#e8c3e75775e2124f11a60ec4d6a6090d012dbb35" dependencies = [ "anyhow", "async-lock 2.8.0", @@ -4300,7 +4300,7 @@ dependencies = [ [[package]] name = "hotshot-macros" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" +source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.77#e8c3e75775e2124f11a60ec4d6a6090d012dbb35" dependencies = [ "derive_builder", "proc-macro2", @@ -4311,7 +4311,7 @@ dependencies = [ [[package]] name = "hotshot-orchestrator" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" +source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.77#e8c3e75775e2124f11a60ec4d6a6090d012dbb35" dependencies = [ "anyhow", "async-compatibility-layer", @@ -4341,7 +4341,7 @@ dependencies = [ [[package]] name = "hotshot-query-service" version = "0.1.62" -source = "git+https://github.com/EspressoSystems/hotshot-query-service?branch=hotshot/rc-0.5.77#3003620bfac88e14a76bee9998929a606b586585" +source = "git+https://github.com/EspressoSystems/hotshot-query-service?tag=0.1.62#36bb46fef9087045739225b9342f899bcbf9e1ab" dependencies = [ "anyhow", "ark-serialize", @@ -4397,7 +4397,7 @@ dependencies = [ [[package]] name = "hotshot-stake-table" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" +source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.77#e8c3e75775e2124f11a60ec4d6a6090d012dbb35" dependencies = [ "ark-bn254", "ark-ed-on-bn254", @@ -4459,7 +4459,7 @@ dependencies = [ [[package]] name = "hotshot-task" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" +source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.77#e8c3e75775e2124f11a60ec4d6a6090d012dbb35" dependencies = [ "anyhow", "async-broadcast", @@ -4474,7 +4474,7 @@ dependencies = [ [[package]] name = "hotshot-task-impls" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" +source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.77#e8c3e75775e2124f11a60ec4d6a6090d012dbb35" dependencies = [ "anyhow", "async-broadcast", @@ -4511,7 +4511,7 @@ dependencies = [ [[package]] name = "hotshot-testing" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" +source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.77#e8c3e75775e2124f11a60ec4d6a6090d012dbb35" dependencies = [ "anyhow", "async-broadcast", @@ -4557,7 +4557,7 @@ dependencies = [ [[package]] name = "hotshot-types" version = "0.1.11" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" +source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.77#e8c3e75775e2124f11a60ec4d6a6090d012dbb35" dependencies = [ "anyhow", "ark-bn254", @@ -5813,7 +5813,7 @@ dependencies = [ [[package]] name = "libp2p-networking" version = "0.5.77" -source = "git+https://github.com/EspressoSystems/hotshot?branch=bump/rc-0.5.77#37608bbc478744c5ccfe913dfa5bd22ed469e86c" +source = "git+https://github.com/EspressoSystems/hotshot?tag=0.5.77#e8c3e75775e2124f11a60ec4d6a6090d012dbb35" dependencies = [ "anyhow", "async-compatibility-layer", @@ -6201,7 +6201,7 @@ dependencies = [ [[package]] name = "marketplace-builder-core" version = "0.1.51" -source = "git+https://github.com/EspressoSystems/marketplace-builder-core?branch=hotshot/rc-0.5.77#08a9c61d3ae17348e3043a517aef46bab445c971" +source = "git+https://github.com/EspressoSystems/marketplace-builder-core?tag=0.1.51#da0ac05c1c3dd8708a6cebc74334194e6816337e" dependencies = [ "anyhow", "async-broadcast", diff --git a/Cargo.toml b/Cargo.toml index 73295c033..5fb6bbf4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,23 +57,23 @@ dyn-clone = "1.0" ethers = { version = "2.0", features = ["solc"] } futures = "0.3" -hotshot = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77", features = ["dependency-tasks"] } +hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77", features = ["dependency-tasks"] } # Hotshot imports -hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", branch = "bump/rc-0.5.77" } -hotshot-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", branch = "hotshot/rc-0.5.77" } -marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", branch = "hotshot/rc-0.5.77" } -hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", branch = "hotshot/rc-0.5.77" } -hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77" } -hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", branch = "hotshot/rc-0.5.77" } -hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77" } +hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.77" } +hotshot-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.1.51" } +marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.1.51" } +hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.49" } +hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77" } +hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.62" } +hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77" } hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" } -hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77" } -hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77", features = ["dependency-tasks"] } -hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77" } -libp2p-networking = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77" } +hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77" } +hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77", features = ["dependency-tasks"] } +hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77" } +libp2p-networking = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77" } hotshot-contract-adapter = { version = "0.1.0", path = "contracts/rust/adapter" } # Temporary, used to pull in the mock auction results provider -hotshot-example-types = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/rc-0.5.77" } +hotshot-example-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.77" } # Push CDN imports cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", features = [ From 5df0df7b353aa4535f575bc02e679be1143f73f1 Mon Sep 17 00:00:00 2001 From: lukeiannucci Date: Mon, 7 Oct 2024 15:17:24 -0600 Subject: [PATCH 08/12] fix typo --- sequencer/src/persistence/sql.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index b6ccd66d2..4833f9185 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -570,7 +570,7 @@ impl SequencerPersistence for Persistence { decided_upgrade_certificate: Option>, ) -> anyhow::Result<()> { let upgrade_certificate_bytes = - bincode::serialize(&decided_upgrade_certificate).context("serializing proposal")?; + bincode::serialize(&decided_upgrade_certificate).context("serializing upgrade certificate")?; let mut tx = self.db.write().await?; tx.upsert( "upgrade_certificate", From ce1043e75ff7342a3d4f6a7dd42b8f6bf3b49728 Mon Sep 17 00:00:00 2001 From: lukeiannucci Date: Mon, 7 Oct 2024 15:22:20 -0600 Subject: [PATCH 09/12] lint --- sequencer/src/persistence/sql.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index 4833f9185..2e5fda601 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -569,8 +569,8 @@ impl SequencerPersistence for Persistence { &self, decided_upgrade_certificate: Option>, ) -> anyhow::Result<()> { - let upgrade_certificate_bytes = - bincode::serialize(&decided_upgrade_certificate).context("serializing upgrade certificate")?; + let upgrade_certificate_bytes = bincode::serialize(&decided_upgrade_certificate) + .context("serializing upgrade certificate")?; let mut tx = self.db.write().await?; tx.upsert( "upgrade_certificate", From 0f8f8e4efe3280784afbb40ebb2051e383263a6d Mon Sep 17 00:00:00 2001 From: lukeiannucci Date: Tue, 8 Oct 2024 09:53:33 -0600 Subject: [PATCH 10/12] add tests --- sequencer/src/persistence.rs | 48 +++++++++++++++++++++++++++++++- sequencer/src/persistence/sql.rs | 4 --- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/sequencer/src/persistence.rs b/sequencer/src/persistence.rs index 752d02343..494a5b97b 100644 --- a/sequencer/src/persistence.rs +++ b/sequencer/src/persistence.rs @@ -66,13 +66,15 @@ mod persistence_tests { data::{DaProposal, QuorumProposal, VidDisperseShare, ViewNumber}, event::{EventType, HotShotAction, LeafInfo}, message::Proposal, - simple_certificate::QuorumCertificate, + simple_certificate::{QuorumCertificate, UpgradeCertificate}, + simple_vote::UpgradeProposalData, traits::{node_implementation::ConsensusTime, EncodeBytes}, vid::vid_scheme, }; use jf_vid::VidScheme; use sequencer_utils::test_utils::setup_test; use testing::TestablePersistence; + use vbs::version::Version; use super::*; @@ -469,6 +471,50 @@ mod persistence_tests { ); } + #[async_std::test] + pub async fn test_upgrade_certificate() { + setup_test(); + + let tmp = P::tmp_storage().await; + let storage = P::connect(&tmp).await; + + // Test get upgrade certificate + assert_eq!(storage.load_upgrade_certificate().await.unwrap(), None); + + let upgrade_data = UpgradeProposalData { + old_version: Version { major: 0, minor: 1 }, + new_version: Version { major: 1, minor: 0 }, + decide_by: ViewNumber::genesis(), + new_version_hash: Default::default(), + old_version_last_view: ViewNumber::genesis(), + new_version_first_view: ViewNumber::genesis(), + }; + + let decide_upgrade_certificate = UpgradeCertificate::::new( + upgrade_data.clone(), + upgrade_data.commit(), + ViewNumber::genesis(), + Default::default(), + Default::default(), + ); + storage + .store_upgrade_certificate(Some(decide_upgrade_certificate.clone())) + .await + .unwrap(); + + let res = storage.load_upgrade_certificate().await.unwrap(); + let view_number = res.unwrap().view_number; + assert_eq!(view_number, ViewNumber::genesis()); + + let new_view_number_for_certificate = ViewNumber::new(50); + let mut new_upgrade_certificate = decide_upgrade_certificate.clone(); + new_upgrade_certificate.view_number = new_view_number_for_certificate; + + let res = storage.load_upgrade_certificate().await.unwrap(); + let view_number = res.unwrap().view_number; + assert_eq!(view_number, new_view_number_for_certificate); + } + #[async_std::test] pub async fn test_decide_with_failing_event_consumer() { #[derive(Clone, Copy, Debug)] diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index 2e5fda601..ed406ea71 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -659,10 +659,6 @@ async fn collect_garbage( ) .await?; - // Cleanup upgrade certificate, should only be 1 - tx.execute("DELETE FROM upgrade_certificate where id <= $1", [&(0i32)]) - .await?; - // Exclude from the decide event any leaves which have definitely already been processed. We may // have selected an already-processed leaf because the oldest leaf -- the last leaf processed in // the previous decide event -- remained in the database to serve as the anchor leaf, so our From 85931e584a6aa078a85e3e6ad2f57ea955903b97 Mon Sep 17 00:00:00 2001 From: lukeiannucci Date: Tue, 8 Oct 2024 11:03:00 -0600 Subject: [PATCH 11/12] only store if certificate is provided --- sequencer/src/persistence.rs | 11 ++++++++--- sequencer/src/persistence/fs.rs | 8 ++++++-- sequencer/src/persistence/sql.rs | 8 ++++++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/sequencer/src/persistence.rs b/sequencer/src/persistence.rs index 494a5b97b..69266db6f 100644 --- a/sequencer/src/persistence.rs +++ b/sequencer/src/persistence.rs @@ -497,10 +497,10 @@ mod persistence_tests { Default::default(), Default::default(), ); - storage + let res = storage .store_upgrade_certificate(Some(decide_upgrade_certificate.clone())) - .await - .unwrap(); + .await; + assert!(res.is_ok()); let res = storage.load_upgrade_certificate().await.unwrap(); let view_number = res.unwrap().view_number; @@ -510,6 +510,11 @@ mod persistence_tests { let mut new_upgrade_certificate = decide_upgrade_certificate.clone(); new_upgrade_certificate.view_number = new_view_number_for_certificate; + let res = storage + .store_upgrade_certificate(Some(new_upgrade_certificate.clone())) + .await; + assert!(res.is_ok()); + let res = storage.load_upgrade_certificate().await.unwrap(); let view_number = res.unwrap().view_number; assert_eq!(view_number, new_view_number_for_certificate); diff --git a/sequencer/src/persistence/fs.rs b/sequencer/src/persistence/fs.rs index b0d571030..2f346be26 100644 --- a/sequencer/src/persistence/fs.rs +++ b/sequencer/src/persistence/fs.rs @@ -723,6 +723,10 @@ impl SequencerPersistence for Persistence { ) -> anyhow::Result<()> { let mut inner = self.inner.write().await; let path = &inner.upgrade_certificate_dir_path(); + let certificate = match decided_upgrade_certificate { + Some(cert) => cert, + None => return Ok(()), + }; inner.replace( path, |_| { @@ -730,8 +734,8 @@ impl SequencerPersistence for Persistence { Ok(true) }, |mut file| { - let bytes = bincode::serialize(&decided_upgrade_certificate) - .context("serializing upgrade certificate")?; + let bytes = + bincode::serialize(&certificate).context("serializing upgrade certificate")?; file.write_all(&bytes)?; Ok(()) }, diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index ed406ea71..34322fba1 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -569,8 +569,12 @@ impl SequencerPersistence for Persistence { &self, decided_upgrade_certificate: Option>, ) -> anyhow::Result<()> { - let upgrade_certificate_bytes = bincode::serialize(&decided_upgrade_certificate) - .context("serializing upgrade certificate")?; + let certificate = match decided_upgrade_certificate { + Some(cert) => cert, + None => return Ok(()), + }; + let upgrade_certificate_bytes = + bincode::serialize(&certificate).context("serializing upgrade certificate")?; let mut tx = self.db.write().await?; tx.upsert( "upgrade_certificate", From e225c69f829e2558a3c9a0d1a677a1c7568a59c2 Mon Sep 17 00:00:00 2001 From: lukeiannucci Date: Tue, 8 Oct 2024 13:56:28 -0600 Subject: [PATCH 12/12] force one entry --- sequencer/api/migrations/V38__upgrade_certificate.sql | 5 ++--- sequencer/src/persistence/sql.rs | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/sequencer/api/migrations/V38__upgrade_certificate.sql b/sequencer/api/migrations/V38__upgrade_certificate.sql index 27b1d37ec..0fa909bfe 100644 --- a/sequencer/api/migrations/V38__upgrade_certificate.sql +++ b/sequencer/api/migrations/V38__upgrade_certificate.sql @@ -1,6 +1,5 @@ CREATE TABLE upgrade_certificate ( - -- The ID is always set to 0. Setting it explicitly allows us to enforce with every insert or - -- update that there is only a single entry in this table: the latest known state. - id INT PRIMARY KEY, + id bool PRIMARY KEY DEFAULT true, data BYTEA ); +REVOKE DELETE, TRUNCATE ON upgrade_certificate FROM public; diff --git a/sequencer/src/persistence/sql.rs b/sequencer/src/persistence/sql.rs index 34322fba1..86fae931d 100644 --- a/sequencer/src/persistence/sql.rs +++ b/sequencer/src/persistence/sql.rs @@ -554,7 +554,7 @@ impl SequencerPersistence for Persistence { .db .read() .await? - .query_opt_static("SELECT * FROM upgrade_certificate where id = 0") + .query_opt_static("SELECT * FROM upgrade_certificate where id = true") .await?; result @@ -580,7 +580,7 @@ impl SequencerPersistence for Persistence { "upgrade_certificate", ["id", "data"], ["id"], - [[sql_param(&0i32), sql_param(&upgrade_certificate_bytes)]], + [[sql_param(&true), sql_param(&upgrade_certificate_bytes)]], ) .await?; tx.commit().await