From 0953aa8d4ba6c52a5859f772fd2c37e7a1b09ef1 Mon Sep 17 00:00:00 2001 From: Harry Solovay Date: Fri, 6 Sep 2024 10:31:26 -0400 Subject: [PATCH] Bump Versions, Change Ledger Loading, Update Proposal Schema (#8) Co-authored-by: John Marcou --- .env.example | 2 + .vscode/settings.json | 6 ++ server/Cargo.lock | 86 ++++++++++++++++++- server/Cargo.toml | 7 +- .../up.sql | 58 +++++++++++-- server/src/config.rs | 10 ++- server/src/main.rs | 3 +- server/src/models/diesel.rs | 1 + server/src/models/ledger.rs | 71 +++++++-------- server/src/routes/proposal.rs | 9 +- server/src/schema.rs | 1 + web/package.json | 2 +- 12 files changed, 199 insertions(+), 57 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.env.example b/.env.example index abd1a0c9a..91d49fa90 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,8 @@ DB_PORT=5432 DB_NAME=db DATABASE_URL=postgresql://granola:systems@127.0.0.1:5432/db +BUCKET_NAME="673156464838-mina-staking-ledgers" + # [REQUIRED] - the connection URL for the archive database. # ARCHIVE_DATABASE_URL=postgresql://granola:systems@127.0.0.1:5432/db diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..34f21e8a9 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "[rust]": { + "editor.defaultFormatter": "rust-lang.rust-analyzer" + }, + "editor.formatOnSave": true +} diff --git a/server/Cargo.lock b/server/Cargo.lock index a017372b4..6eeddc4b0 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "ahash" version = "0.7.8" @@ -185,7 +191,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.4", "object", "rustc-demangle", ] @@ -416,6 +422,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.13" @@ -608,6 +623,28 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + +[[package]] +name = "flate2" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" +dependencies = [ + "crc32fast", + "miniz_oxide 0.8.0", +] + [[package]] name = "fnv" version = "1.0.7" @@ -931,6 +968,17 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.6.0", + "libc", + "redox_syscall", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -982,15 +1030,17 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mina-ocv-server" -version = "0.3.0" +version = "0.4.0" dependencies = [ "anyhow", "axum", + "bigdecimal", "bs58", "bytes", "clap", "diesel", "diesel-derive-enum", + "flate2", "futures-util", "moka", "r2d2", @@ -998,6 +1048,7 @@ dependencies = [ "rust_decimal", "serde", "serde_json", + "tar", "thiserror", "tokio", "tower", @@ -1015,6 +1066,15 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "1.0.2" @@ -1865,6 +1925,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb797dad5fb5b76fcf519e702f4a589483b5ef06567f160c392832c1f5e44909" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.12.0" @@ -2474,6 +2545,17 @@ dependencies = [ "tap", ] +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + [[package]] name = "zerocopy" version = "0.7.35" diff --git a/server/Cargo.toml b/server/Cargo.toml index 25d11af70..8fcb41fff 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,12 +1,12 @@ [package] name = "mina-ocv-server" -version = "0.3.0" +version = "0.4.0" edition = "2021" [dependencies] # Core dependencies axum = { version = "0.6.4", features = ["tower-log"] } -bs58 = {version = "0.4.0", features = ["check"]} +bs58 = { version = "0.4.0", features = ["check"] } diesel = { version = "2.0.3", features = ["postgres", "r2d2", "numeric"] } diesel-derive-enum = { version = "2.0.0", features = ["postgres"] } clap = { version = "4.1.4", features = ["derive", "env"] } @@ -27,3 +27,6 @@ bytes = "1.4.0" futures-util = "0.3" rust_decimal = "1.28.0" r2d2 = "0.8.10" +flate2 = "1.0.33" +tar = "0.4.41" +bigdecimal = "0.4.5" diff --git a/server/migrations/2023-08-20-211633_add_initial_schema/up.sql b/server/migrations/2023-08-20-211633_add_initial_schema/up.sql index 8c5858ac7..0b84e21ed 100644 --- a/server/migrations/2023-08-20-211633_add_initial_schema/up.sql +++ b/server/migrations/2023-08-20-211633_add_initial_schema/up.sql @@ -1,13 +1,18 @@ -- Your SQL goes here - CREATE TYPE proposal_version AS ENUM ('V1', 'V2'); -CREATE TYPE proposal_category AS ENUM ('Core', 'Networking', 'Interface', 'ERC', 'Cryptography'); - +CREATE TYPE proposal_category AS ENUM ( + 'Core', + 'Networking', + 'Interface', + 'ERC', + 'Cryptography' +); CREATE TABLE mina_proposals ( id SERIAL PRIMARY KEY, key TEXT NOT NULL UNIQUE, start_time BIGINT NOT NULL, end_time BIGINT NOT NULL, + epoch BIGINT NOT NULL, ledger_hash TEXT, category proposal_category NOT NULL, version proposal_version NOT NULL DEFAULT 'V2', @@ -15,9 +20,46 @@ CREATE TABLE mina_proposals ( description TEXT NOT NULL, url TEXT NOT NULL ); - CREATE INDEX mina_proposals_key_idx ON mina_proposals (key); - -INSERT INTO mina_proposals VALUES (1, 'MIP1', 1672848000000, 1673685000000, 'jxQXzUkst2L9Ma9g9YQ3kfpgB5v5Znr1vrYb1mupakc5y7T89H8', 'Core', 'V1', 'Remove supercharged rewards', 'Removing the short-term incentive of supercharged rewards.', 'https://github.com/MinaProtocol/MIPs/blob/main/MIPS/mip-remove-supercharged-rewards.md'); -INSERT INTO mina_proposals VALUES (3, 'MIP3', 1684562400000, 1685253600000, 'jw8dXuUqXVgd6NvmpryGmFLnRv1176oozHAro8gMFwj8yuvhBeS', 'Cryptography', 'V2', 'Kimchi, a new proof system', 'Kimchi is an update to the proof system currently used by Mina.', 'https://github.com/MinaProtocol/MIPs/blob/main/MIPS/mip-kimchi.md'); -INSERT INTO mina_proposals VALUES (4, 'MIP4', 1684562400000, 1685253600000, 'jw8dXuUqXVgd6NvmpryGmFLnRv1176oozHAro8gMFwj8yuvhBeS', 'Core', 'V2', 'Easier zkApp programmability on mainnet', 'Adding programmable smart contracts (zkApps) to the Mina protocol.', 'https://github.com/MinaProtocol/MIPs/blob/main/MIPS/mip-zkapps.md') +INSERT INTO mina_proposals +VALUES ( + 1, + 'MIP1', + 1672848000000, + 1673685000000, + 46, + 'jxQXzUkst2L9Ma9g9YQ3kfpgB5v5Znr1vrYb1mupakc5y7T89H8', + 'Core', + 'V1', + 'Remove supercharged rewards', + 'Removing the short-term incentive of supercharged rewards.', + 'https://github.com/MinaProtocol/MIPs/blob/main/MIPS/mip-remove-supercharged-rewards.md' + ); +INSERT INTO mina_proposals +VALUES ( + 3, + 'MIP3', + 1684562400000, + 1685253600000, + 55, + 'jw8dXuUqXVgd6NvmpryGmFLnRv1176oozHAro8gMFwj8yuvhBeS', + 'Cryptography', + 'V2', + 'Kimchi, a new proof system', + 'Kimchi is an update to the proof system currently used by Mina.', + 'https://github.com/MinaProtocol/MIPs/blob/main/MIPS/mip-kimchi.md' + ); +INSERT INTO mina_proposals +VALUES ( + 4, + 'MIP4', + 1684562400000, + 1685253600000, + 55, + 'jw8dXuUqXVgd6NvmpryGmFLnRv1176oozHAro8gMFwj8yuvhBeS', + 'Core', + 'V2', + 'Easier zkApp programmability on mainnet', + 'Adding programmable smart contracts (zkApps) to the Mina protocol.', + 'https://github.com/MinaProtocol/MIPs/blob/main/MIPS/mip-zkapps.md' + ) \ No newline at end of file diff --git a/server/src/config.rs b/server/src/config.rs index 741cb58b8..5813dd0c9 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -17,7 +17,8 @@ pub(crate) struct Context { pub(crate) cache: Arc, pub(crate) conn_manager: Arc, pub(crate) network: NetworkConfig, - pub(crate) ledger_storage_path: Option, + pub(crate) ledger_storage_path: String, + pub(crate) bucket_name: String, } #[derive(Clone, Copy, Parser, ValueEnum, Debug)] @@ -54,9 +55,12 @@ pub(crate) struct Config { /// Origins allowed to make cross-site requests. #[clap(long, env = "SERVER_ALLOWED_ORIGINS", value_parser = parse_allowed_origins )] pub(crate) allowed_origins: HashSet, - /// Override ledger storage location. + /// Set the name of the bucket containing the ledgers #[clap(long, env)] - pub(crate) ledger_storage_path: Option, + pub(crate) bucket_name: String, + /// Path to store the ledgers + #[clap(long, env, default_value = "/tmp/ledgers")] + pub(crate) ledger_storage_path: String, } #[allow(clippy::unnecessary_wraps)] diff --git a/server/src/main.rs b/server/src/main.rs index dde61120f..3000b84be 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -6,7 +6,7 @@ use tokio::signal; use tower::ServiceBuilder; use tower_http::trace::TraceLayer; -use crate::config::{Context, Config}; +use crate::config::{Config, Context}; use crate::database::{cache::CacheManager, DBConnectionManager}; use crate::prelude::*; use crate::routes::Build; @@ -47,6 +47,7 @@ async fn main() -> Result<()> { conn_manager: Arc::new(conn_manager), network: config.mina_network, ledger_storage_path: config.ledger_storage_path, + bucket_name: config.bucket_name, })), ); diff --git a/server/src/models/diesel.rs b/server/src/models/diesel.rs index 5daa11aeb..3ed8ba5aa 100644 --- a/server/src/models/diesel.rs +++ b/server/src/models/diesel.rs @@ -28,6 +28,7 @@ pub(crate) struct MinaProposal { pub(crate) key: String, pub(crate) start_time: i64, pub(crate) end_time: i64, + pub(crate) epoch: i64, pub(crate) ledger_hash: Option, pub(crate) category: ProposalCategory, pub(crate) version: ProposalVersion, diff --git a/server/src/models/ledger.rs b/server/src/models/ledger.rs index 2aa023ed7..956d3adc4 100644 --- a/server/src/models/ledger.rs +++ b/server/src/models/ledger.rs @@ -1,11 +1,11 @@ use std::collections::HashMap; use std::io::Read; +use std::path::Path; use anyhow::anyhow; use anyhow::Context; use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; -use tracing::error; use crate::config::NetworkConfig; use crate::models::diesel::ProposalVersion; @@ -25,52 +25,45 @@ pub(crate) struct LedgerAccount { pub(crate) delegate: String, } +// let url = "https://673156464838-mina-staking-ledgers.s3.us-west-2.amazonaws.com/mainnet/mainnet-74-jxvumaCvujr7UzW1qCB87YR2RWu8CqvkwrCmHY8kkwpvN4WbTJn.json.tar.gz"; + impl Ledger { pub(crate) async fn fetch( hash: impl Into, + ledger_storage_path: String, network: NetworkConfig, - ledger_storage_location: &Option, + bucket_name: String, + epoch: i64, ) -> Result { - let hash = hash.into(); - - match ledger_storage_location { - Some(ledger_storage_location) => { - let mut bytes = Vec::new(); - - std::fs::File::open(f!("{ledger_storage_location}/{hash}.json")) - .with_context(|| f!("failed to open ledger {hash}"))? - .read_to_end(&mut bytes) - .with_context(|| f!("failed to read ledger {hash}"))?; - - Ok(Ledger(serde_json::from_slice(&bytes).with_context( - || f!("failed to deserialize ledger {hash}"), - )?)) + let hash: String = hash.into(); + let ledger_file_path = f!("{ledger_storage_path}/{network}-{epoch}-{hash}.json"); + if !Path::new(&ledger_file_path).exists() { + if !Path::new(&ledger_storage_path).exists() { + let _ = std::fs::create_dir_all(ledger_storage_path.clone()); } - None => { - let ledger_url = format!( - "https://raw.githubusercontent.com/Granola-Team/mina-ledger/main/{network}/{hash}.json"); - - let ledger_response = reqwest::get(&ledger_url) - .await - .with_context(|| format!("failed to fetch ledger from URL: {ledger_url}")); - - if let Err(error) = ledger_response { - error!("The reason fetching ledger from URL failed: {error:?}"); - - return Err(error.into()); - } - - let ledger_bytes = ledger_response - .expect("Failed to fetch ledger response") - .bytes() - .await - .with_context(|| "failed to parse ledger response body")?; - - Ok(Ledger(serde_json::from_slice(&ledger_bytes).with_context( - || format!("failed to deserialize ledger data from URL: {ledger_url}"), - )?)) + let url = f!("https://{bucket_name}.s3.us-west-2.amazonaws.com/{network}/{network}-{epoch}-{hash}.json.tar.gz"); + tracing::info!( + "Ledger path not found, downloading {} to {}", + url, + ledger_file_path + ); + let response = reqwest::get(url).await.unwrap(); + if response.status().is_success() { + // Get the object body as bytes + let body = response.bytes().await.unwrap(); + let tar_gz = flate2::read::GzDecoder::new(&body[..]); + let mut archive = tar::Archive::new(tar_gz); + archive.unpack(ledger_storage_path).unwrap(); } } + let mut bytes = Vec::new(); + std::fs::File::open(ledger_file_path) + .with_context(|| f!("failed to open ledger {hash}"))? + .read_to_end(&mut bytes) + .with_context(|| f!("failed to read ledger {hash}"))?; + Ok(Ledger(serde_json::from_slice(&bytes).with_context( + || f!("failed to deserialize ledger {hash}"), + )?)) } pub(crate) fn get_stake_weight( diff --git a/server/src/routes/proposal.rs b/server/src/routes/proposal.rs index f1ad810e7..3e6b5eec8 100644 --- a/server/src/routes/proposal.rs +++ b/server/src/routes/proposal.rs @@ -147,7 +147,14 @@ async fn get_mina_proposal_result( let ledger = if let Some(cached_ledger) = ctx.cache.ledger.get(&hash).await { Ledger(cached_ledger.to_vec()) } else { - let ledger = Ledger::fetch(&hash, ctx.network, &ctx.ledger_storage_path).await?; + let ledger = Ledger::fetch( + &hash, + ctx.ledger_storage_path.clone(), + ctx.network, + ctx.bucket_name.clone(), + proposal.epoch, + ) + .await?; ctx.cache .ledger diff --git a/server/src/schema.rs b/server/src/schema.rs index e9d63e16f..dcf577956 100644 --- a/server/src/schema.rs +++ b/server/src/schema.rs @@ -20,6 +20,7 @@ diesel::table! { key -> Text, start_time -> Int8, end_time -> Int8, + epoch -> Int8, ledger_hash -> Nullable, category -> ProposalCategory, version -> ProposalVersion, diff --git a/web/package.json b/web/package.json index b44e1bf99..b5d669d12 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "mina-ocv-web", - "version": "0.3.0", + "version": "0.4.0", "private": true, "engines": { "node": ">=18.0.0",