[Auto Bench] Scripts of Running Benchmarks in AWS (#2793)

* update setting * update webserver pub ip * updated dockerfile * dockerfile updated * clean up log and scripts running on ecs * real script added * fix * update script and config file * parameterizing webserver url * add to ci * to pass ci docker test * to pass ci docker test again * test ci with script, just and ecs * fix duplicate part for ci test * fix ci test * today's last try with ci * fix merge * really last ci try * commented out ci * fix scripts * add file * fix config * fix config * fix bug after upgradability? * init try on parameterization in script * better parameterization for script * clean up and more parameterization * fix lint * finish parameterization for scripts * preserve yml for ci * add blank line * fix less than or equal in script * upload results for init run * last blank line * nit for nano representation * change back transaction size * remove autobench on ci * remove autobench on ci * remove ci_ecs_benchmarks.sh * remove ci autobench script
EspressoSystems · Mar 23, 2024 · 66c8b3d · 66c8b3d
1 parent beee7b9
commit 66c8b3d
Show file tree

Hide file tree

Showing 14 changed files with 281 additions and 57 deletions.
diff --git a/.gitignore b/.gitignore
@@ -12,4 +12,9 @@
 **/.direnv
 **/*.bkp
 *.sqlite
-results.csv
+results.csv
+.github/workflows/preserve-build-and-test-self-hosted.yml
+.github/workflows/preserve-build-and-test.yml
+config/ValidatorConfigOutput
+scripts/aws_ecs_nginx_benchmarks.sh
+scripts/preserve_ci_ecs_benchmarks.sh
diff --git a/config/ValidatorConfigExample b/config/ValidatorConfigExample
@@ -1,28 +1,32 @@
 ValidatorConfig { 
-    public_key: BLSPubKey { 
-        pub_key: VerKey(
-            (
-                QuadExtField(
-                    2264797523581107490935262917175769123227923636811928330606075281145117212394
-                     + 15807017392833049888165434456991157794698032464874424842715555348468160607934 * u
-                     ), 
-                QuadExtField(
-                    7996517616082121122160563552650547601395271017260499735456299700133762512689
-                     + 7504045709281061282278228438613345070383424761478787301859187055302953740948 * u
-                     ), 
-                QuadExtField(
-                    1515973040548822760825076242090160370742046237881440422068330135941139244581
-                     + 20251846261653098602911417004145145971080304248810966341160788194007704966108 * u
-                     )
-                )
+    public_key: VerKey(
+        (
+            QuadExtField(2264797523581107490935262917175769123227923636811928330606075281145117212394 + 15807017392833049888165434456991157794698032464874424842715555348468160607934 * u), 
+            QuadExtField(7996517616082121122160563552650547601395271017260499735456299700133762512689 + 7504045709281061282278228438613345070383424761478787301859187055302953740948 * u), 
+            QuadExtField(1515973040548822760825076242090160370742046237881440422068330135941139244581 + 20251846261653098602911417004145145971080304248810966341160788194007704966108 * u)
+        )
+    ), 
+    private_key: SignKey(
+        BigInt(
+            [3505488234151006356, 6655477166151225138, 3291219027844407676, 2153641080015542578]
+        )
+    ), 
+    stake_value: 1, 
+    state_key_pair: StateKeyPair(
+        KeyPair { 
+            sk: SignKey(
+                    BigInt(
+                        [2822822805887490846, 6664316196088353173, 4926510007447087464, 116097479308258694]
+                    )
+                ), 
+            vk: VerKey(
+                Projective { 
+                    x: BigInt([11315198235793138814, 4744451806709910489, 6921831025042192557, 1125393823825936625]), 
+                    y: BigInt([13035879815613524256, 18225673961538637854, 12006860967936477969, 1516668567229692859]), 
+                    t: BigInt([13450777528397789701, 12242009376162249168, 12596256366242272750, 3368076418495976469]), 
+                    z: BigInt([10465708325245823445, 13967918689717629445, 14943426723808572731, 621075342718756551]) 
+                }
+                ) 
+            }
         ) 
-    },
-    private_key: BLSPrivKey { 
-        priv_key: SignKey(
-            BigInt(
-                [3505488234151006356, 6655477166151225138, 3291219027844407676, 2153641080015542578]
-            )
-        ) 
-    },
-    stake_value: 1
-}
+    }
diff --git a/crates/examples/infra/mod.rs b/crates/examples/infra/mod.rs
@@ -64,7 +64,7 @@ use std::{collections::BTreeSet, sync::Arc};
 use std::{fs, time::Instant};
 use std::{num::NonZeroUsize, str::FromStr};
 use surf_disco::Url;
-use tracing::{error, info, warn};
+use tracing::{debug, error, info, warn};
 use versioned_binary_serialization::version::StaticVersionType;
 
 #[derive(Debug, Clone)]
@@ -172,6 +172,22 @@ pub fn read_orchestrator_init_config<TYPES: NodeType>() -> (
                 .help("Sets the url of the orchestrator")
                 .required(false),
         )
+        .arg(
+            Arg::new("webserver_url")
+                .short('w')
+                .long("webserver_url")
+                .value_name("URL")
+                .help("Sets the url of the webserver")
+                .required(false),
+        )
+        .arg(
+            Arg::new("da_webserver_url")
+                .short('a')
+                .long("da_webserver_url")
+                .value_name("URL")
+                .help("Sets the url of the da webserver")
+                .required(false),
+        )
         .get_matches();
 
     if let Some(config_file_string) = matches.get_one::<String>("config_file") {
@@ -212,6 +228,20 @@ pub fn read_orchestrator_init_config<TYPES: NodeType>() -> (
     if let Some(orchestrator_url_string) = matches.get_one::<String>("orchestrator_url") {
         orchestrator_url = Url::parse(orchestrator_url_string).unwrap();
     }
+    if let Some(webserver_url_string) = matches.get_one::<String>("webserver_url") {
+        let updated_web_server_config = WebServerConfig {
+            url: Url::parse(webserver_url_string).unwrap(),
+            wait_between_polls: config.web_server_config.unwrap().wait_between_polls,
+        };
+        config.web_server_config = Some(updated_web_server_config);
+    }
+    if let Some(da_webserver_url_string) = matches.get_one::<String>("da_webserver_url") {
+        let updated_da_web_server_config = WebServerConfig {
+            url: Url::parse(da_webserver_url_string).unwrap(),
+            wait_between_polls: config.da_web_server_config.unwrap().wait_between_polls,
+        };
+        config.da_web_server_config = Some(updated_da_web_server_config);
+    }
 
     (config, orchestrator_url)
 }
@@ -260,7 +290,7 @@ pub async fn run_orchestrator<
 >(
     OrchestratorArgs { url, config }: OrchestratorArgs<TYPES>,
 ) {
-    error!("Starting orchestrator",);
+    println!("Starting orchestrator",);
     let _result = hotshot_orchestrator::run_orchestrator::<
         TYPES::SignatureKey,
         TYPES::ElectionConfigType,
@@ -540,10 +570,10 @@ pub trait RunDA<
         let mut total_latency = 0;
         let mut num_latency = 0;
 
-        error!("Sleeping for {start_delay_seconds} seconds before starting hotshot!");
+        debug!("Sleeping for {start_delay_seconds} seconds before starting hotshot!");
         async_sleep(Duration::from_secs(start_delay_seconds)).await;
 
-        error!("Starting HotShot example!");
+        debug!("Starting HotShot example!");
         let start = Instant::now();
 
         let mut event_stream = context.get_event_stream();
@@ -638,7 +668,7 @@ pub trait RunDA<
                             failed_num_views += 1;
                             warn!("Timed out in view {:?}", view_number);
                         }
-                        _ => {}
+                        _ => {} // mostly DA proposal
                     }
                 }
             }
@@ -647,19 +677,21 @@ pub trait RunDA<
         let consensus = consensus_lock.read().await;
         let total_num_views = usize::try_from(consensus.locked_view.get_u64()).unwrap();
         // When posting to the orchestrator, note that the total number of views also include un-finalized views.
-        error!("Failed views: {failed_num_views}, Total views: {total_num_views}, num_successful_commits: {num_successful_commits}");
+        println!("[{node_index}]: Total views: {total_num_views}, Failed views: {failed_num_views}, num_successful_commits: {num_successful_commits}");
         // +2 is for uncommitted views
         assert!(total_num_views <= (failed_num_views + num_successful_commits + 2));
         // Output run results
         let total_time_elapsed = start.elapsed(); // in seconds
-        error!("[{node_index}]: {rounds} rounds completed in {total_time_elapsed:?} - Total transactions sent: {total_transactions_sent} - Total transactions committed: {total_transactions_committed} - Total commitments: {num_successful_commits}");
+        println!("[{node_index}]: {rounds} rounds completed in {total_time_elapsed:?} - Total transactions sent: {total_transactions_sent} - Total transactions committed: {total_transactions_committed} - Total commitments: {num_successful_commits}");
         if total_transactions_committed != 0 {
             // extra 8 bytes for timestamp
             let throughput_bytes_per_sec = total_transactions_committed
                 * (transaction_size_in_bytes + 8)
                 / total_time_elapsed.as_secs();
+            let avg_latency_in_sec = total_latency / num_latency;
+            println!("[{node_index}]: throughput: {throughput_bytes_per_sec} bytes/sec, avg_latency: {avg_latency_in_sec} sec.");
             BenchResults {
-                avg_latency_in_sec: total_latency / num_latency,
+                avg_latency_in_sec,
                 num_latency,
                 minimum_latency_in_sec: minimum_latency,
                 maximum_latency_in_sec: maximum_latency,
@@ -1056,7 +1088,7 @@ pub async fn main_entry_point<
     setup_logging();
     setup_backtrace();
 
-    error!("Starting validator");
+    debug!("Starting validator");
 
     // see what our public identity will be
     let public_ip = match args.public_ip {
@@ -1128,13 +1160,13 @@ pub async fn main_entry_point<
     }
 
     if let NetworkConfigSource::Orchestrator = source {
-        error!("Waiting for the start command from orchestrator");
+        debug!("Waiting for the start command from orchestrator");
         orchestrator_client
             .wait_for_all_nodes_ready(run_config.clone().node_index)
             .await;
     }
 
-    error!("Starting HotShot");
+    println!("Starting HotShot");
     let bench_results = run
         .run_hotshot(
             hotshot,

diff --git a/crates/orchestrator/run-config.toml b/crates/orchestrator/run-config.toml
@@ -1,4 +1,4 @@
-rounds = 100
+rounds = 10
 transactions_per_round = 10
 transaction_size = 1000
 node_index = 0
@@ -77,25 +77,25 @@ url = "http://localhost:9001"
 
 [web_server_config.wait_between_polls]
 secs = 0
-nanos = 100000000 # 10 ms
+nanos = 10_000_000
 
 [da_web_server_config.wait_between_polls]
 secs = 0
-nanos = 100000000 # 10 ms
+nanos = 10_000_000
 
 
+[config.view_sync_timeout]
+secs = 2
+nanos = 0
+
 [config.data_request_delay]
 secs = 0
-nanos = 2000000000 # 200 ms
+nanos = 200_000_000
 
 [config.propose_min_round_time]
 secs = 0
 nanos = 0
 
-[config.view_sync_timeout]
-secs = 2
-nanos = 0
-
 # TODO (Keyao) Clean up configuration parameters.
 # <https://github.com/EspressoSystems/HotShot/issues/1823>
 [config.propose_max_round_time]

diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs
@@ -11,6 +11,7 @@ use hotshot_types::{
     traits::{election::ElectionConfig, signature_key::SignatureKey},
     PeerConfig,
 };
+use std::fs::OpenOptions;
 use std::{
     collections::HashSet,
     io,
@@ -125,8 +126,14 @@ impl<KEY: SignatureKey + 'static, ELECTION: ElectionConfig + 'static>
             total_num_views: self.bench_results.total_num_views,
             failed_num_views: self.bench_results.failed_num_views,
         };
+        // Open the CSV file in append mode
+        let results_csv_file = OpenOptions::new()
+            .create(true)
+            .append(true) // Open in append mode
+            .open("scripts/benchmarks_results/results.csv")
+            .unwrap();
         // Open a file for writing
-        let mut wtr = Writer::from_path("scripts/benchmarks_results/results.csv").unwrap();
+        let mut wtr = Writer::from_writer(results_csv_file);
         let _ = wtr.serialize(output_csv);
         let _ = wtr.flush();
         println!("Results successfully saved in scripts/benchmarks_results/results.csv");
@@ -266,7 +273,7 @@ where
         }
         self.pub_posted.insert(node_index);
 
-        // The guess is the first extra 8 bytes are from orchestrator serialization
+        // The guess is the first extra 12 bytes are from orchestrator serialization
         pubkey.drain(..12);
         let register_pub_key_with_stake = PeerConfig::<KEY>::from_bytes(pubkey).unwrap();
         self.config.config.known_nodes_with_stake[node_index as usize] =

diff --git a/docker/orchestrator-webserver.Dockerfile b/docker/orchestrator-webserver.Dockerfile
@@ -15,5 +15,4 @@ ENV RUST_LOG="warn"
 # log format. JSON no ansi
 ENV RUST_LOG_FORMAT="json"
 
-ENTRYPOINT ["tini", "--"]
-CMD ["orchestrator-webserver"]
+ENTRYPOINT ["orchestrator-webserver"]
diff --git a/docker/validator-webserver-local.Dockerfile b/docker/validator-webserver-local.Dockerfile
@@ -0,0 +1,17 @@
+FROM ubuntu:jammy
+
+RUN apt-get update \
+    &&  apt-get install -y curl libcurl4 wait-for-it tini \
+    &&  rm -rf /var/lib/apt/lists/*
+
+ARG ASYNC_EXECUTOR=async-std
+
+COPY ./target/${ASYNC_EXECUTOR}/debug/examples/validator-webserver /usr/local/bin/validator-webserver
+
+# logging
+ENV RUST_LOG="warn"
+
+# log format. JSON no ansi
+ENV RUST_LOG_FORMAT="json"
+
+ENTRYPOINT ["validator-webserver"]
diff --git a/docker/validator-webserver.Dockerfile b/docker/validator-webserver.Dockerfile
@@ -15,5 +15,4 @@ ENV RUST_LOG="warn"
 # log format. JSON no ansi
 ENV RUST_LOG_FORMAT="json"
 
-ENTRYPOINT ["tini", "--"]
-CMD ["validator-webserver"]
+ENTRYPOINT ["validator-webserver"]
diff --git a/docker/webserver.Dockerfile b/docker/webserver.Dockerfile
@@ -15,5 +15,4 @@ ENV RUST_LOG="warn"
 # log format. JSON no ansi
 ENV RUST_LOG_FORMAT="json"
 
-ENTRYPOINT ["tini", "--"]
-CMD ["webserver"]
+ENTRYPOINT ["webserver"]
diff --git a/scripts/.gitignore b/scripts/.gitignore
@@ -1,2 +1,3 @@
 __pycache__
-*.output
+*.output
+benchmarks_results/results.csv
diff --git a/scripts/aws_ecs_benchmarks.sh b/scripts/aws_ecs_benchmarks.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+
+source "$HOME/.cargo/env"
+
+# assign local ip
+ip=`curl http://169.254.169.254/latest/meta-data/local-ipv4`
+webserver_url=http://"$ip":9000
+da_webserver_url=http://"$ip":9001
+orchestrator_url=http://"$ip":4444
+
+# build
+just async_std build
+sleep 30s
+
+# docker build and push
+docker build . -f ./docker/validator-webserver-local.Dockerfile -t ghcr.io/espressosystems/hotshot/validator-webserver:main-async-std
+docker push ghcr.io/espressosystems/hotshot/validator-webserver:main-async-std
+
+# ecs deploy
+ecs deploy --region us-east-2 hotshot hotshot_centralized -i centralized ghcr.io/espressosystems/hotshot/validator-webserver:main-async-std
+ecs deploy --region us-east-2 hotshot hotshot_centralized -c centralized ${orchestrator_url}
+
+for total_nodes in 10 50 100
+do
+    for da_committee_size in 10 50 100
+    do
+        if [ $da_committee_size -le $total_nodes ]
+        then
+            for transactions_per_round in 1 10 50 100
+            do
+                for transaction_size in 512 4096 #1000000 20000000
+                do
+                    rounds=100
+                    # start webserver
+                    just async_std example webserver -- http://0.0.0.0:9000 &
+                    just async_std example webserver -- http://0.0.0.0:9001 &
+                    sleep 1m
+
+                    # start orchestrator
+                    just async_std example orchestrator-webserver -- --config_file ./crates/orchestrator/run-config.toml \
+                                                                    --orchestrator_url http://0.0.0.0:4444 \
+                                                                    --webserver_url ${webserver_url} \
+                                                                    --da_webserver_url ${da_webserver_url} \
+                                                                    --total_nodes ${total_nodes} \
+                                                                    --da_committee_size ${da_committee_size} \
+                                                                    --transactions_per_round ${transactions_per_round} \
+                                                                    --transaction_size ${transaction_size} \
+                                                                    --rounds ${rounds} \
+                                                                    --commit_sha test &
+                    sleep 30
+
+                    # start validators
+                    ecs scale --region us-east-2 hotshot hotshot_centralized ${total_nodes} --timeout -1
+                    sleep $(($rounds + $total_nodes))
+
+                    # kill them
+                    ecs scale --region us-east-2 hotshot hotshot_centralized 0 --timeout -1
+                    sleep 1m
+                    for pid in $(ps -ef | grep "orchestrator" | awk '{print $2}'); do kill -9 $pid; done
+                    for pid in $(ps -ef | grep "webserver" | awk '{print $2}'); do kill -9 $pid; done
+                done
+            done
+        fi
+    done
+done
diff --git a/scripts/benchmarks_results/results.csv b/scripts/benchmarks_results/results.csv