Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/clap #179

Merged
merged 66 commits into from
Nov 20, 2024
Merged
Show file tree
Hide file tree
Changes from 42 commits
Commits
Show all changes
66 commits
Select commit Hold shift + click to select a range
0dffedf
update: basic cli setup
heemankv Nov 5, 2024
d963ab2
update: run_cmd for settlement client
heemankv Nov 6, 2024
9cb3702
update: working instrumentation run_cmd
heemankv Nov 6, 2024
9be43e0
update: run_cmd for storage
heemankv Nov 6, 2024
a743141
update: run_cmd for server, aws config
heemankv Nov 6, 2024
1aafaa5
update: run_cmd for da and alert
heemankv Nov 6, 2024
bc44afd
update: run_cmd for sharp and snos
heemankv Nov 6, 2024
2c1d5cb
update: run_cmd for madara rpc url
heemankv Nov 6, 2024
00b2e31
update: run_cmd adaptation for: testcases
heemankv Nov 6, 2024
8685ce2
update: rework e2e
heemankv Nov 6, 2024
153bc93
update: lint cleanu
heemankv Nov 6, 2024
715c9b6
update: working mutual exclusion
heemankv Nov 6, 2024
bb99760
Merge branch 'main' into feature/clap
heemankv Nov 6, 2024
fa1efc4
update: fixed telemetry tests
heemankv Nov 6, 2024
1cbf00c
update: tests cleanup
heemankv Nov 7, 2024
0c2b489
update: coverage build fix
heemankv Nov 7, 2024
4a8de78
update: lint fix
heemankv Nov 7, 2024
5201fad
update: lint e2e cargo fix
heemankv Nov 7, 2024
7e7ab9b
lint fix
heemankv Nov 7, 2024
e4dacd9
update: reworked cli params for tests
heemankv Nov 7, 2024
87b571c
update: aws args renamed
heemankv Nov 7, 2024
c92a27e
update: moved cli from utils to orchestrator
heemankv Nov 7, 2024
ee8195b
update: removed envsettingsprovider
heemankv Nov 7, 2024
da11a0b
update: removed serde added strum for queuetype
heemankv Nov 7, 2024
3fc6d65
update: added orchestrator config
heemankv Nov 7, 2024
7529986
update: env vars renamed
heemankv Nov 7, 2024
5b1cbc2
update: env fixes for tests
heemankv Nov 7, 2024
103e462
update: fixed aws related issue
heemankv Nov 7, 2024
44bbe55
update: fixing env
heemankv Nov 8, 2024
4cf1786
update: coverage fixes
heemankv Nov 8, 2024
815437d
update: coverage fixes
heemankv Nov 8, 2024
fd6e88e
update: fixes for test_Settle and e2e
heemankv Nov 8, 2024
e1d09fe
fixed: coverage tests
heemankv Nov 8, 2024
c1046cc
update: test_settle fixed
heemankv Nov 8, 2024
6525c63
update: allow e2e and coverage to work together based off env
heemankv Nov 8, 2024
b8a7e4a
update: fixed db issue
heemankv Nov 8, 2024
82738c0
update: lint fixes
heemankv Nov 8, 2024
cde59b3
update: lint fix
heemankv Nov 8, 2024
98a5bc0
update: clean up
heemankv Nov 8, 2024
7c90490
update: unwraps to expect and regrouping
heemankv Nov 9, 2024
89d4bdd
Merge branch 'main' into feature/clap
heemankv Nov 9, 2024
306ddad
update: env correctiong
heemankv Nov 9, 2024
498e5c3
update: removed config
heemankv Nov 9, 2024
b477b29
update: removed config.rs
heemankv Nov 9, 2024
83114fe
update: renamed XYZParams to XYZValidatedArgs
heemankv Nov 9, 2024
4cbb3a1
update: lint fixes
heemankv Nov 9, 2024
ef3127c
update: lint fixes
heemankv Nov 9, 2024
abf8df2
Merge branch 'main' into feature/clap
heemankv Nov 12, 2024
ccabb49
update: fixed e2e testcase
heemankv Nov 12, 2024
58dd6bb
update: added tests for validate fns
heemankv Nov 12, 2024
602f1a2
update: testcase fix
heemankv Nov 12, 2024
fa75b56
update: added command run / setup and e2e with setup_config
heemankv Nov 12, 2024
55c4454
update: setup and run fns
heemankv Nov 12, 2024
2c980a0
update: systematic rework
heemankv Nov 13, 2024
a79aaeb
update: ProviderValidatedArgs
heemankv Nov 13, 2024
1a60dc2
update: reworked commands run and setup
heemankv Nov 13, 2024
3c3f83d
update: optimised fixes
heemankv Nov 13, 2024
414d624
update: added e2e corrections
heemankv Nov 13, 2024
02618d1
update: e2e fixed
heemankv Nov 13, 2024
ad0c754
update: fixed coverage
heemankv Nov 13, 2024
ec4ad85
update: MADARA_ORCHESTRATOR_EVENT_BRIDGE_CRON_TIME to 60 seconds
heemankv Nov 13, 2024
982aed9
update: testcase fixes
heemankv Nov 14, 2024
a7e5947
resolved: comments
Nov 19, 2024
712b6ed
resolve: removed bucket name from require_all
Nov 19, 2024
81491c3
chore: comments added for the decisions taken
Nov 20, 2024
58b6dda
chore: setup cloud todo removed, plan dropped to move it to main.rs
Nov 20, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 22 additions & 22 deletions .env.example
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
##### ORCHESTRATOR #####

HOST=
PORT=
MAX_BLOCK_TO_PROCESS=
MADARA_ORCHESTRATOR_HOST=
MADARA_ORCHESTRATOR_PORT=
MADARA_ORCHESTRATOR_MAX_BLOCK_NO_TO_PROCESS=

##### AWS CONFIG #####

Expand All @@ -17,7 +17,7 @@ AWS_DEFAULT_REGION="localhost"
##### STORAGE #####

DATA_STORAGE=
apoorvsadana marked this conversation as resolved.
Show resolved Hide resolved
AWS_S3_BUCKET_NAME=
MADARA_ORCHESTRATOR_AWS_S3_BUCKET_NAME=

##### QUEUE #####

Expand All @@ -30,39 +30,39 @@ SQS_WORKER_TRIGGER_QUEUE_URL=
##### SNS #####

ALERTS="sns"
AWS_SNS_ARN="arn:aws:sns:us-east-1:000000000000:madara-orchestrator-arn"
MADARA_ORCHESTRATOR_AWS_SNS_ARN="arn:aws:sns:us-east-1:000000000000:madara-orchestrator-arn"

##### DATABASE #####

DATABASE=
MONGODB_CONNECTION_STRING=
DATABASE_NAME=
MADARA_ORCHESTRATOR_MONGODB_CONNECTION_URL=
MADARA_ORCHESTRATOR_DATABASE_NAME=

##### PROVER #####

PROVER_SERVICE=
SHARP_CUSTOMER_ID=
SHARP_URL=
SHARP_USER_CRT=
SHARP_USER_KEY=
SHARP_SERVER_CRT=
SHARP_PROOF_LAYOUT=
MADARA_ORCHESTRATOR_SHARP_CUSTOMER_ID=
MADARA_ORCHESTRATOR_SHARP_URL=
MADARA_ORCHESTRATOR_SHARP_USER_CRT=
MADARA_ORCHESTRATOR_SHARP_USER_KEY=
MADARA_ORCHESTRATOR_SHARP_SERVER_CRT=
MADARA_ORCHESTRATOR_SHARP_PROOF_LAYOUT=

##### ON CHAIN CONFIG #####

DA_LAYER=
SETTLEMENT_LAYER=
SETTLEMENT_RPC_URL=
MADARA_RPC_URL=
GPS_VERIFIER_CONTRACT_ADDRESS=
MADARA_ORCHESTRATOR_ETHEREUM_SETTLEMENT_RPC_URL=
MADARA_ORCHESTRATOR_MADARA_RPC_URL=
MADARA_ORCHESTRATOR_GPS_VERIFIER_CONTRACT_ADDRESS=
PRIVATE_KEY=
ETHEREUM_PRIVATE_KEY=
L1_CORE_CONTRACT_ADDRESS=
MADARA_ORCHESTRATOR_ETHEREUM_PRIVATE_KEY=
MADARA_ORCHESTRATOR_L1_CORE_CONTRACT_ADDRESS=

##### STARKNET SETTLEMENT (L3s) #####
STARKNET_PRIVATE_KEY=
STARKNET_ACCOUNT_ADDRESS=
MADARA_ORCHESTRATOR_STARKNET_PRIVATE_KEY=
MADARA_ORCHESTRATOR_STARKNET_ACCOUNT_ADDRESS=

##### Instrumentation #####
OTEL_SERVICE_NAME=
OTEL_COLLECTOR_ENDPOINT=
MADARA_ORCHESTRATOR_OTEL_SERVICE_NAME=
MADARA_ORCHESTRATOR_OTEL_COLLECTOR_ENDPOINT=
140 changes: 75 additions & 65 deletions .env.test

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions .github/workflows/coverage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ jobs:
mv target/debug/madara ../madara-binary
cd ..
echo -e "
MADARA_BINARY_PATH=\"$(pwd)/madara-binary\"" >> .env.test
MADARA_ORCHESTRATOR_MADARA_BINARY_PATH=\"$(pwd)/madara-binary\"" >> .env.test
cat .env.test

- name: Getting neccesary files for testing
Expand All @@ -128,8 +128,8 @@ jobs:

- name: Run llvm-cov tests
env:
SETTLEMENT_RPC_URL: ${{ secrets.ETHEREUM_SEPOLIA_BLAST_RPC }}
RPC_FOR_SNOS: ${{ secrets.RPC_FOR_SNOS }}
MADARA_ORCHESTRATOR_ETHEREUM_SETTLEMENT_RPC_URL: ${{ secrets.ETHEREUM_SEPOLIA_BLAST_RPC }}
MADARA_ORCHESTRATOR_RPC_FOR_SNOS: ${{ secrets.RPC_FOR_SNOS }}
# the self hosted runner has a different region so we override it here
AWS_REGION: us-east-1
run: RUST_LOG=debug RUST_BACKTRACE=1 cargo llvm-cov nextest --release --features testing --lcov --output-path lcov.info --test-threads=1 --workspace --exclude=e2e-tests --no-fail-fast
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/e2e-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@ jobs:

- name: Run e2e test
env:
SETTLEMENT_RPC_URL: ${{ secrets.ETHEREUM_SEPOLIA_BLAST_RPC }}
RPC_FOR_SNOS: ${{ secrets.RPC_FOR_SNOS }}
MADARA_ORCHESTRATOR_ETHEREUM_SETTLEMENT_RPC_URL: ${{ secrets.ETHEREUM_SEPOLIA_BLAST_RPC }}
MADARA_ORCHESTRATOR_RPC_FOR_SNOS: ${{ secrets.RPC_FOR_SNOS }}
# the self hosted runner has a different region so we override it here
AWS_REGION: us-east-1
run: |
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).

## Added

- Added cli args support for all the services
- panic handling in process job
- upgrade ETH L1 bridge for withdrawals to work
- added makefile and submodules
Expand Down
6 changes: 6 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ lazy_static = "1.4.0"
stark_evm_adapter = "0.1.1"
hex = "0.4"
itertools = "0.13.0"
clap = { version = "4.4", features = ["derive", "env"] }
mockall = "0.13.0"
testcontainers = "0.18.0"
once_cell = "1.8"
Expand Down
10 changes: 7 additions & 3 deletions crates/da-clients/ethereum/src/config.rs
heemankv marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ use alloy::providers::ProviderBuilder;
use alloy::rpc::client::RpcClient;
use serde::{Deserialize, Serialize};
use url::Url;
use utils::settings::Settings;

use crate::EthereumDaClient;

Expand All @@ -15,8 +14,8 @@ pub struct EthereumDaConfig {
}

impl EthereumDaConfig {
pub fn new_with_settings(settings: &impl Settings) -> color_eyre::Result<Self> {
Ok(Self { rpc_url: settings.get_settings_or_panic("SETTLEMENT_RPC_URL") })
pub fn new_with_params(ethereum_da_params: &EthereumDaParams) -> color_eyre::Result<Self> {
Ok(Self { rpc_url: ethereum_da_params.ethereum_da_rpc_url.to_string() })
}

pub async fn build_client(&self) -> EthereumDaClient {
Expand All @@ -27,3 +26,8 @@ impl EthereumDaConfig {
EthereumDaClient { provider }
}
}

#[derive(Debug, Clone)]
pub struct EthereumDaParams {
pub ethereum_da_rpc_url: Url,
}
22 changes: 2 additions & 20 deletions crates/da-clients/ethereum/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,29 +1,22 @@
#![allow(missing_docs)]
#![allow(clippy::missing_docs_in_private_items)]

use std::str::FromStr;

use alloy::network::Ethereum;
use alloy::providers::{ProviderBuilder, RootProvider};
use alloy::rpc::client::RpcClient;
use alloy::providers::RootProvider;
use alloy::transports::http::Http;
use async_trait::async_trait;
use color_eyre::Result;
use da_client_interface::{DaClient, DaVerificationStatus};
use mockall::automock;
use mockall::predicate::*;
use reqwest::Client;
use url::Url;
use utils::settings::Settings;

use crate::config::EthereumDaConfig;

pub const DA_SETTINGS_NAME: &str = "ethereum";
Mohiiit marked this conversation as resolved.
Show resolved Hide resolved

pub mod config;
pub struct EthereumDaClient {
#[allow(dead_code)]
provider: RootProvider<Ethereum, Http<Client>>,
pub provider: RootProvider<Ethereum, Http<Client>>,
heemankv marked this conversation as resolved.
Show resolved Hide resolved
}

#[automock]
Expand All @@ -47,14 +40,3 @@ impl DaClient for EthereumDaClient {
131072
}
}

impl EthereumDaClient {
pub fn new_with_settings(settings: &impl Settings) -> Self {
let config = EthereumDaConfig::new_with_settings(settings)
.expect("Not able to create EthereumDaClient from given settings.");
let client =
RpcClient::new_http(Url::from_str(config.rpc_url.as_str()).expect("Failed to parse SETTLEMENT_RPC_URL"));
let provider = ProviderBuilder::<_, Ethereum>::new().on_client(client);
EthereumDaClient { provider }
}
}
4 changes: 3 additions & 1 deletion crates/orchestrator/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ bytes.workspace = true
c-kzg = { workspace = true }
cairo-vm = { workspace = true }
chrono = { workspace = true }
clap.workspace = true
color-eyre = { workspace = true }
da-client-interface = { workspace = true }
dotenvy = { workspace = true }
Expand Down Expand Up @@ -61,7 +62,8 @@ starknet = { workspace = true }
starknet-core = "0.9.0"
starknet-os = { workspace = true }
starknet-settlement-client = { workspace = true }
strum_macros = "0.26.4"
strum = "0.26.0"
strum_macros = "0.26.0"
tempfile = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] }
Expand Down
19 changes: 0 additions & 19 deletions crates/orchestrator/src/alerts/aws_sns/config.rs

This file was deleted.

23 changes: 14 additions & 9 deletions crates/orchestrator/src/alerts/aws_sns/mod.rs
Original file line number Diff line number Diff line change
@@ -1,28 +1,33 @@
mod config;

use std::sync::Arc;

use async_trait::async_trait;
use aws_sdk_sns::Client;
use utils::settings::Settings;

use crate::alerts::aws_sns::config::AWSSNSConfig;
use crate::alerts::Alerts;
use crate::config::ProviderConfig;

pub const AWS_SNS_SETTINGS_NAME: &str = "sns";
#[derive(Debug, Clone)]
pub struct AWSSNSParams {
// TODO: convert to ARN type, and validate it
// NOTE: aws is using str to represent ARN : https://docs.aws.amazon.com/sdk-for-rust/latest/dg/rust_sns_code_examples.html
heemankv marked this conversation as resolved.
Show resolved Hide resolved
pub sns_arn: String,
}

impl AWSSNSParams {
pub fn get_topic_name(&self) -> String {
self.sns_arn.split(":").last().unwrap().to_string()
}
}

pub struct AWSSNS {
client: Client,
topic_arn: String,
}

impl AWSSNS {
pub async fn new_with_settings(settings: &impl Settings, provider_config: Arc<ProviderConfig>) -> Self {
let sns_config =
AWSSNSConfig::new_with_settings(settings).expect("Not able to get Aws sns config from provided settings");
pub async fn new_with_params(aws_sns_params: &AWSSNSParams, provider_config: Arc<ProviderConfig>) -> Self {
let config = provider_config.get_aws_client_or_panic();
Self { client: Client::new(config), topic_arn: sns_config.sns_arn }
Self { client: Client::new(config), topic_arn: aws_sns_params.sns_arn.clone() }
}
}

Expand Down
14 changes: 14 additions & 0 deletions crates/orchestrator/src/cli/alert/aws_sns.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
use clap::Args;

/// Parameters used to config AWS SNS.
#[derive(Debug, Clone, Args)]
#[group()]
apoorvsadana marked this conversation as resolved.
Show resolved Hide resolved
pub struct AWSSNSCliArgs {
/// Use the AWS SNS client
#[arg(long)]
pub aws_sns: bool,
apoorvsadana marked this conversation as resolved.
Show resolved Hide resolved

/// The name of the S3 bucket.
Mohiiit marked this conversation as resolved.
Show resolved Hide resolved
#[arg(env = "MADARA_ORCHESTRATOR_AWS_SNS_ARN", long, default_value = Some("madara-orchestrator-arn"))]
pub sns_arn: Option<String>,
}
8 changes: 8 additions & 0 deletions crates/orchestrator/src/cli/alert/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
use crate::alerts::aws_sns::AWSSNSParams;

pub mod aws_sns;

#[derive(Clone, Debug)]
pub enum AlertParams {
heemankv marked this conversation as resolved.
Show resolved Hide resolved
AWSSNS(AWSSNSParams),
}
28 changes: 28 additions & 0 deletions crates/orchestrator/src/cli/aws_config.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
use clap::Args;
use serde::Serialize;
use url::Url;

/// Parameters used to config AWS.
#[derive(Debug, Clone, Args, Serialize)]
#[group(requires_all = ["aws_access_key_id", "aws_secret_access_key", "aws_region"])]
pub struct AWSConfigCliArgs {
heemankv marked this conversation as resolved.
Show resolved Hide resolved
/// The access key ID.
#[arg(env = "AWS_ACCESS_KEY_ID", long)]
pub aws_access_key_id: String,

/// The secret access key.
#[arg(env = "AWS_SECRET_ACCESS_KEY", long)]
pub aws_secret_access_key: String,

/// The region.
#[arg(env = "AWS_REGION", long)]
pub aws_region: String,
}

pub struct AWSConfigParams {
pub aws_access_key_id: String,
pub aws_secret_access_key: String,
pub aws_region: String,
pub aws_endpoint_url: Url,
pub aws_default_region: String,
}
15 changes: 15 additions & 0 deletions crates/orchestrator/src/cli/da/ethereum.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
use clap::Args;
use url::Url;

/// Parameters used to config Ethereum.
#[derive(Debug, Clone, Args)]
#[group(requires_all = ["ethereum_da_rpc_url"])]
pub struct EthereumDaCliArgs {
/// Use the Ethereum DA layer.
#[arg(long)]
pub da_on_ethereum: bool,

/// The RPC URL of the Ethereum node.
#[arg(env = "MADARA_ORCHESTRATOR_ETHEREUM_DA_RPC_URL", long)]
pub ethereum_da_rpc_url: Option<Url>,
}
Loading
Loading