diff --git a/.env.example b/.env.example index 31e75396..7c69f686 100644 --- a/.env.example +++ b/.env.example @@ -1,77 +1,88 @@ -##### ORCHESTRATOR ##### - -MADARA_ORCHESTRATOR_HOST= -MADARA_ORCHESTRATOR_PORT= -MADARA_ORCHESTRATOR_MAX_BLOCK_NO_TO_PROCESS= - -##### AWS CONFIG ##### - -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= -AWS_REGION= - -##### Omniqueue ##### - -AWS_DEFAULT_REGION="localhost" - -##### STORAGE ##### - -DATA_STORAGE= -MADARA_ORCHESTRATOR_AWS_S3_BUCKET_NAME= - -##### QUEUE ##### - -QUEUE_PROVIDER= -SQS_JOB_PROCESSING_QUEUE_URL= -SQS_JOB_VERIFICATION_QUEUE_URL= -SQS_JOB_HANDLE_FAILURE_QUEUE_URL= -SQS_WORKER_TRIGGER_QUEUE_URL= - -##### SNS ##### - -ALERTS="sns" -MADARA_ORCHESTRATOR_AWS_SNS_ARN="arn:aws:sns:us-east-1:000000000000:madara-orchestrator-arn" - -##### DATABASE ##### - -DATABASE= -MADARA_ORCHESTRATOR_MONGODB_CONNECTION_URL= -MADARA_ORCHESTRATOR_DATABASE_NAME= - -##### PROVER ##### - -PROVER_SERVICE= - -## if using sharp -MADARA_ORCHESTRATOR_SHARP_CUSTOMER_ID= -MADARA_ORCHESTRATOR_SHARP_URL= -MADARA_ORCHESTRATOR_SHARP_USER_CRT= -MADARA_ORCHESTRATOR_SHARP_USER_KEY= -MADARA_ORCHESTRATOR_SHARP_SERVER_CRT= -MADARA_ORCHESTRATOR_SHARP_PROOF_LAYOUT= - - -## if using atlantic -MADARA_ORCHESTRATOR_ATLANTIC_API_KEY= -MADARA_ORCHESTRATOR_ATLANTIC_URL= -MADARA_ORCHESTRATOR_MOCK_FACT_HASH= -MADARA_ORCHESTRATOR_PROVER_TYPE= - -##### ON CHAIN CONFIG ##### - -DA_LAYER= -SETTLEMENT_LAYER= -MADARA_ORCHESTRATOR_ETHEREUM_SETTLEMENT_RPC_URL= -MADARA_ORCHESTRATOR_MADARA_RPC_URL= -MADARA_ORCHESTRATOR_GPS_VERIFIER_CONTRACT_ADDRESS= -PRIVATE_KEY= -MADARA_ORCHESTRATOR_ETHEREUM_PRIVATE_KEY= -MADARA_ORCHESTRATOR_L1_CORE_CONTRACT_ADDRESS= - -##### STARKNET SETTLEMENT (L3s) ##### -MADARA_ORCHESTRATOR_STARKNET_PRIVATE_KEY= -MADARA_ORCHESTRATOR_STARKNET_ACCOUNT_ADDRESS= - -##### Instrumentation ##### -MADARA_ORCHESTRATOR_OTEL_SERVICE_NAME= -MADARA_ORCHESTRATOR_OTEL_COLLECTOR_ENDPOINT= +#### AWS CONFIG #### +AWS_ACCESS_KEY_ID= # Your AWS access key ID +AWS_SECRET_ACCESS_KEY= # Your AWS secret access key +AWS_REGION= # AWS region +# For AWS SDK +AWS_ENDPOINT_URL= # AWS endpoint URL +# For Omniqueue +AWS_DEFAULT_REGION= # AWS default region + +# For EventBridge +MADARA_ORCHESTRATOR_EVENT_BRIDGE_TRIGGER_RULE_NAME= # EventBridge rule name +MADARA_ORCHESTRATOR_EVENT_BRIDGE_TRIGGER_ROLE_NAME= # EventBridge role name +MADARA_ORCHESTRATOR_EVENT_BRIDGE_TRIGGER_POLICY_NAME= # EventBridge policy name + +#### ALERTS #### +MADARA_ORCHESTRATOR_AWS_SNS_ARN= # SNS ARN for alerts + +#### DATA AVAILABILITY #### +## ETHEREUM ## +MADARA_ORCHESTRATOR_ETHEREUM_DA_RPC_URL= # Ethereum RPC URL for data availability + +#### DATABASE #### +## MONGODB ## +MADARA_ORCHESTRATOR_MONGODB_CONNECTION_URL= # MongoDB connection URL +MADARA_ORCHESTRATOR_DATABASE_NAME= # MongoDB database name + +#### PROVER #### +## SHARP ## +MADARA_ORCHESTRATOR_SHARP_CUSTOMER_ID= # SHARP customer ID +MADARA_ORCHESTRATOR_SHARP_URL= # SHARP service URL +MADARA_ORCHESTRATOR_SHARP_USER_CRT= # SHARP user certificate +MADARA_ORCHESTRATOR_SHARP_USER_KEY= # SHARP user private key +MADARA_ORCHESTRATOR_SHARP_SERVER_CRT= # SHARP server certificate +MADARA_ORCHESTRATOR_SHARP_RPC_NODE_URL= # SHARP RPC node URL +MADARA_ORCHESTRATOR_SHARP_PROOF_LAYOUT= # SHARP proof layout +MADARA_ORCHESTRATOR_GPS_VERIFIER_CONTRACT_ADDRESS= # GPS verifier contract address + +## ATLANTIC ## +MADARA_ORCHESTRATOR_ATLANTIC_API_KEY= # Atlantic API key +MADARA_ORCHESTRATOR_ATLANTIC_SERVICE_URL= # Atlantic service URL +MADARA_ORCHESTRATOR_ATLANTIC_MOCK_FACT_HASH= # Whether to use mock fact registry (true/false) +MADARA_ORCHESTRATOR_ATLANTIC_PROVER_TYPE= # Prover type (herodotus/starkware) +MADARA_ORCHESTRATOR_ATLANTIC_SETTLEMENT_LAYER= # Settlement layer (ethereum/starknet) +MADARA_ORCHESTRATOR_ATLANTIC_VERIFIER_CONTRACT_ADDRESS= # Atlantic verifier contract address +MADARA_ORCHESTRATOR_ATLANTIC_RPC_NODE_URL= # Atlantic RPC node URL + +#### QUEUE #### +## AWS SQS ## +MADARA_ORCHESTRATOR_SQS_PREFIX= # SQS queue prefix +MADARA_ORCHESTRATOR_SQS_SUFFIX= # SQS queue suffix +MADARA_ORCHESTRATOR_EVENT_BRIDGE_TARGET_QUEUE_NAME= # EventBridge target queue name +MADARA_ORCHESTRATOR_SQS_BASE_QUEUE_URL= # SQS base queue URL + +#### SETTLEMENT #### +## ETHEREUM ## +MADARA_ORCHESTRATOR_ETHEREUM_SETTLEMENT_RPC_URL= # Ethereum settlement RPC URL +MADARA_ORCHESTRATOR_ETHEREUM_PRIVATE_KEY= # Ethereum private key +MADARA_ORCHESTRATOR_L1_CORE_CONTRACT_ADDRESS= # L1 core contract address +MADARA_ORCHESTRATOR_STARKNET_OPERATOR_ADDRESS= # Starknet operator address + +## STARKNET ## +MADARA_ORCHESTRATOR_STARKNET_SETTLEMENT_RPC_URL= # Starknet settlement RPC URL +MADARA_ORCHESTRATOR_STARKNET_PRIVATE_KEY= # Starknet private key +MADARA_ORCHESTRATOR_STARKNET_ACCOUNT_ADDRESS= # Starknet account address +MADARA_ORCHESTRATOR_STARKNET_CAIRO_CORE_CONTRACT_ADDRESS= # Starknet Cairo core contract address +MADARA_ORCHESTRATOR_STARKNET_FINALITY_RETRY_WAIT_IN_SECS= # Retry wait time in seconds +# MADARA_ORCHESTRATOR_MADARA_BINARY_PATH= # Optional: Madara binary path + +#### STORAGE #### +## AWS S3 ## +MADARA_ORCHESTRATOR_AWS_S3_BUCKET_NAME= # S3 bucket name + +#### INSTRUMENTATION #### +## OTEL ## +MADARA_ORCHESTRATOR_OTEL_SERVICE_NAME= # OpenTelemetry service name +MADARA_ORCHESTRATOR_OTES_COLLECTOR_ENDPOINT= # OpenTelemetry collector endpoint + +#### SERVER #### +MADARA_ORCHESTRATOR_HOST= # Server host +MADARA_ORCHESTRATOR_PORT= # Server port + +#### SERVICE #### +MADARA_ORCHESTRATOR_MAX_BLOCK_NO_TO_PROCESS= # Maximum block number to process (optional) +MADARA_ORCHESTRATOR_MIN_BLOCK_NO_TO_PROCESS= # Minimum block number to process (optional) +MADARA_ORCHESTRATOR_MADARA_RPC_URL= # Madara RPC URL + +#### SNOS #### +MADARA_ORCHESTRATOR_RPC_FOR_SNOS= # SNOS RPC URL \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 29e14f69..061aad6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## Changed +- refactor: Readme and .env.example - refactor: http_mock version updated - refactor: prover-services renamed to prover-clients - refactor: update json made generic to update any json file diff --git a/README.md b/README.md index 8ed59c5f..f05e12ef 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,282 @@ -# Madara Orchestrator +# Madara Orchestrator ๐ŸŽญ The Madara orchestrator is designed to be an additional service which runs in -parallel to Madara and handles +parallel to Madara and handles various critical jobs that ensure proper block +processing, proof generation, data submission and state transitions. -1. publishing data to the respective DA layer -2. running SNOS and submitting jobs to the prover -3. updating the state on Cairo core contracts +## ๐Ÿ“‹ Overview -The tentative flow of the orchestrator looks like this but this is subject to -change as we learn more about external systems and the constraints involved. +The Madara Orchestrator coordinates and triggers five primary jobs in sequence, +managing their execution through a centralized queue system, alowing +for multiple orchestrator to run together! -![orchestrator_da_sequencer_diagram](./docs/orchestrator_da_sequencer_diagram.png) +1. **SNOS (Starknet OS) Job** ๐Ÿ”„ -## Testing + - Identifies blocks that need processing. + - Triggers SNOS run on identified blocks. + - Tracks SNOS execution status and PIE (Program Independent Execution) generation -- Files needed for tests can be fetched through s3 : +2. **Proving Job** โœ… - ```shell - wget -P ./crates/prover-services/sharp-service/tests/artifacts https://madara-orchestrator-sharp-pie.s3.amazonaws.com/238996-SN.zip - ``` + - Coordinates proof generation by submitting PIE to proving services + - Tracks proof generation progress and completion -- To run all the tests : +3. **Data Submission Job** ๐Ÿ“ค - ```shell - cargo llvm-cov nextest --release --lcov --output-path lcov.info --test-threads=1 - ``` + - Manages state update data preparation for availability layers + - If needed, coordinates blob submission to data availability layers + - Currently integrates with Ethereum (EIP-4844 blob transactions) + - Additional DA layer integrations in development (e.g., Celestia) + +4. **State Transition Job** ๐Ÿ”„ + - Coordinates state transitions with settlement layers + - Manages proof and state update submissions + - Handles integration with Ethereum and Starknet settlement layers + +Each job is managed through a queue-based system where the orchestrator: + +- Determines when and which blocks need processing +- Triggers the appropriate services +- Monitors job progress +- Manages job dependencies and sequencing +- Handles retries and failure cases + +## ๐Ÿ›๏ธ Architecture + +### Job Processing Model + +The orchestrator implements a queue-based architecture where each job type +follows a three-phase execution model: + +1. **Creation**: Jobs are spawned based on block availability +2. **Processing**: Core job logic execution +3. **Verification**: Result validation and confirmation + +### Queue Structure + +The system uses dedicated queues for managing different job phases: + +- Worker Trigger Queue +- SNOS Processing/Verification Queues +- Proving Processing/Verification Queues +- Data Submission Processing/Verification Queues +- State Update Processing/Verification Queues +- Job Failure Handling Queue + +### Workflow + +1. Cron jobs trigger worker tasks via the worker-trigger queue +2. Workers determine block-level job requirements +3. Jobs are created and added to processing queues +4. Processed jobs move to verification queues +5. Verified jobs are marked as complete in the database + +## ๐Ÿ› ๏ธ Technical Requirements + +### Prerequisites + +1. **Madara Node** + + - Required for block processing + - Follow setup instructions at [Madara Documentation](https://github.com/madara-alliance/madara) + +2. **Prover Service** + + - ATLANTIC must be running + +3. **Infrastructure Dependencies** + - MongoDB for job management + - AWS services (or Localstack for local development): + - SQS for queues + - S3 for data storage + - SNS for alerts + - EventBridge for scheduling + +## ๐Ÿš€ Installation & Setup + +### Setup Mode + +Setup mode configures the required AWS services and dependencies. +Use the following command: + +```bash +cargo run --release --bin orchestrator setup --aws --aws-s3 --aws-sqs --aws-sns --aws-event-bridge +``` + +Note: Setup mode is currently in development. A fresh setup is required +if the process fails mid-way. + +### Run Mode + +Run mode executes the orchestrator's job processing workflow. Example command: + +```bash +RUST_LOG=info cargo run --release --bin orchestrator run --atlantic --aws --settle-on-ethereum --aws-s3 --aws-sqs --aws-sns --da-on-ethereum --mongodb +``` + +### Command Line Options + +1. **Prover Services** (choose one): + + - `--atlantic`: Use Atlantic prover + - `--sharp`: Use SHARP prover + +2. **Settlement Layer** (choose one): + + - `--settle-on-ethereum`: Use Ethereum + - `--settle-on-starknet`: Use Starknet + +3. **Data Availability**: + + - `--da-on-ethereum`: Use Ethereum + +4. **Infrastructure**: + + - `--aws`: Use AWS services (or Localstack) + +5. **Data Storage**: + + - `--aws-s3`: Store state updates and program outputs + +6. **Database**: + + - `--mongodb`: Store job information + +7. **Queue System**: + + - `--aws-sqs`: Message queue service + +8. **Alerting**: + + - `--aws-sns`: Notification service + +9. **Scheduling**: + + - `--aws-event-bridge`: Cron job scheduling + +10. **Monitoring**: + - `--otel-service-name`: OpenTelemetry service name + - `--otel-collector-endpoint`: OpenTelemetry collector endpoint + +## โš™๏ธ Configuration + +The orchestrator uses environment variables for configuration. +Create a `.env` file with the following sections: + +### AWS Configuration + +```env +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_REGION= +``` + +Note: These configurations are also picked up from + +### Prover Configuration + +```env +# SHARP Configuration +MADARA_ORCHESTRATOR_SHARP_CUSTOMER_ID= +MADARA_ORCHESTRATOR_SHARP_URL= +# or +# ATLANTIC Configuration +MADARA_ORCHESTRATOR_ATLANTIC_API_KEY= +MADARA_ORCHESTRATOR_ATLANTIC_SERVICE_URL= +``` + +### Database Configuration + +```env +MADARA_ORCHESTRATOR_MONGODB_CONNECTION_URL=mongodb://localhost:27017 +MADARA_ORCHESTRATOR_DATABASE_NAME=orchestrator +``` + +For a complete list of configuration options, refer to the `.env.example` file +in the repository. + +## ๐Ÿ” Monitoring + +The orchestrator includes a telemetry system that tracks: + +- Job execution metrics +- Processing time statistics +- RPC performance metrics + +OpenTelemetry integration is available for detailed monitoring. +It requires a `Otel-collector` url to be able to send metrics/logs/traces. + +## ๐Ÿ› Error Handling + +- Failed jobs are moved to a dedicated failure handling queue +- Automatic retry mechanism with configurable intervals +- Failed jobs are tracked in the database for manual inspection after maximum retries +- Integrated telemetry system for monitoring job failures + +## ๐Ÿงช Testing + +The Madara Orchestrator supports three types of tests: + +### Types of Tests + +1. **E2E Tests** ๐Ÿ”„ + + - End-to-end workflow testing + - Tests orchestrator functionality on block 66645 of Starknet + - Uses mocked proving endpoints + +2. **Integration & Unit Tests** ๐Ÿ”Œ + +### Running Tests + +#### Required Services + +- MongoDB running on port 27017 +- Localstack running on port 4566 +- Anvil (local Ethereum node) + +#### Environment Configuration + +```bash +export MADARA_ORCHESTRATOR_ETHEREUM_SETTLEMENT_RPC_URL= +export MADARA_ORCHESTRATOR_RPC_FOR_SNOS= +export AWS_REGION=us-east-1 +``` + +#### Running E2E Tests + +```bash +RUST_LOG=info cargo test --features testing test_orchestrator_workflow -- --nocapture +``` + +#### Running Integration and Unit Tests + +The orchestrator uses LLVM coverage testing to ensure comprehensive test coverage +of the codebase. + +```bash +RUST_LOG=debug RUST_BACKTRACE=1 cargo llvm-cov nextest \ + --release \ + --features testing \ + --lcov \ + --output-path lcov.info \ + --test-threads=1 \ + --workspace \ + --exclude=e2e-tests \ + --no-fail-fast +``` + +This command: + +- Generates detailed coverage reports in LCOV format +- Excludes E2E tests from coverage analysis +- Runs tests sequentially (single thread) +- Continues testing even if failures occur +- Enables debug logging and full backtraces for better error diagnosis + +The coverage report (`lcov.info`) can be used with various code coverage +visualization tools. + +## ๐Ÿ““ More Information + +- Read the architecture present at `./docs/orchestrator_da_sequencer_diagram.png`