From 55b03bbbd55105763e173d1455f985d1b30f19b4 Mon Sep 17 00:00:00 2001 From: Micaiah Reid Date: Tue, 29 Aug 2023 13:49:55 -0400 Subject: [PATCH] feat: allow stacks-network to be run as standalone chain coordinator (#1064) * feat: add devnet package command * fix: lint * feat: use base structs in devnet package command * feat: add serde_with library * feat: using base structs and adding serializer/deserializer to their types * fix: contracts data serialization * fix: lint * feat: better serialization handling * fix: lint * feat: add custom serializers * feat: add serializer for QualifiedContractIdentifier * feat: add principal data serializer * feat: add memo serializer and deserializer * feat: wip deserealization * feat: wip deserialization * feat: wip deserialization * deserialization of deployment types * add serde_json dep * fix ProjectManifest deser * add todo * change devnet package help text * feat: refactoring type serialization and deserialization * better error handling * memo ser/de to bytes * default manifest file location for deserialization * changes on ConfigurationPackage struct * fix tmp dir * clean up error handling * changes from ludo's review * simplify contract id serde * git mv file * move cli/chainhooks to stacks-network/chainhooks * add deps * separate event and log modules for stacks-network * add separate functions for instantiating orchestrator * move event/log * make do_run_devnet able to run for devnet service * have `clarinet integrate` run local devnet version * add ability to run a non-local devnet service * dockerfile for stacks-network image * spelling on logging * fix url * some renaming * fix imports for stacks-devnet-js * remove working_dir override * remove unused import * fix logging for devnet * use same builder as other docker images * add serde_yaml dep * reduce how many times network manifest is read from disk * read files directly, not using `_File` type * add default project_manifest cache_location * fix stacks-devnet-js --------- Co-authored-by: Chris Guimaraes --- Cargo.lock | 227 +++++++-- components/clarinet-cli/src/bin.rs | 1 - .../clarinet-cli/src/chainhooks/types.rs | 433 ------------------ components/clarinet-cli/src/frontend/cli.rs | 27 +- components/clarinet-cli/src/integrate/mod.rs | 10 +- components/clarinet-cli/src/lib.rs | 1 - .../clarinet-deployments/src/onchain/mod.rs | 7 +- .../clarinet-files/src/project_manifest.rs | 2 +- components/stacks-devnet-js/src/lib.rs | 6 +- components/stacks-network/Cargo.toml | 16 + .../src/chainhooks.rs} | 9 +- .../stacks-network/src/chains_coordinator.rs | 45 +- components/stacks-network/src/event.rs | 95 ++++ components/stacks-network/src/lib.rs | 292 +++++------- components/stacks-network/src/log.rs | 52 +++ components/stacks-network/src/main.rs | 105 +++++ components/stacks-network/src/orchestrator.rs | 68 ++- components/stacks-network/src/ui/app.rs | 3 +- components/stacks-network/src/ui/ui.rs | 3 +- .../components/stacks-network.dockerfile | 11 + 20 files changed, 716 insertions(+), 697 deletions(-) delete mode 100644 components/clarinet-cli/src/chainhooks/types.rs rename components/{clarinet-cli/src/chainhooks/mod.rs => stacks-network/src/chainhooks.rs} (94%) create mode 100644 components/stacks-network/src/event.rs create mode 100644 components/stacks-network/src/log.rs create mode 100644 components/stacks-network/src/main.rs create mode 100644 dockerfiles/components/stacks-network.dockerfile diff --git a/Cargo.lock b/Cargo.lock index 23dc4c765..3025b9746 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -135,6 +135,54 @@ dependencies = [ "winapi", ] +[[package]] +name = "anstream" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" + +[[package]] +name = "anstyle-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys 0.48.0", +] + +[[package]] +name = "anstyle-wincon" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +dependencies = [ + "anstyle", + "windows-sys 0.48.0", +] + [[package]] name = "anyhow" version = "1.0.71" @@ -734,8 +782,8 @@ checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "atty", "bitflags 1.3.2", - "clap_derive", - "clap_lex", + "clap_derive 3.2.18", + "clap_lex 0.2.4", "indexmap 1.9.2", "once_cell", "strsim", @@ -743,13 +791,36 @@ dependencies = [ "textwrap", ] +[[package]] +name = "clap" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d" +dependencies = [ + "clap_builder", + "clap_derive 4.4.0", + "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6" +dependencies = [ + "anstream", + "anstyle", + "clap_lex 0.5.1", + "strsim", +] + [[package]] name = "clap_complete" version = "3.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f7a2e0a962c45ce25afce14220bc24f9dade0a1787f185cecf96bfba7847cd8" dependencies = [ - "clap", + "clap 3.2.23", ] [[package]] @@ -765,13 +836,25 @@ dependencies = [ "syn 1.0.107", ] +[[package]] +name = "clap_derive" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a" +dependencies = [ + "heck 0.4.1", + "proc-macro2 1.0.66", + "quote 1.0.28", + "syn 2.0.28", +] + [[package]] name = "clap_generate" version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e1b28c4a802ac3628604fd267cac62aaea74dc61af3410db6b1c44c03b42599" dependencies = [ - "clap", + "clap 3.2.23", "clap_complete", ] @@ -784,6 +867,12 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "clap_lex" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" + [[package]] name = "clarinet-cli" version = "1.7.1" @@ -796,7 +885,7 @@ dependencies = [ "bitcoin", "cache_control", "chrono", - "clap", + "clap 3.2.23", "clap_generate", "clarinet-deployments", "clarinet-files", @@ -977,7 +1066,7 @@ dependencies = [ name = "clarity-events" version = "1.0.0" dependencies = [ - "clap", + "clap 3.2.23", "clarinet-files", "clarity-repl 1.7.1", "serde", @@ -1150,6 +1239,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "colored" version = "1.9.3" @@ -5733,6 +5828,8 @@ dependencies = [ name = "stacks-network" version = "1.7.1" dependencies = [ + "ansi_term", + "atty", "base58 0.2.0", "bitcoin", "bitcoincore-rpc", @@ -5740,6 +5837,7 @@ dependencies = [ "bytes", "chainhook-sdk", "chrono", + "clap 4.4.0", "clarinet-deployments", "clarinet-files", "clarinet-utils 1.0.0", @@ -5754,6 +5852,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", + "serde_yaml", "stacks-rpc-client 1.0.7", "tracing", "tracing-appender", @@ -6740,17 +6839,17 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" [[package]] name = "toml_edit" -version = "0.19.5" +version = "0.19.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7082a95d48029677a28f181e5f6422d0c8339ad8396a39d3f33d62a90c1f6c30" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" dependencies = [ - "indexmap 1.9.2", + "indexmap 2.0.0", "toml_datetime", "winnow", ] @@ -7182,9 +7281,9 @@ checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "utf8parse" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" @@ -7440,7 +7539,7 @@ version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e745dab35a0c4c77aa3ce42d595e13d2003d6902d6b08c9ef5fc326d08da12b" dependencies = [ - "windows-targets", + "windows-targets 0.42.1", ] [[package]] @@ -7449,13 +7548,13 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.1", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm 0.42.1", + "windows_x86_64_msvc 0.42.1", ] [[package]] @@ -7464,7 +7563,16 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets", + "windows-targets 0.42.1", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", ] [[package]] @@ -7473,13 +7581,28 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.1", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm 0.42.1", + "windows_x86_64_msvc 0.42.1", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -7488,47 +7611,89 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_i686_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_x86_64_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "winnow" -version = "0.3.5" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee7b2c67f962bf5042bfd8b6a916178df33a26eec343ae064cb8e069f638fa6f" +checksum = "d09770118a7eb1ccaf4a594a221334119a44a814fcb0d31c5b85e83e97227a97" dependencies = [ "memchr", ] diff --git a/components/clarinet-cli/src/bin.rs b/components/clarinet-cli/src/bin.rs index 64623159c..f462ba61a 100644 --- a/components/clarinet-cli/src/bin.rs +++ b/components/clarinet-cli/src/bin.rs @@ -9,7 +9,6 @@ extern crate serde_json; #[macro_use] extern crate hiro_system_kit; -mod chainhooks; mod deployments; mod devnet; mod frontend; diff --git a/components/clarinet-cli/src/chainhooks/types.rs b/components/clarinet-cli/src/chainhooks/types.rs deleted file mode 100644 index e9d6828db..000000000 --- a/components/clarinet-cli/src/chainhooks/types.rs +++ /dev/null @@ -1,433 +0,0 @@ -use serde::{Deserialize, Serialize}; -use stacks_network::chainhook_sdk::chainhook_types::{BitcoinNetwork, StacksNetwork}; -use stacks_network::chainhook_sdk::chainhooks::types::*; -use std::collections::BTreeMap; -use std::fs::File; -use std::io::{BufReader, Read}; -use std::path::PathBuf; - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -pub struct ChainhookSpecificationFile { - id: Option, - name: String, - version: Option, - chain: String, - networks: BTreeMap, -} - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub struct ChainhookNetworkSpecificationFile { - start_block: Option, - end_block: Option, - expire_after_occurrence: Option, - predicate: ChainhookPredicateFile, - action: HookActionFile, -} - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub struct ChainhookPredicateFile { - print_event: Option, - ft_event: Option, - nft_event: Option, - stx_event: Option, - contract_call: Option>, - contract_deploy: Option, - txid: Option, - op_return: Option>, - p2pkh: Option>, - p2sh: Option>, - p2wpkh: Option>, - p2wsh: Option>, - script: Option>, - scope: Option, - protocol: Option, - operation: Option, -} - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub struct PrintEventPredicateFile { - contract_identifier: String, - contains: String, -} - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub struct ContractDeploymentPredicateFile { - deployer: Option, -} - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub struct FtEventPredicateFile { - asset_identifier: String, - actions: Vec, -} - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub struct NftEventPredicateFile { - asset_identifier: String, - actions: Vec, -} - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub struct StxEventPredicateFile { - actions: Vec, -} - -#[derive(Debug, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub struct HookActionFile { - http: Option>, - file: Option>, -} - -impl ChainhookSpecificationFile { - pub fn parse( - path: &PathBuf, - networks: &(BitcoinNetwork, StacksNetwork), - ) -> Result { - let file = ChainhookSpecificationFile::new(path)?; - file.to_specification(networks) - } - - pub fn new(path: &PathBuf) -> Result { - let path = match File::open(path) { - Ok(path) => path, - Err(_e) => { - return Err(format!("unable to locate {}", path.display())); - } - }; - let mut hook_spec_file_reader = BufReader::new(path); - let mut hook_spec_file_buffer = vec![]; - hook_spec_file_reader - .read_to_end(&mut hook_spec_file_buffer) - .unwrap(); - - let specification: ChainhookSpecificationFile = - match serde_yaml::from_slice(&hook_spec_file_buffer[..]) { - Ok(res) => res, - Err(msg) => return Err(format!("unable to read file {}", msg)), - }; - - Ok(specification) - } - - pub fn to_specification( - &self, - networks: &(BitcoinNetwork, StacksNetwork), - ) -> Result { - let res = if self.chain.to_lowercase() == "stacks" { - let res = self.to_stacks_specification(&networks.1)?; - ChainhookFullSpecification::Stacks(res) - } else if self.chain.to_lowercase() == "bitcoin" { - let res = self.to_bitcoin_specification(&networks.0)?; - ChainhookFullSpecification::Bitcoin(res) - } else { - return Err(format!( - "chain '{}' not supported (stacks, bitcoin)", - self.chain - )); - }; - Ok(res) - } - - pub fn to_bitcoin_specification( - &self, - network: &BitcoinNetwork, - ) -> Result { - let network_ser = format!("{:?}", network).to_lowercase(); - let network_spec = match self.networks.get(&network_ser) { - Some(entry) => entry, - None => { - return Err(format!( - "network '{}' not found in chainhook specification file", - network_ser - )) - } - }; - - let mut networks = BTreeMap::new(); - networks.insert( - network.clone(), - BitcoinChainhookNetworkSpecification { - start_block: network_spec.start_block, - end_block: network_spec.end_block, - expire_after_occurrence: network_spec.expire_after_occurrence, - predicate: network_spec.predicate.to_bitcoin_predicate()?, - action: network_spec.action.to_specifications()?, - }, - ); - Ok(BitcoinChainhookFullSpecification { - uuid: format!("{}", self.id.unwrap_or(1)), - owner_uuid: None, - version: self.version.unwrap_or(1), - name: self.name.to_string(), - networks, - }) - } - - pub fn to_stacks_specification( - &self, - network: &StacksNetwork, - ) -> Result { - let network_ser = format!("{:?}", network).to_lowercase(); - let network_spec = match self.networks.get(&network_ser) { - Some(entry) => entry, - None => { - return Err(format!( - "network '{}' not found in chainhook specification file", - network_ser - )) - } - }; - - let mut networks = BTreeMap::new(); - networks.insert( - network.clone(), - StacksChainhookNetworkSpecification { - capture_all_events: None, - decode_clarity_values: None, - start_block: network_spec.start_block, - end_block: network_spec.end_block, - expire_after_occurrence: network_spec.expire_after_occurrence, - predicate: network_spec.predicate.to_stacks_predicate()?, - action: network_spec.action.to_specifications()?, - }, - ); - Ok(StacksChainhookFullSpecification { - uuid: format!("{}", self.id.unwrap_or(1)), - owner_uuid: None, - version: self.version.unwrap_or(1), - name: self.name.to_string(), - networks, - }) - } -} - -impl HookActionFile { - pub fn to_specifications(&self) -> Result { - if let Some(ref specs) = self.http { - let url = match specs.get("url") { - Some(url) => Ok(url.to_string()), - None => Err(format!("url missing for http")), - }?; - let authorization_header = match specs.get("authorization-header") { - Some(authorization_header) => Ok(authorization_header.to_string()), - None => Err(format!("authorization-header missing for http")), - }?; - Ok(HookAction::HttpPost(HttpHook { - url, - authorization_header, - })) - } else if let Some(ref specs) = self.file { - let path = match specs.get("path") { - Some(path) => Ok(path.to_string()), - None => Err(format!("path missing for file")), - }?; - Ok(HookAction::FileAppend(FileHook { path })) - } else { - Err(format!("action not supported (http, file)")) - } - } -} - -impl ChainhookPredicateFile { - pub fn to_bitcoin_predicate(&self) -> Result { - if let Some(ref specs) = self.op_return { - let predicate = BitcoinPredicateType::Outputs(OutputPredicate::OpReturn( - self.extract_matching_rule(specs)?, - )); - return Ok(predicate); - } else if let Some(ref specs) = self.p2pkh { - let predicate = BitcoinPredicateType::Outputs(OutputPredicate::P2pkh( - self.extract_exact_matching_rule(specs)?, - )); - return Ok(predicate); - } else if let Some(ref specs) = self.p2sh { - let predicate = BitcoinPredicateType::Outputs(OutputPredicate::P2sh( - self.extract_exact_matching_rule(specs)?, - )); - return Ok(predicate); - } else if let Some(ref specs) = self.p2wpkh { - let predicate = BitcoinPredicateType::Outputs(OutputPredicate::P2wpkh( - self.extract_exact_matching_rule(specs)?, - )); - return Ok(predicate); - } else if let Some(ref specs) = self.p2wsh { - let predicate = BitcoinPredicateType::Outputs(OutputPredicate::P2wsh( - self.extract_exact_matching_rule(specs)?, - )); - return Ok(predicate); - } else if let Some(ref _specs) = self.protocol { - let predicate = BitcoinPredicateType::Protocol(Protocols::Ordinal( - OrdinalOperations::InscriptionRevealed, - )); - return Ok(predicate); - } - return Err(format!( - "trigger not specified (op-return, p2pkh, p2sh, p2wpkh, p2wsh)" - )); - } - - pub fn extract_matching_rule( - &self, - specs: &BTreeMap, - ) -> Result { - if let Some(rule) = specs.get("starts-with") { - return Ok(MatchingRule::StartsWith(rule.to_string())); - }; - - if let Some(rule) = specs.get("ends-with") { - return Ok(MatchingRule::EndsWith(rule.to_string())); - }; - - if let Some(rule) = specs.get("equals") { - return Ok(MatchingRule::Equals(rule.to_string())); - }; - - return Err(format!( - "predicate rule not specified (starts-with, ends-with, equals)" - )); - } - - pub fn extract_exact_matching_rule( - &self, - specs: &BTreeMap, - ) -> Result { - if let Some(rule) = specs.get("equals") { - return Ok(ExactMatchingRule::Equals(rule.to_string())); - }; - - return Err(format!("predicate rule not specified (equals)")); - } - - pub fn to_stacks_predicate(&self) -> Result { - if let Some(ref specs) = self.contract_call { - let predicate = self.extract_contract_call_predicate(specs)?; - return Ok(StacksPredicate::ContractCall(predicate)); - } else if let Some(ref specs) = self.print_event { - let predicate = self.extract_print_event_predicate(specs)?; - return Ok(StacksPredicate::PrintEvent(predicate)); - } else if let Some(ref specs) = self.ft_event { - let predicate = self.extract_ft_event_predicate(specs)?; - return Ok(StacksPredicate::FtEvent(predicate)); - } else if let Some(ref specs) = self.nft_event { - let predicate = self.extract_nft_event_predicate(specs)?; - return Ok(StacksPredicate::NftEvent(predicate)); - } else if let Some(ref specs) = self.stx_event { - let predicate = self.extract_stx_event_predicate(specs)?; - return Ok(StacksPredicate::StxEvent(predicate)); - } else if let Some(ref specs) = self.txid { - return Ok(StacksPredicate::Txid(specs.clone())); - } else if let Some(ref specs) = self.contract_deploy { - let predicate = self.extract_contract_deploy_predicate(specs)?; - return Ok(StacksPredicate::ContractDeployment(predicate)); - } - return Err(format!("trigger not specified (print-event, ft-event, nft-event, stx-event, contract-deploy, txid)")); - } - - pub fn extract_contract_call_predicate( - &self, - specs: &BTreeMap, - ) -> Result { - let contract_identifier = match specs.get("contract-identifier") { - Some(contract) => Ok(contract.to_string()), - None => Err(format!( - "contract-identifier missing for predicate.contract-call" - )), - }?; - let method = match specs.get("method") { - Some(method) => Ok(method.to_string()), - None => Err(format!("method missing for predicate.contract-call")), - }?; - Ok(StacksContractCallBasedPredicate { - contract_identifier, - method, - }) - } - - pub fn extract_contract_deploy_predicate( - &self, - specs: &ContractDeploymentPredicateFile, - ) -> Result { - if let Some(ref deployer) = specs.deployer { - return Ok(StacksContractDeploymentPredicate::Deployer( - deployer.clone(), - )); - } - return Err(format!( - "deployer not specified ('any', 'ST1PQHQKV0RJXZFY1DGX8MNSNYVE3VGZJSRTPGZGM', etc)" - )); - } - - pub fn extract_print_event_predicate( - &self, - specs: &PrintEventPredicateFile, - ) -> Result { - Ok(StacksPrintEventBasedPredicate { - contract_identifier: specs.contract_identifier.clone(), - contains: specs.contains.clone(), - }) - } - - pub fn extract_ft_event_predicate( - &self, - specs: &FtEventPredicateFile, - ) -> Result { - let available_actions = vec!["burn", "mint", "transfer"]; - for action in specs.actions.iter() { - if !available_actions.contains(&action.as_str()) { - return Err(format!( - "action not supported ({})", - available_actions.join(", ") - )); - } - } - Ok(StacksFtEventBasedPredicate { - asset_identifier: specs.asset_identifier.clone(), - actions: specs.actions.clone(), - }) - } - - pub fn extract_nft_event_predicate( - &self, - specs: &NftEventPredicateFile, - ) -> Result { - let available_actions = vec!["burn", "mint", "transfer"]; - for action in specs.actions.iter() { - if !available_actions.contains(&action.as_str()) { - return Err(format!( - "action not supported ({})", - available_actions.join(", ") - )); - } - } - Ok(StacksNftEventBasedPredicate { - asset_identifier: specs.asset_identifier.clone(), - actions: specs.actions.clone(), - }) - } - - pub fn extract_stx_event_predicate( - &self, - specs: &StxEventPredicateFile, - ) -> Result { - let available_actions = vec!["lock", "mint", "transfer"]; - for action in specs.actions.iter() { - if !available_actions.contains(&action.as_str()) { - return Err(format!( - "action not supported ({})", - available_actions.join(", ") - )); - } - } - Ok(StacksStxEventBasedPredicate { - actions: specs.actions.clone(), - }) - } -} diff --git a/components/clarinet-cli/src/frontend/cli.rs b/components/clarinet-cli/src/frontend/cli.rs index a9fb88b29..aeec3d5a7 100644 --- a/components/clarinet-cli/src/frontend/cli.rs +++ b/components/clarinet-cli/src/frontend/cli.rs @@ -1,4 +1,3 @@ -use crate::chainhooks::{check_chainhooks, load_chainhooks, parse_chainhook_full_specification}; use crate::deployments::types::DeploymentSynthesis; use crate::deployments::{ self, check_deployments, generate_default_deployment, get_absolute_deployment_path, @@ -24,7 +23,8 @@ use clarinet_deployments::{ use clarinet_files::chainhook_types::Chain; use clarinet_files::chainhook_types::StacksNetwork; use clarinet_files::{ - get_manifest_location, FileLocation, ProjectManifest, ProjectManifestFile, RequirementConfig, + get_manifest_location, FileLocation, NetworkManifest, ProjectManifest, ProjectManifestFile, + RequirementConfig, }; use clarity_repl::analysis::call_checker::ContractAnalysis; use clarity_repl::analysis::coverage::parse_coverage_str; @@ -37,7 +37,9 @@ use clarity_repl::repl::diagnostic::{output_code, output_diagnostic}; use clarity_repl::repl::{ClarityCodeSource, ClarityContract, ContractDeployer, DEFAULT_EPOCH}; use clarity_repl::{analysis, repl, Terminal}; use stacks_network::chainhook_sdk::chainhooks::types::ChainhookFullSpecification; -use stacks_network::{self, DevnetOrchestrator}; +use stacks_network::{ + self, check_chainhooks, load_chainhooks, parse_chainhook_full_specification, DevnetOrchestrator, +}; use std::collections::HashMap; use std::fs::{self, File}; use std::io::prelude::*; @@ -844,11 +846,24 @@ pub fn main() { } else { get_initial_transactions_trackers(&deployment) }; - + let network_moved = network.clone(); std::thread::spawn(move || { let manifest = manifest_moved; + let network_manifest = NetworkManifest::from_project_manifest_location( + &manifest.location, + &network_moved.get_networks(), + Some(&manifest.project.cache_location), + None, + ) + .expect("unable to load network manifest"); apply_on_chain_deployment( - &manifest, deployment, event_tx, command_rx, true, None, None, + network_manifest, + deployment, + event_tx, + command_rx, + true, + None, + None, ); }); @@ -1386,7 +1401,7 @@ pub fn main() { } }; - let orchestrator = match DevnetOrchestrator::new(manifest, None) { + let orchestrator = match DevnetOrchestrator::new(manifest, None, None, true) { Ok(orchestrator) => orchestrator, Err(e) => { println!("{}", format_err!(e)); diff --git a/components/clarinet-cli/src/integrate/mod.rs b/components/clarinet-cli/src/integrate/mod.rs index 82c354bf6..0f47d758a 100644 --- a/components/clarinet-cli/src/integrate/mod.rs +++ b/components/clarinet-cli/src/integrate/mod.rs @@ -5,14 +5,14 @@ use std::{ sync::mpsc::{self, channel, Sender}, }; -use crate::chainhooks::load_chainhooks; use clarinet_deployments::types::DeploymentSpecification; use hiro_system_kit::Drain; use hiro_system_kit::{slog, slog_async, slog_term}; -use stacks_network::chainhook_sdk::chainhook_types::{BitcoinNetwork, StacksNetwork}; -use stacks_network::chainhook_sdk::utils::Context; use stacks_network::{ - do_run_devnet, ChainsCoordinatorCommand, DevnetEvent, DevnetOrchestrator, LogData, + chainhook_sdk::chainhook_types::{BitcoinNetwork, StacksNetwork}, + chainhook_sdk::utils::Context, + do_run_local_devnet, load_chainhooks, ChainsCoordinatorCommand, DevnetEvent, + DevnetOrchestrator, LogData, }; use std::fs::OpenOptions; @@ -70,7 +70,7 @@ pub fn run_devnet( }; let (orchestrator_terminated_tx, orchestrator_terminated_rx) = channel(); - let res = hiro_system_kit::nestable_block_on(do_run_devnet( + let res = hiro_system_kit::nestable_block_on(do_run_local_devnet( devnet, deployment, &mut Some(hooks), diff --git a/components/clarinet-cli/src/lib.rs b/components/clarinet-cli/src/lib.rs index f7f799e77..74b4c8312 100644 --- a/components/clarinet-cli/src/lib.rs +++ b/components/clarinet-cli/src/lib.rs @@ -13,7 +13,6 @@ extern crate lazy_static; pub extern crate clarity_repl; -pub mod chainhooks; pub mod deployments; pub mod generate; pub mod integrate; diff --git a/components/clarinet-deployments/src/onchain/mod.rs b/components/clarinet-deployments/src/onchain/mod.rs index 4b8f5894f..3931bdbcf 100644 --- a/components/clarinet-deployments/src/onchain/mod.rs +++ b/components/clarinet-deployments/src/onchain/mod.rs @@ -1,6 +1,6 @@ use bitcoincore_rpc::{Auth, Client}; use clarinet_files::chainhook_types::StacksNetwork; -use clarinet_files::{AccountConfig, NetworkManifest, ProjectManifest}; +use clarinet_files::{AccountConfig, NetworkManifest}; use clarinet_utils::get_bip39_seed_from_mnemonic; use clarity_repl::clarity::codec::StacksMessageCodec; use clarity_repl::clarity::stacks_common::types::chainstate::StacksAddress; @@ -326,7 +326,7 @@ pub fn update_deployment_costs( } pub fn apply_on_chain_deployment( - manifest: &ProjectManifest, + network_manifest: NetworkManifest, deployment: DeploymentSpecification, deployment_event_tx: Sender, deployment_command_rx: Receiver, @@ -335,9 +335,6 @@ pub fn apply_on_chain_deployment( override_stacks_rpc_url: Option, ) { let network = deployment.network.get_networks(); - let network_manifest = - NetworkManifest::from_project_manifest_location(&manifest.location, &network, None, None) - .expect("unable to load network manifest"); let delay_between_checks: u64 = if network.1.is_devnet() { 1 } else { 10 }; // Load deployers, deployment_fee_rate // Check fee, balances and deployers diff --git a/components/clarinet-files/src/project_manifest.rs b/components/clarinet-files/src/project_manifest.rs index a2e8965f6..796cdaf30 100644 --- a/components/clarinet-files/src/project_manifest.rs +++ b/components/clarinet-files/src/project_manifest.rs @@ -217,7 +217,7 @@ impl Serialize for ProjectConfig { &self .cache_location .get_relative_location() - .expect("invalida cache_dir property"), + .unwrap_or(self.cache_location.to_string()), )?; if self.requirements.is_some() { map.serialize_entry("requirements", &self.requirements)?; diff --git a/components/stacks-devnet-js/src/lib.rs b/components/stacks-devnet-js/src/lib.rs index 0142a8354..bca194ed5 100644 --- a/components/stacks-devnet-js/src/lib.rs +++ b/components/stacks-devnet-js/src/lib.rs @@ -29,7 +29,7 @@ use std::{env, process}; type DevnetCallback = Box; use clarinet_deployments::types::{DeploymentGenerationArtifacts, DeploymentSpecification}; -use stacks_network::{do_run_devnet, ChainsCoordinatorCommand}; +use stacks_network::{do_run_local_devnet, ChainsCoordinatorCommand}; pub fn read_deployment_or_generate_default( manifest: &ProjectManifest, @@ -111,7 +111,7 @@ impl StacksDevnet { let (deployment, _) = read_deployment_or_generate_default(&manifest, &StacksNetwork::Devnet) .expect("Unable to generate deployment"); - let devnet = match DevnetOrchestrator::new(manifest, Some(devnet_overrides)) { + let devnet = match DevnetOrchestrator::new(manifest, None, Some(devnet_overrides), true) { Ok(devnet) => devnet, Err(message) => { if logs_enabled { @@ -147,7 +147,7 @@ impl StacksDevnet { match rx.recv() { Ok(DevnetCommand::Start(callback)) => { // Start devnet - let res = hiro_system_kit::nestable_block_on(do_run_devnet( + let res = hiro_system_kit::nestable_block_on(do_run_local_devnet( devnet, deployment, &mut None, diff --git a/components/stacks-network/Cargo.toml b/components/stacks-network/Cargo.toml index e2e67a6ef..94e10ac26 100644 --- a/components/stacks-network/Cargo.toml +++ b/components/stacks-network/Cargo.toml @@ -6,6 +6,8 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +atty = "0.2.14" +ansi_term = "0.12.1" bollard = "0.11.0" bytes = "1.4.0" crossterm = { version = "0.22.1", optional = true } @@ -41,6 +43,20 @@ clarinet-utils = { path = "../clarinet-utils" } hiro-system-kit = { path = "../hiro-system-kit", features = ["log"] } clarity-repl = { path = "../clarity-repl", features = ["cli"] } dirs = { version = "4.0.0" } +clap = {version = "4.2.7", features = [ "derive" ] } +serde_yaml = "0.8.23" + +[lib] +name = "stacks_network" +path = "src/lib.rs" +# Default type +# crate-type = ["lib"] +# Use this instead for WASM builds +crate-type = ["cdylib", "rlib"] + +[[bin]] +name = "stacks-network" +path = "src/main.rs" [features] default = ["crossterm"] diff --git a/components/clarinet-cli/src/chainhooks/mod.rs b/components/stacks-network/src/chainhooks.rs similarity index 94% rename from components/clarinet-cli/src/chainhooks/mod.rs rename to components/stacks-network/src/chainhooks.rs index 17d6c55db..4a514466b 100644 --- a/components/clarinet-cli/src/chainhooks/mod.rs +++ b/components/stacks-network/src/chainhooks.rs @@ -1,14 +1,11 @@ +use chainhook_sdk::chainhook_types::{BitcoinNetwork, StacksNetwork}; +use chainhook_sdk::chainhooks::types::{ChainhookConfig, ChainhookFullSpecification}; use clarinet_files::FileLocation; +use hiro_system_kit::{green, red}; use std::fs::File; use std::io::BufReader; use std::path::PathBuf; -use stacks_network::chainhook_sdk::chainhooks::types::{ - ChainhookConfig, ChainhookFullSpecification, -}; - -use stacks_network::chainhook_sdk::chainhook_types::{BitcoinNetwork, StacksNetwork}; - use std::fs; pub fn parse_chainhook_full_specification( diff --git a/components/stacks-network/src/chains_coordinator.rs b/components/stacks-network/src/chains_coordinator.rs index b5cc167a9..5a76649f8 100644 --- a/components/stacks-network/src/chains_coordinator.rs +++ b/components/stacks-network/src/chains_coordinator.rs @@ -1,7 +1,8 @@ use super::ChainsCoordinatorCommand; -use super::DevnetEvent; +use crate::event::DevnetEvent; +use crate::event::ServiceStatusData; +use crate::event::Status; use crate::orchestrator::ServicesMapHosts; -use crate::{ServiceStatusData, Status}; use base58::FromBase58; use chainhook_sdk::chainhook_types::BitcoinBlockSignaling; use chainhook_sdk::chainhook_types::BitcoinNetwork; @@ -55,6 +56,7 @@ pub struct DevnetEventObserverConfig { pub manifest: ProjectManifest, pub deployment_fee_rate: u64, pub services_map_hosts: ServicesMapHosts, + pub network_manifest: NetworkManifest, } impl DevnetEventObserverConfig { @@ -90,20 +92,23 @@ impl DevnetEventObserverConfig { pub fn new( devnet_config: DevnetConfig, manifest: ProjectManifest, + network_manifest: Option, deployment: DeploymentSpecification, chainhooks: ChainhookConfig, ctx: &Context, services_map_hosts: ServicesMapHosts, ) -> Self { ctx.try_log(|logger| slog::info!(logger, "Checking contracts")); - let network_manifest = NetworkManifest::from_project_manifest_location( - &manifest.location, - &StacksNetwork::Devnet.get_networks(), - Some(&manifest.project.cache_location), - None, - ) - .expect("unable to load network manifest"); - + let network_manifest = match network_manifest { + Some(n) => n, + None => NetworkManifest::from_project_manifest_location( + &manifest.location, + &StacksNetwork::Devnet.get_networks(), + Some(&manifest.project.cache_location), + None, + ) + .expect("unable to load network manifest"), + }; let event_observer_config = EventObserverConfig { bitcoin_rpc_proxy_enabled: true, event_handlers: vec![], @@ -123,11 +128,16 @@ impl DevnetEventObserverConfig { DevnetEventObserverConfig { devnet_config, event_observer_config, - accounts: network_manifest.accounts.into_values().collect::>(), + accounts: network_manifest + .accounts + .clone() + .into_values() + .collect::>(), manifest, deployment, deployment_fee_rate: network_manifest.network.deployment_fee_rate, services_map_hosts, + network_manifest, } } } @@ -155,7 +165,7 @@ pub async fn start_chains_coordinator( // This thread becomes dormant once the encoding is done, and proceed to the actual deployment once // the event DeploymentCommand::Start is received. perform_protocol_deployment( - &config.manifest, + &config.network_manifest, &config.deployment, deployment_events_tx, deployments_command_rx, @@ -378,6 +388,7 @@ pub async fn start_chains_coordinator( let res = publish_stacking_orders( &config.devnet_config, &config.accounts, + &config.services_map_hosts, config.deployment_fee_rate, bitcoin_block_height as u32, ) @@ -453,19 +464,18 @@ pub async fn start_chains_coordinator( } pub fn perform_protocol_deployment( - manifest: &ProjectManifest, + network_manifest: &NetworkManifest, deployment: &DeploymentSpecification, deployment_event_tx: Sender, deployment_command_rx: Receiver, override_bitcoin_rpc_url: Option, override_stacks_rpc_url: Option, ) { - let manifest = manifest.clone(); let deployment = deployment.clone(); - + let network_manifest = network_manifest.clone(); let _ = hiro_system_kit::thread_named("Deployment execution").spawn(move || { apply_on_chain_deployment( - &manifest, + network_manifest, deployment, deployment_event_tx, deployment_command_rx, @@ -516,6 +526,7 @@ pub fn relay_devnet_protocol_deployment( pub async fn publish_stacking_orders( devnet_config: &DevnetConfig, accounts: &Vec, + services_map_hosts: &ServicesMapHosts, fee_rate: u64, bitcoin_block_height: u32, ) -> Option { @@ -523,7 +534,7 @@ pub async fn publish_stacking_orders( return None; } - let stacks_node_rpc_url = format!("http://localhost:{}", devnet_config.stacks_node_rpc_port); + let stacks_node_rpc_url = format!("http://{}", &services_map_hosts.stacks_node_host); let mut transactions = 0; let pox_info: PoxInfo = reqwest::get(format!("{}/v2/pox", stacks_node_rpc_url)) diff --git a/components/stacks-network/src/event.rs b/components/stacks-network/src/event.rs new file mode 100644 index 000000000..eb57f6d03 --- /dev/null +++ b/components/stacks-network/src/event.rs @@ -0,0 +1,95 @@ +use std::sync::mpsc::Sender; + +use chainhook_sdk::{ + chainhook_types::{BitcoinChainEvent, StacksChainEvent}, + observer::MempoolAdmissionData, +}; + +use crate::{ + chains_coordinator::BitcoinMiningCommand, + log::{LogData, LogLevel}, +}; + +#[allow(dead_code)] +#[derive(Debug)] +pub enum DevnetEvent { + Log(LogData), + KeyEvent(crossterm::event::KeyEvent), + Tick, + ServiceStatus(ServiceStatusData), + ProtocolDeployingProgress(ProtocolDeployingData), + BootCompleted(Sender), + StacksChainEvent(StacksChainEvent), + BitcoinChainEvent(BitcoinChainEvent), + MempoolAdmission(MempoolAdmissionData), + FatalError(String), +} + +#[allow(dead_code)] +impl DevnetEvent { + pub fn error(message: String) -> DevnetEvent { + DevnetEvent::Log(Self::log_error(message)) + } + + #[allow(dead_code)] + pub fn warning(message: String) -> DevnetEvent { + DevnetEvent::Log(Self::log_warning(message)) + } + + pub fn info(message: String) -> DevnetEvent { + DevnetEvent::Log(Self::log_info(message)) + } + + pub fn success(message: String) -> DevnetEvent { + DevnetEvent::Log(Self::log_success(message)) + } + + pub fn debug(message: String) -> DevnetEvent { + DevnetEvent::Log(Self::log_debug(message)) + } + + pub fn log_error(message: String) -> LogData { + LogData::new(LogLevel::Error, message) + } + + pub fn log_warning(message: String) -> LogData { + LogData::new(LogLevel::Warning, message) + } + + pub fn log_info(message: String) -> LogData { + LogData::new(LogLevel::Info, message) + } + + pub fn log_success(message: String) -> LogData { + LogData::new(LogLevel::Success, message) + } + + pub fn log_debug(message: String) -> LogData { + LogData::new(LogLevel::Debug, message) + } +} + +#[derive(Clone, Debug)] +pub enum Status { + Red, + Yellow, + Green, +} + +#[derive(Clone, Debug)] +pub struct ServiceStatusData { + pub order: usize, + pub status: Status, + pub name: String, + pub comment: String, +} + +#[derive(Clone, Debug)] +pub struct ProtocolDeployingData { + pub new_contracts_deployed: Vec, +} + +#[derive(Clone, Debug)] +pub struct BootCompletedData { + pub contracts_deployed: Vec, +} diff --git a/components/stacks-network/src/lib.rs b/components/stacks-network/src/lib.rs index ad9b738cb..f109be664 100644 --- a/components/stacks-network/src/lib.rs +++ b/components/stacks-network/src/lib.rs @@ -3,16 +3,26 @@ extern crate serde; #[macro_use] extern crate serde_derive; +mod chainhooks; pub mod chains_coordinator; +mod event; +mod log; mod orchestrator; mod ui; +pub use chainhook_sdk::observer::MempoolAdmissionData; pub use chainhook_sdk::{self, utils::Context}; +pub use chainhooks::{check_chainhooks, load_chainhooks, parse_chainhook_full_specification}; +use chains_coordinator::BitcoinMiningCommand; +use clarinet_files::NetworkManifest; +pub use event::DevnetEvent; +pub use log::{LogData, LogLevel}; pub use orchestrator::DevnetOrchestrator; +use orchestrator::ServicesMapHosts; use std::{ - fmt, sync::{ + atomic::{AtomicBool, Ordering}, mpsc::{self, channel, Receiver, Sender}, Arc, }, @@ -20,18 +30,14 @@ use std::{ time::Duration, }; -use chainhook_sdk::chainhook_types::{BitcoinChainEvent, StacksChainEvent}; -use chainhook_sdk::{chainhooks::types::ChainhookConfig, observer::MempoolAdmissionData}; -use chains_coordinator::{start_chains_coordinator, BitcoinMiningCommand}; -use chrono::prelude::*; +use chainhook_sdk::chainhooks::types::ChainhookConfig; +use chains_coordinator::start_chains_coordinator; use clarinet_deployments::types::DeploymentSpecification; use hiro_system_kit; use hiro_system_kit::slog; -use std::sync::atomic::{AtomicBool, Ordering}; use tracing_appender; use self::chains_coordinator::DevnetEventObserverConfig; - #[allow(dead_code)] #[derive(Debug)] pub enum ChainsCoordinatorCommand { @@ -46,7 +52,7 @@ where rt.block_on(future) } -pub async fn do_run_devnet( +async fn do_run_devnet( mut devnet: DevnetOrchestrator, deployment: DeploymentSpecification, chainhooks: &mut Option, @@ -55,6 +61,9 @@ pub async fn do_run_devnet( ctx: Context, orchestrator_terminated_tx: Sender, orchestrator_terminated_rx: Option>, + ip_address_setup: ServicesMapHosts, + start_local_devnet_services: bool, + network_manifest: Option, ) -> Result< ( Option>, @@ -74,17 +83,23 @@ pub async fn do_run_devnet( }, _ => Err("Unable to retrieve config"), }?; - - let file_appender = - tracing_appender::rolling::never(&devnet_config.working_dir, "networking.log"); - let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender); - - let _ = tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) - .with_writer(non_blocking) - .try_init(); - - let ip_address_setup = devnet.prepare_network().await?; + // if we're starting all services, all trace logs go to networking.log + if start_local_devnet_services { + let file_appender = + tracing_appender::rolling::never(&devnet_config.working_dir, "networking.log"); + let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender); + + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .with_writer(non_blocking) + .try_init(); + } else { + // for the devnet, we can't write to a file, so we log everything to stdout, but we still want to set + // the max trace level so we don't get too much information in the logs + let _ = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .try_init(); + } // The event observer should be able to send some events to the UI thread, // and should be able to be terminated @@ -96,11 +111,13 @@ pub async fn do_run_devnet( let config = DevnetEventObserverConfig::new( devnet_config.clone(), devnet.manifest.clone(), + network_manifest, deployment, hooks, &ctx, ip_address_setup, ); + let chains_coordinator_tx = devnet_events_tx.clone(); let (chains_coordinator_commands_tx, chains_coordinator_commands_rx) = crossbeam_channel::unbounded(); @@ -134,19 +151,29 @@ pub async fn do_run_devnet( let orchestrator_event_tx = devnet_events_tx.clone(); let chains_coordinator_commands_tx_moved = chains_coordinator_commands_tx.clone(); let ctx_moved = ctx.clone(); - let orchestrator_handle = hiro_system_kit::thread_named("Devnet orchestrator") - .spawn(move || { - let future = devnet.start(orchestrator_event_tx.clone(), terminator_rx, &ctx_moved); - let rt = hiro_system_kit::create_basic_runtime(); - let res = rt.block_on(future); - if let Err(ref e) = res { - let _ = orchestrator_event_tx.send(DevnetEvent::FatalError(e.clone())); - let _ = - chains_coordinator_commands_tx_moved.send(ChainsCoordinatorCommand::Terminate); - } - res - }) - .expect("unable to retrieve join handle"); + let orchestrator_handle = { + hiro_system_kit::thread_named("Initializing bitcoin node") + .spawn(move || { + let moved_orchestrator_event_tx = orchestrator_event_tx.clone(); + let res = if start_local_devnet_services { + let future = + devnet.start(moved_orchestrator_event_tx, terminator_rx, &ctx_moved); + let rt = hiro_system_kit::create_basic_runtime(); + rt.block_on(future) + } else { + let future = devnet.initialize_bitcoin_node(&moved_orchestrator_event_tx); + let rt = hiro_system_kit::create_basic_runtime(); + rt.block_on(future) + }; + if let Err(ref e) = res { + let _ = orchestrator_event_tx.send(DevnetEvent::FatalError(e.clone())); + let _ = chains_coordinator_commands_tx_moved + .send(ChainsCoordinatorCommand::Terminate); + } + res + }) + .expect("unable to retrieve join handle") + }; if display_dashboard { ctx.try_log(|logger| slog::info!(logger, "Starting Devnet")); @@ -193,7 +220,6 @@ pub async fn do_run_devnet( if let Some(ref log_tx) = log_tx { let _ = log_tx.send(log.clone()); } else { - println!("{}", log.message); match log.level { LogLevel::Debug => { ctx.try_log(|logger| slog::debug!(logger, "{}", log.message)) @@ -234,142 +260,70 @@ pub async fn do_run_devnet( Ok((None, None, Some(chains_coordinator_commands_tx))) } -#[allow(dead_code)] -#[derive(Debug)] -pub enum DevnetEvent { - Log(LogData), - KeyEvent(crossterm::event::KeyEvent), - Tick, - ServiceStatus(ServiceStatusData), - ProtocolDeployingProgress(ProtocolDeployingData), - BootCompleted(Sender), - StacksChainEvent(StacksChainEvent), - BitcoinChainEvent(BitcoinChainEvent), - MempoolAdmission(MempoolAdmissionData), - FatalError(String), - // Restart, - // Terminate, -} - -#[allow(dead_code)] -impl DevnetEvent { - pub fn error(message: String) -> DevnetEvent { - DevnetEvent::Log(Self::log_error(message)) - } - - #[allow(dead_code)] - pub fn warning(message: String) -> DevnetEvent { - DevnetEvent::Log(Self::log_warning(message)) - } - - pub fn info(message: String) -> DevnetEvent { - DevnetEvent::Log(Self::log_info(message)) - } - - pub fn success(message: String) -> DevnetEvent { - DevnetEvent::Log(Self::log_success(message)) - } - - pub fn debug(message: String) -> DevnetEvent { - DevnetEvent::Log(Self::log_debug(message)) - } - - pub fn log_error(message: String) -> LogData { - LogData::new(LogLevel::Error, message) - } - - pub fn log_warning(message: String) -> LogData { - LogData::new(LogLevel::Warning, message) - } - - pub fn log_info(message: String) -> LogData { - LogData::new(LogLevel::Info, message) - } - - pub fn log_success(message: String) -> LogData { - LogData::new(LogLevel::Success, message) - } - - pub fn log_debug(message: String) -> LogData { - LogData::new(LogLevel::Debug, message) - } -} - -#[derive(Clone, Debug)] -pub enum LogLevel { - Error, - Warning, - Info, - Success, - Debug, -} - -impl fmt::Display for LogLevel { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "{}", - match &self { - LogLevel::Error => "erro", - LogLevel::Warning => "warn", - LogLevel::Info => "info", - LogLevel::Success => "succ", - LogLevel::Debug => "debg", - } - ) - } -} - -#[derive(Clone, Debug)] -pub struct LogData { - pub occurred_at: String, - pub message: String, - pub level: LogLevel, -} - -impl LogData { - pub fn new(level: LogLevel, message: String) -> LogData { - let now: DateTime = Utc::now(); - LogData { - level, - message, - occurred_at: now.format("%b %e %T%.6f").to_string(), - } - } -} - -impl fmt::Display for LogData { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{} [{}] {}", self.occurred_at, self.level, self.message) - } -} - -#[derive(Clone, Debug)] -pub enum Status { - Red, - Yellow, - Green, -} - -#[derive(Clone, Debug)] -pub struct ServiceStatusData { - pub order: usize, - pub status: Status, - pub name: String, - pub comment: String, -} - -#[derive(Clone, Debug)] -pub struct ProtocolDeployingData { - pub new_contracts_deployed: Vec, +pub async fn do_run_chain_coordinator( + mut devnet: DevnetOrchestrator, + deployment: DeploymentSpecification, + chainhooks: &mut Option, + log_tx: Option>, + ctx: Context, + orchestrator_terminated_tx: Sender, + namespace: &str, + network_manifest: NetworkManifest, +) -> Result< + ( + Option>, + Option>, + Option>, + ), + String, +> { + let ip_address_setup = devnet.prepare_network_k8s_coordinator(namespace)?; + do_run_devnet( + devnet, + deployment, + chainhooks, + log_tx, + false, + ctx, + orchestrator_terminated_tx, + None, + ip_address_setup, + false, + Some(network_manifest), + ) + .await } -#[derive(Clone, Debug)] -pub struct BootCompletedData { - pub contracts_deployed: Vec, +pub async fn do_run_local_devnet( + mut devnet: DevnetOrchestrator, + deployment: DeploymentSpecification, + chainhooks: &mut Option, + log_tx: Option>, + display_dashboard: bool, + ctx: Context, + orchestrator_terminated_tx: Sender, + orchestrator_terminated_rx: Option>, +) -> Result< + ( + Option>, + Option>, + Option>, + ), + String, +> { + let ip_address_setup = devnet.prepare_local_network().await?; + do_run_devnet( + devnet, + deployment, + chainhooks, + log_tx, + display_dashboard, + ctx, + orchestrator_terminated_tx, + orchestrator_terminated_rx, + ip_address_setup, + true, + None, + ) + .await } - -// pub struct MicroblockData { -// pub seq: u32, -// pub transactions: Vec -// } diff --git a/components/stacks-network/src/log.rs b/components/stacks-network/src/log.rs new file mode 100644 index 000000000..0f2a07ce6 --- /dev/null +++ b/components/stacks-network/src/log.rs @@ -0,0 +1,52 @@ +use std::fmt; + +use chrono::{DateTime, Utc}; + +#[derive(Clone, Debug)] +pub enum LogLevel { + Error, + Warning, + Info, + Success, + Debug, +} + +impl fmt::Display for LogLevel { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "{}", + match &self { + LogLevel::Error => "erro", + LogLevel::Warning => "warn", + LogLevel::Info => "info", + LogLevel::Success => "succ", + LogLevel::Debug => "debg", + } + ) + } +} + +#[derive(Clone, Debug)] +pub struct LogData { + pub occurred_at: String, + pub message: String, + pub level: LogLevel, +} + +impl LogData { + pub fn new(level: LogLevel, message: String) -> LogData { + let now: DateTime = Utc::now(); + LogData { + level, + message, + occurred_at: now.format("%b %e %T%.6f").to_string(), + } + } +} + +impl fmt::Display for LogData { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{} [{}] {}", self.occurred_at, self.level, self.message) + } +} diff --git a/components/stacks-network/src/main.rs b/components/stacks-network/src/main.rs new file mode 100644 index 000000000..0a5847687 --- /dev/null +++ b/components/stacks-network/src/main.rs @@ -0,0 +1,105 @@ +use chainhook_sdk::chainhook_types::{BitcoinNetwork, StacksNetwork}; + +use clap::Parser; +use clarinet_files::{FileLocation, NetworkManifest, ProjectManifest}; +use hiro_system_kit::slog; + +use std::path::PathBuf; +use std::sync::mpsc::channel; + +use stacks_network::{do_run_chain_coordinator, load_chainhooks}; +use stacks_network::{Context, DevnetOrchestrator}; + +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] +struct Args { + /// Devnet namespace + #[clap(short, long)] + namespace: String, + /// Path of the project manifest to load + #[clap(short, long)] + manifest_path: Option, + /// Path of the network manifest to load + #[clap(short, long)] + network_manifest_path: Option, + /// Path of the deployment plan + #[clap(short, long)] + deployment_plan_path: Option, + /// Path of the project's root + #[clap(short, long)] + project_root_path: Option, +} + +fn main() { + let args = Args::parse(); + let manifest_location = get_config_location_from_path_or_exit(&args.manifest_path); + let network_manifest_path = get_config_location_from_path_or_exit(&args.network_manifest_path); + let deployment_location = get_config_location_from_path_or_exit(&args.deployment_plan_path); + + let project_manifest_file_content = manifest_location + .read_content() + .unwrap_or_else(|e| panic!("failed to read manifest data {:?}", e)); + + let manifest: ProjectManifest = serde_yaml::from_slice(&project_manifest_file_content[..]) + .unwrap_or_else(|e| panic!("Clarinet.toml file malformatted {:?}", e)); + + let network_manifest_file_content = network_manifest_path + .read_content() + .unwrap_or_else(|e| panic!("failed to read network manifest data {:?}", e)); + + let network_manifest: NetworkManifest = + serde_yaml::from_slice(&network_manifest_file_content[..]) + .unwrap_or_else(|e| panic!("Devnet.toml file malformatted {:?}", e)); + + let orchestrator = + DevnetOrchestrator::new(manifest, Some(network_manifest.clone()), None, false).unwrap(); + + let deployment_specification_file_content = deployment_location + .read_content() + .unwrap_or_else(|e| panic!("failed to read manifest data {:?}", e)); + let deployment = serde_yaml::from_slice(&deployment_specification_file_content) + .unwrap_or_else(|e| panic!("deployment plan malformatted {:?}", e)); + + let chainhooks = match load_chainhooks( + &manifest_location, + &(BitcoinNetwork::Regtest, StacksNetwork::Devnet), + ) { + Ok(hooks) => hooks, + Err(e) => { + panic!("failed to load chainhooks {}", e); + } + }; + + let logger = hiro_system_kit::log::setup_logger(); + let _guard = hiro_system_kit::log::setup_global_logger(logger.clone()); + let ctx = Context { + logger: Some(logger), + tracer: false, + }; + ctx.try_log(|logger| slog::info!(logger, "starting devnet coordinator")); + + let (orchestrator_terminated_tx, _) = channel(); + let res = hiro_system_kit::nestable_block_on(do_run_chain_coordinator( + orchestrator, + deployment, + &mut Some(chainhooks), + None, + ctx, + orchestrator_terminated_tx, + &args.namespace, + network_manifest, + )); + println!("{:?}", res.unwrap()); +} + +fn get_config_location_from_path_or_exit(path: &Option) -> FileLocation { + if let Some(path) = path { + let path_buf = PathBuf::from(path); + if !path_buf.exists() { + std::process::exit(1); + } + FileLocation::from_path(path_buf) + } else { + std::process::exit(1); + } +} diff --git a/components/stacks-network/src/orchestrator.rs b/components/stacks-network/src/orchestrator.rs index 1b0181fa0..5c736bb9b 100644 --- a/components/stacks-network/src/orchestrator.rs +++ b/components/stacks-network/src/orchestrator.rs @@ -1,5 +1,3 @@ -use super::DevnetEvent; -use crate::{ServiceStatusData, Status}; use bollard::container::{ Config, CreateContainerOptions, KillContainerOptions, ListContainersOptions, PruneContainersOptions, WaitContainerOptions, @@ -25,6 +23,8 @@ use std::path::PathBuf; use std::sync::mpsc::{Receiver, Sender}; use std::time::Duration; +use crate::event::{DevnetEvent, ServiceStatusData, Status}; + #[derive(Debug)] pub struct DevnetOrchestrator { pub name: String, @@ -70,15 +70,19 @@ pub struct ServicesMapHosts { impl DevnetOrchestrator { pub fn new( manifest: ProjectManifest, + network_manifest: Option, devnet_override: Option, + should_use_docker: bool, ) -> Result { - let mut network_config = NetworkManifest::from_project_manifest_location( - &manifest.location, - &StacksNetwork::Devnet.get_networks(), - Some(&manifest.project.cache_location), - devnet_override, - )?; - + let mut network_config = match network_manifest { + Some(n) => Ok(n), + None => NetworkManifest::from_project_manifest_location( + &manifest.location, + &StacksNetwork::Devnet.get_networks(), + Some(&manifest.project.cache_location), + devnet_override, + ), + }?; if let Some(ref mut devnet) = network_config.devnet { let working_dir = PathBuf::from(&devnet.working_dir); let devnet_path = if working_dir.is_absolute() { @@ -110,9 +114,14 @@ impl DevnetOrchestrator { network_name.push_str(".net"); } - let docker_client = match network_config.devnet { - Some(ref devnet) => { - Docker::connect_with_socket(&devnet.docker_host, 120, bollard::API_DEFAULT_VERSION) + let docker_client = match should_use_docker { + true => match network_config.devnet { + Some(ref devnet) => { + let client = Docker::connect_with_socket( + &devnet.docker_host, + 120, + bollard::API_DEFAULT_VERSION, + ) .or_else(|_| Docker::connect_with_socket_defaults()) .or_else(|_| { let mut user_space_docker_socket = @@ -126,9 +135,12 @@ impl DevnetOrchestrator { bollard::API_DEFAULT_VERSION, ) }) - .map_err(|e| format!("unable to connect to docker: {:?}", e))? - } - None => unreachable!(), + .map_err(|e| format!("unable to connect to docker: {:?}", e))?; + Some(client) + } + None => unreachable!(), + }, + false => None, }; Ok(DevnetOrchestrator { @@ -136,7 +148,7 @@ impl DevnetOrchestrator { network_name, manifest, network_config: Some(network_config), - docker_client: Some(docker_client), + docker_client: docker_client, can_exit: true, termination_success_tx: None, stacks_node_container_id: None, @@ -151,7 +163,29 @@ impl DevnetOrchestrator { }) } - pub async fn prepare_network(&mut self) -> Result { + pub fn prepare_network_k8s_coordinator( + &mut self, + namespace: &str, + ) -> Result { + let services_map_hosts = ServicesMapHosts { + bitcoin_node_host: format!( + "bitcoind-chain-coordinator-service.{namespace}.svc.cluster.local:18443" + ), + stacks_node_host: format!("stacks-node-service.{namespace}.svc.cluster.local:20443"), + postgres_host: format!("stacks-api-service.{namespace}.svc.cluster.local:5432"), + stacks_api_host: format!("stacks-api-service.{namespace}.svc.cluster.local:3999"), + stacks_explorer_host: "localhost".into(), // todo (micaiah) + bitcoin_explorer_host: "localhost".into(), // todo (micaiah) + subnet_node_host: "localhost".into(), // todo (micaiah) + subnet_api_host: "localhost".into(), // todo (micaiah) + }; + + self.services_map_hosts = Some(services_map_hosts.clone()); + + Ok(services_map_hosts) + } + + pub async fn prepare_local_network(&mut self) -> Result { let (docker, devnet_config) = match (&self.docker_client, &self.network_config) { (Some(ref docker), Some(ref network_config)) => match network_config.devnet { Some(ref devnet_config) => (docker, devnet_config), diff --git a/components/stacks-network/src/ui/app.rs b/components/stacks-network/src/ui/app.rs index aa22c5d70..107507dd8 100644 --- a/components/stacks-network/src/ui/app.rs +++ b/components/stacks-network/src/ui/app.rs @@ -1,5 +1,6 @@ use super::util::{StatefulList, TabsState}; -use crate::{LogData, MempoolAdmissionData, ServiceStatusData}; +use crate::event::ServiceStatusData; +use crate::{LogData, MempoolAdmissionData}; use chainhook_sdk::chainhook_types::{ StacksBlockData, StacksMicroblockData, StacksTransactionData, }; diff --git a/components/stacks-network/src/ui/ui.rs b/components/stacks-network/src/ui/ui.rs index 4dd03f096..eb7fe4c13 100644 --- a/components/stacks-network/src/ui/ui.rs +++ b/components/stacks-network/src/ui/ui.rs @@ -1,8 +1,9 @@ -use crate::{LogLevel, Status}; use chainhook_sdk::chainhook_types::{ StacksBlockData, StacksMicroblockData, StacksTransactionData, }; +use crate::{event::Status, log::LogLevel}; + use super::{app::BlockData, App}; use tui::{ backend::Backend, diff --git a/dockerfiles/components/stacks-network.dockerfile b/dockerfiles/components/stacks-network.dockerfile new file mode 100644 index 000000000..2dadda077 --- /dev/null +++ b/dockerfiles/components/stacks-network.dockerfile @@ -0,0 +1,11 @@ +FROM rust:bullseye as build + +COPY . ./ + +RUN cargo build --release --manifest-path ./components/stacks-network/Cargo.toml + +# prod stage +FROM debian:bullseye-slim +COPY --from=build target/release/stacks-network / + +ENTRYPOINT ["./stacks-network"] \ No newline at end of file