diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b0f94ab..ad9bc3c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: fail-fast: false matrix: os: [ ubuntu-20.04 ] - feature: [ "22_0", "0_21_1", "0_21_0", "0_20_1", "0_20_0", "0_19_1", "0_19_0_1", "0_18_1", "0_18_0", "0_17_1"] + feature: [ "23_0", "22_0", "0_21_1", "0_21_0", "0_20_1", "0_20_0", "0_19_1", "0_19_0_1", "0_18_1", "0_18_0", "0_17_1"] include: - os: "macos-10.15" feature: "0_21_1" @@ -31,7 +31,7 @@ jobs: with: toolchain: stable override: true - - run: echo "TEMPDIR_ROOT=/dev/shm" >> $GITHUB_ENV + #- run: echo "TEMPDIR_ROOT=/dev/shm" >> $GITHUB_ENV # conflicts with test `test_data_persistence` if: ${{ matrix.os != 'macos-10.15' }} - uses: actions-rs/cargo@v1 with: diff --git a/Cargo.toml b/Cargo.toml index 1634046..1e478b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bitcoind" -version = "0.24.0" +version = "0.26.1" authors = ["Riccardo Casatta "] description = "Utility to run a regtest bitcoind process, useful in integration testing environment" license = "MIT" @@ -9,10 +9,11 @@ documentation = "https://docs.rs/bitcoind/" edition = "2018" [dependencies] -bitcoincore-rpc = "0.14.0" +bitcoincore-rpc = "0.15.0" tempfile = "3.1" log = "0.4" home = "0.5.3" # use same ver in build-dep +which = "4.2.5" [dev-dependencies] env_logger = "0.8" @@ -20,10 +21,12 @@ env_logger = "0.8" [build-dependencies] ureq = "2.1" bitcoin_hashes = "0.10" -flate2 = "1.0" tar = "0.4" +filetime = "=0.2.15" +flate2 = "=1.0.22" [features] +"23_0" = [] "22_0" = [] "0_21_1" = [] "0_21_0" = [] diff --git a/Readme.md b/README.md similarity index 93% rename from Readme.md rename to README.md index 4568419..e4c3d72 100644 --- a/Readme.md +++ b/README.md @@ -7,7 +7,8 @@ Utility to run a regtest bitcoind process, useful in integration testing environ ```rust use bitcoincore_rpc::RpcApi; -let bitcoind = bitcoind::BitcoinD::new("/usr/local/bin/bitcoind").unwrap(); +let exe_path = exe_path().expect("bitcoind executable must be provided in BITCOIND_EXE, or with a feature like '22_0', or be in PATH"); +let bitcoind = bitcoind::BitcoinD::new(exe_path).unwrap(); assert_eq!(0, bitcoind.client.get_blockchain_info().unwrap().blocks); ``` diff --git a/build.rs b/build.rs index d93ec4d..e7ef361 100644 --- a/build.rs +++ b/build.rs @@ -21,6 +21,11 @@ fn download_filename() -> String { format!("bitcoin-{}-x86_64-linux-gnu.tar.gz", &VERSION) } +#[cfg(all(target_os = "linux", target_arch = "aarch64"))] +fn download_filename() -> String { + format!("bitcoin-{}-aarch64-linux-gnu.tar.gz", &VERSION) +} + fn get_expected_sha256(filename: &str) -> Result { let sha256sums_filename = format!("sha256/bitcoin-core-{}-SHA256SUMS", &VERSION); #[cfg(any( diff --git a/sha256/bitcoin-core-23.0-SHA256SUMS b/sha256/bitcoin-core-23.0-SHA256SUMS new file mode 100644 index 0000000..2786db9 --- /dev/null +++ b/sha256/bitcoin-core-23.0-SHA256SUMS @@ -0,0 +1,27 @@ +868d4be1f776a4d5e3230d9d6067bc5a6cbe094e6fbc270f567786e253cdc8d1 bitcoin-23.0-aarch64-linux-gnu-debug.tar.gz +06f4c78271a77752ba5990d60d81b1751507f77efda1e5981b4e92fd4d9969fb bitcoin-23.0-aarch64-linux-gnu.tar.gz +198c575150de5501940a120ecfb49670a7ac94d95bd875a854efd54a527f83fe bitcoin-23.0-arm-linux-gnueabihf-debug.tar.gz +952c574366aff76f6d6ad1c9ee45a361d64fa04155e973e926dfe7e26f9703a3 bitcoin-23.0-arm-linux-gnueabihf.tar.gz +a3059280451d17a77d2260e4671c884be93a14dbff6b6cd19a3c9c8c54421e97 bitcoin-23.0-arm64-apple-darwin.dmg +c991de5922cb2880f0f54a7f21ba650da40872b4a6dad73fae09d7a89d8c8f28 bitcoin-23.0-arm64-apple-darwin-unsigned.dmg +cdb380556e9858b22d9e9bfe3c2200f2d7efdc12af505390ca7798d97f6cd57c bitcoin-23.0-arm64-apple-darwin-unsigned.tar.gz +7c8bc63731aa872b7b334a8a7d96e33536ad77d49029bad179b09dca32cd77ac bitcoin-23.0-arm64-apple-darwin.tar.gz +a382c13777c090fbe1353f19a579c5f79a07285c6d7b04176180c1ce829cd7f7 bitcoin-23.0-codesignatures-e36a046909ad.tar.gz +26748bf49d6d6b4014d0fedccac46bf2bcca42e9d34b3acfd9e3467c415acc05 bitcoin-23.0.tar.gz +7815a86a30bcebe10b0c27e465172b2bc79e93ea1ddda724eb33508685a02fe9 bitcoin-23.0-powerpc64-linux-gnu-debug.tar.gz +2caa5898399e415f61d9af80a366a3008e5856efa15aaff74b88acf429674c99 bitcoin-23.0-powerpc64-linux-gnu.tar.gz +dda063652da7cad4839c11dff7a815a78ed8cd0f97dfdc6b10708cecb701085e bitcoin-23.0-powerpc64le-linux-gnu-debug.tar.gz +217dd0469d0f4962d22818c368358575f6a0abcba8804807bb75325eb2f28b19 bitcoin-23.0-powerpc64le-linux-gnu.tar.gz +5faabaec6e217bdea1b1f85b116659692a2f0e437767d35d17eb4b0e5c04c09c bitcoin-23.0-riscv64-linux-gnu-debug.tar.gz +078f96b1e92895009c798ab827fb3fde5f6719eee886bd0c0e93acab18ea4865 bitcoin-23.0-riscv64-linux-gnu.tar.gz +52eefbaf8cfd292822e470a48a51e1eb51081d43a0a16db7441f34a017ff6097 bitcoin-23.0-x86_64-apple-darwin.dmg +050eb122ef226ab9cfd6fe19a71c0a49029b9eac13faf033dbb60cdfa3a55c65 bitcoin-23.0-x86_64-apple-darwin-unsigned.dmg +ac43412ba3fea4436a90ada71decb25ee122b60396ce6a67f933424dc3dceb17 bitcoin-23.0-x86_64-apple-darwin-unsigned.tar.gz +c816780583009a9dad426dc0c183c89be9da98906e1e2c7ebae91041c1aaaaf3 bitcoin-23.0-x86_64-apple-darwin.tar.gz +a5a86632775fb2c1db4235bd56396ecfeb233bfa24431baf936c41e51cc24fdf bitcoin-23.0-x86_64-linux-gnu-debug.tar.gz +2cca490c1f2842884a3c5b0606f179f9f937177da4eadd628e3f7fd7e25d26d0 bitcoin-23.0-x86_64-linux-gnu.tar.gz +4198eba8ac326d8746ab43364a44a5f20c157b6701f8c35b80d639a676df9011 bitcoin-23.0-win64-setup.exe +02f6c3bde5448527282aafafe7fdb80f35d4f984d9b012a9cb5e5efd28861614 bitcoin-23.0-win64-debug.zip +beb4e86f629048e1a6616a882a2d66407ca6f368c9bc63cf117f2fb291ce1ced bitcoin-23.0-win64-setup-unsigned.exe +39a7b022b38f301029ee9d5732ed7e77851d465d44177b4c734cdcc33e6763b6 bitcoin-23.0-win64-unsigned.tar.gz +004b2e25b21e0f14cbcce6acec37f221447abbb3ea7931c689e508054bfc6cf6 bitcoin-23.0-win64.zip diff --git a/src/lib.rs b/src/lib.rs index 9c5c613..c87439d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,13 +15,13 @@ mod versions; use crate::bitcoincore_rpc::jsonrpc::serde_json::Value; use bitcoincore_rpc::{Auth, Client, RpcApi}; -use log::debug; +use log::{debug, error, warn}; use std::ffi::OsStr; use std::net::{Ipv4Addr, SocketAddrV4, TcpListener}; use std::path::PathBuf; use std::process::{Child, Command, ExitStatus, Stdio}; use std::time::Duration; -use std::{env, fmt, thread}; +use std::{env, fmt, fs, thread}; use tempfile::TempDir; pub extern crate bitcoincore_rpc; @@ -33,14 +33,32 @@ pub struct BitcoinD { process: Child, /// Rpc client linked to this bitcoind process pub client: Client, - /// Work directory, where the node store blocks and other stuff. It is kept in the struct so that - /// directory is deleted only when this struct is dropped - _work_dir: TempDir, + /// Work directory, where the node store blocks and other stuff. + work_dir: DataDir, /// Contains information to connect to this node pub params: ConnectParams, } +/// The DataDir struct defining the kind of data directory the node +/// will contain. Data directory can be either persistent, or temporary. +pub enum DataDir { + /// Persistent Data Directory + Persistent(PathBuf), + /// Temporary Data Directory + Temporary(TempDir), +} + +impl DataDir { + /// Return the data directory path + fn path(&self) -> PathBuf { + match self { + Self::Persistent(path) => path.to_owned(), + Self::Temporary(tmp_dir) => tmp_dir.path().to_path_buf(), + } + } +} + #[derive(Debug, Clone)] /// Contains all the information to connect to this node pub struct ConnectParams { @@ -55,7 +73,7 @@ pub struct ConnectParams { } /// Enum to specify p2p settings -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone)] pub enum P2P { /// the node doesn't open a p2p port and work in standalone mode No, @@ -77,10 +95,15 @@ pub enum Error { NoFeature, /// Returned when calling methods requiring a env var to exist, but it's not NoEnvVar, - /// Returned when calling methods requiring either a feature or env var, but both are absent - NeitherFeatureNorEnvVar, + /// Returned when calling methods requiring the bitcoind executable but none is found + /// (no feature, no `BITCOIND_EXE`, no `bitcoind` in `PATH` ) + NoBitcoindExecutableFound, /// Returned when calling methods requiring either a feature or anv var, but both are present BothFeatureAndEnvVar, + /// Wrapper of early exit status + EarlyExit(ExitStatus), + /// Returned when both tmpdir and staticdir is specified in `Conf` options + BothDirsSpecified, } impl fmt::Debug for Error { @@ -90,8 +113,10 @@ impl fmt::Debug for Error { Error::Rpc(e) => write!(f, "{:?}", e), Error::NoFeature => write!(f, "Called a method requiring a feature to be set, but it's not"), Error::NoEnvVar => write!(f, "Called a method requiring env var `BITCOIND_EXE` to be set, but it's not"), - Error::NeitherFeatureNorEnvVar => write!(f, "Called a method requiring env var `BITCOIND_EXE` or a feature to be set, but neither are set"), + Error::NoBitcoindExecutableFound => write!(f, "`bitcoind` executable is required, provide it with one of the following: set env var `BITCOIND_EXE` or use a feature like \"22_0\" or have `bitcoind` executable in the `PATH`"), Error::BothFeatureAndEnvVar => write!(f, "Called a method requiring env var `BITCOIND_EXE` or a feature to be set, but both are set"), + Error::EarlyExit(e) => write!(f, "The bitcoind process terminated early with exit code {}", e), + Error::BothDirsSpecified => write!(f, "tempdir and staticdir cannot be enabled at same time in configuration options") } } } @@ -120,11 +145,13 @@ const LOCAL_IP: Ipv4Addr = Ipv4Addr::new(127, 0, 0, 1); /// conf.p2p = bitcoind::P2P::No; /// conf.network = "regtest"; /// conf.tmpdir = None; +/// conf.staticdir = None; +/// conf.attempts = 3; /// assert_eq!(conf, bitcoind::Conf::default()); /// ``` /// #[non_exhaustive] -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone)] pub struct Conf<'a> { /// Bitcoind command line arguments containing no spaces like `vec!["-dbcache=300", "-regtest"]` /// note that `port`, `rpcport`, `connect`, `datadir`, `listen` @@ -141,12 +168,30 @@ pub struct Conf<'a> { /// directory with different/esoteric networks pub network: &'a str, - /// Optionally specify the root of where the temporary directories will be created. - /// If none and the env var `TEMPDIR_ROOT` is set, the env var is used. - /// If none and the env var `TEMPDIR_ROOT` is not set, the default temp dir of the OS is used. - /// It may be useful for example to set to a ramdisk so that bitcoin nodes spawn very fast - /// because their datadirs are in RAM + /// Optionally specify a temporary or persistent working directory for the node. + /// The following two parameters can be configured to simulate desired working directory configuration. + /// + /// tmpdir is Some() && staticdir is Some() : Error. Cannot be enabled at same time. + /// tmpdir is Some(temp_path) && staticdir is None : Create temporary directory at `tmpdir` path. + /// tmpdir is None && staticdir is Some(work_path) : Create persistent directory at `staticdir` path. + /// tmpdir is None && staticdir is None: Creates a temporary directory in OS default temporary directory (eg /tmp) or `TEMPDIR_ROOT` env variable path. + /// + /// It may be useful for example to set to a ramdisk via `TEMPDIR_ROOT` env option so that + /// bitcoin nodes spawn very fast because their datadirs are in RAM. Should not be enabled with persistent + /// mode, as it cause memory overflows. + + /// Temporary directory path pub tmpdir: Option, + + /// Persistent directory path + pub staticdir: Option, + + /// Try to spawn the process `attempt` time + /// + /// The OS is giving available ports to use, however, they aren't booked, so it could rarely + /// happen they are used at the time the process is spawn. When retrying other available ports + /// are returned reducing the probability of conflicts to negligible. + pub attempts: u8, } impl Default for Conf<'_> { @@ -157,6 +202,8 @@ impl Default for Conf<'_> { p2p: P2P::No, network: "regtest", tmpdir: None, + staticdir: None, + attempts: 3, } } } @@ -171,16 +218,23 @@ impl BitcoinD { /// Launch the bitcoind process from the given `exe` executable with given [Conf] param pub fn with_conf>(exe: S, conf: &Conf) -> Result { - let work_dir = match &conf.tmpdir { - Some(path) => TempDir::new_in(path), - None => match env::var("TEMPDIR_ROOT") { - Ok(env_path) => TempDir::new_in(env_path), - Err(_) => TempDir::new(), - }, - }?; - debug!("work_dir: {:?}", work_dir); - let datadir = work_dir.path().to_path_buf(); - let cookie_file = datadir.join(conf.network).join(".cookie"); + let tmpdir = conf + .tmpdir + .clone() + .or_else(|| env::var("TEMPDIR_ROOT").map(PathBuf::from).ok()); + let work_dir = match (&tmpdir, &conf.staticdir) { + (Some(_), Some(_)) => return Err(Error::BothDirsSpecified), + (Some(tmpdir), None) => DataDir::Temporary(TempDir::new_in(tmpdir)?), + (None, Some(workdir)) => { + fs::create_dir_all(workdir)?; + DataDir::Persistent(workdir.to_owned()) + } + (None, None) => DataDir::Temporary(TempDir::new()?), + }; + + let work_dir_path = work_dir.path(); + debug!("work_dir: {:?}", work_dir_path); + let cookie_file = work_dir_path.join(conf.network).join(".cookie"); let rpc_port = get_available_port()?; let rpc_socket = SocketAddrV4::new(LOCAL_IP, rpc_port); let rpc_url = format!("http://{}", rpc_socket); @@ -211,7 +265,7 @@ impl BitcoinD { Stdio::null() }; - let datadir_arg = format!("-datadir={}", datadir.display()); + let datadir_arg = format!("-datadir={}", work_dir_path.display()); let rpc_arg = format!("-rpcport={}", rpc_port); let default_args = [&datadir_arg, &rpc_arg]; @@ -221,7 +275,7 @@ impl BitcoinD { default_args, p2p_args ); - let process = Command::new(exe) + let mut process = Command::new(exe.as_ref()) .args(&default_args) .args(&p2p_args) .args(&conf.args) @@ -231,6 +285,17 @@ impl BitcoinD { let node_url_default = format!("{}/wallet/default", rpc_url); // wait bitcoind is ready, use default wallet let client = loop { + if let Some(status) = process.try_wait()? { + if conf.attempts > 0 { + warn!("early exit with: {:?}. Trying to launch again ({} attempts remaining), maybe some other process used our available port", status, conf.attempts); + let mut conf = conf.clone(); + conf.attempts -= 1; + return Self::with_conf(exe, &conf); + } else { + error!("early exit with: {:?}", status); + return Err(Error::EarlyExit(status)); + } + } thread::sleep(Duration::from_millis(500)); assert!(process.stderr.is_none()); let client_result = Client::new(&rpc_url, Auth::CookieFile(cookie_file.clone())); @@ -239,11 +304,15 @@ impl BitcoinD { // to be compatible with different version, in the end we are only interested if // the call is succesfull not in the returned value. if client_base.call::("getblockchaininfo", &[]).is_ok() { - client_base + // Try creating new wallet, if fails due to already existing wallet file + // try loading the same. Return if still errors. + if client_base .create_wallet("default", None, None, None, None) - .unwrap(); - break Client::new(&node_url_default, Auth::CookieFile(cookie_file.clone())) - .unwrap(); + .is_err() + { + client_base.load_wallet("default")?; + } + break Client::new(&node_url_default, Auth::CookieFile(cookie_file.clone()))?; } } }; @@ -251,9 +320,9 @@ impl BitcoinD { Ok(BitcoinD { process, client, - _work_dir: work_dir, + work_dir, params: ConnectParams { - datadir, + datadir: work_dir_path, cookie_file, rpc_socket, p2p_socket, @@ -277,6 +346,11 @@ impl BitcoinD { ) } + /// Return the current workdir path of the running node + pub fn workdir(&self) -> PathBuf { + self.work_dir.path() + } + /// Returns the [P2P] enum to connect to this node p2p port pub fn p2p_connect(&self, listen: bool) -> Option { self.params.p2p_socket.map(|s| P2P::Connect(s, listen)) @@ -304,6 +378,9 @@ impl BitcoinD { impl Drop for BitcoinD { fn drop(&mut self) { + if let DataDir::Persistent(_) = self.work_dir { + let _ = self.stop(); + } let _ = self.process.kill(); } } @@ -349,7 +426,9 @@ pub fn exe_path() -> Result { (Ok(_), Ok(_)) => Err(Error::BothFeatureAndEnvVar), (Ok(path), Err(_)) => Ok(path), (Err(_), Ok(path)) => Ok(path), - (Err(_), Err(_)) => Err(Error::NeitherFeatureNorEnvVar), + (Err(_), Err(_)) => which::which("bitcoind") + .map_err(|_| Error::NoBitcoindExecutableFound) + .map(|p| p.display().to_string()), } } @@ -361,6 +440,7 @@ mod test { use crate::{get_available_port, BitcoinD, Conf, LOCAL_IP, P2P}; use bitcoincore_rpc::RpcApi; use std::net::SocketAddrV4; + use tempfile::TempDir; #[test] fn test_local_ip() { @@ -373,7 +453,6 @@ mod test { #[test] fn test_bitcoind() { let exe = init(); - println!("{}", exe); let bitcoind = BitcoinD::new(exe).unwrap(); let info = bitcoind.client.get_blockchain_info().unwrap(); assert_eq!(0, info.blocks); @@ -416,8 +495,43 @@ mod test { assert_eq!(peers_connected(&other_bitcoind.client), 1); } + #[test] + fn test_data_persistence() { + // Create a Conf with staticdir type + let mut conf = Conf::default(); + let datadir = TempDir::new().unwrap(); + conf.staticdir = Some(datadir.path().to_path_buf()); + + // Start BitcoinD with persistent db config + // Generate 101 blocks + // Wallet balance should be 50 + let bitcoind = BitcoinD::with_conf(exe_path().unwrap(), &conf).unwrap(); + let core_addrs = bitcoind.client.get_new_address(None, None).unwrap(); + bitcoind + .client + .generate_to_address(101, &core_addrs) + .unwrap(); + let wallet_balance_1 = bitcoind.client.get_balance(None, None).unwrap(); + let best_block_1 = bitcoind.client.get_best_block_hash().unwrap(); + + drop(bitcoind); + + // Start a new BitcoinD with the same datadir + let bitcoind = BitcoinD::with_conf(exe_path().unwrap(), &conf).unwrap(); + + let wallet_balance_2 = bitcoind.client.get_balance(None, None).unwrap(); + let best_block_2 = bitcoind.client.get_best_block_hash().unwrap(); + + // Check node chain data persists + assert_eq!(best_block_1, best_block_2); + + // Check the node wallet balance persists + assert_eq!(wallet_balance_1, wallet_balance_2); + } + #[test] fn test_multi_p2p() { + let _ = env_logger::try_init(); let mut conf_node1 = Conf::default(); conf_node1.p2p = P2P::Yes; let node1 = BitcoinD::with_conf(exe_path().unwrap(), &conf_node1).unwrap(); diff --git a/src/versions.rs b/src/versions.rs index 81f8ebc..2df09ad 100644 --- a/src/versions.rs +++ b/src/versions.rs @@ -1,4 +1,5 @@ pub const HAS_FEATURE: bool = cfg!(any( + feature = "23_0", feature = "22_0", feature = "0_21_1", feature = "0_21_0", @@ -12,6 +13,7 @@ pub const HAS_FEATURE: bool = cfg!(any( )); #[cfg(not(any( + feature = "23_0", feature = "22_0", feature = "0_21_1", feature = "0_21_0", @@ -25,6 +27,9 @@ pub const HAS_FEATURE: bool = cfg!(any( )))] pub const VERSION: &str = "N/A"; +#[cfg(feature = "23_0")] +pub const VERSION: &str = "23.0"; + #[cfg(feature = "22_0")] pub const VERSION: &str = "22.0";