From d155714f0beec4225768ddcff6305c372248e4e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=B3=CE=BB?= Date: Tue, 10 Dec 2024 21:27:45 +1100 Subject: [PATCH] Env substitution (#203) * Env substitution works * Move to dev-dependencies * Formatting * Ensure that there are no quotes or anything around substitution * use non normal env vars to avoid future issues --- packages/ciphernode/Cargo.lock | 55 ++++++++++------- packages/ciphernode/Cargo.toml | 2 + packages/ciphernode/config/Cargo.toml | 5 ++ packages/ciphernode/config/src/app_config.rs | 64 +++++++++++++++++++- packages/ciphernode/config/src/lib.rs | 1 + packages/ciphernode/config/src/yaml.rs | 51 ++++++++++++++++ 6 files changed, 155 insertions(+), 23 deletions(-) create mode 100644 packages/ciphernode/config/src/yaml.rs diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index 580d8beb..675eb7bd 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -1104,7 +1104,7 @@ checksum = "d7ebdfa2ebdab6b1760375fa7d6f382b9f486eac35fc994625a00e89280bdbb7" dependencies = [ "async-task", "concurrent-queue", - "fastrand 2.1.0", + "fastrand 2.3.0", "futures-lite 2.3.0", "slab", ] @@ -1169,7 +1169,7 @@ dependencies = [ "futures-lite 2.3.0", "parking", "polling 3.7.2", - "rustix 0.38.34", + "rustix 0.38.42", "slab", "tracing", "windows-sys 0.52.0", @@ -1219,7 +1219,7 @@ dependencies = [ "cfg-if", "event-listener 3.1.0", "futures-lite 1.13.0", - "rustix 0.38.34", + "rustix 0.38.42", "windows-sys 0.48.0", ] @@ -1235,7 +1235,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix 0.38.34", + "rustix 0.38.42", "signal-hook-registry", "slab", "windows-sys 0.59.0", @@ -1694,6 +1694,8 @@ dependencies = [ "dirs", "figment", "serde", + "shellexpand", + "tempfile", ] [[package]] @@ -2250,12 +2252,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2337,9 +2339,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fastrlp" @@ -2614,7 +2616,7 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" dependencies = [ - "fastrand 2.1.0", + "fastrand 2.3.0", "futures-core", "futures-io", "parking", @@ -3491,9 +3493,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "libm" @@ -4626,7 +4628,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1d5c74c9876f070d3e8fd503d748c7d974c3e48da8f41350fa5222ef9b4391" dependencies = [ "atomic-waker", - "fastrand 2.1.0", + "fastrand 2.3.0", "futures-io", ] @@ -4672,7 +4674,7 @@ dependencies = [ "concurrent-queue", "hermit-abi 0.4.0", "pin-project-lite", - "rustix 0.38.34", + "rustix 0.38.42", "tracing", "windows-sys 0.52.0", ] @@ -5363,15 +5365,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags 2.6.0", "errno", "libc", "linux-raw-sys 0.4.14", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5667,6 +5669,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" +[[package]] +name = "shellexpand" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da03fa3b94cc19e3ebfc88c4229c49d8f08cdbd1228870a45f0ffdf84988e14b" +dependencies = [ + "dirs", +] + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -5939,15 +5950,15 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.11.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fcd239983515c23a32fb82099f97d0b11b8c72f654ed659363a95c3dad7a53" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", - "fastrand 2.1.0", + "fastrand 2.3.0", "once_cell", - "rustix 0.38.34", - "windows-sys 0.52.0", + "rustix 0.38.42", + "windows-sys 0.59.0", ] [[package]] diff --git a/packages/ciphernode/Cargo.toml b/packages/ciphernode/Cargo.toml index c7df1e8c..c19f1b9d 100644 --- a/packages/ciphernode/Cargo.toml +++ b/packages/ciphernode/Cargo.toml @@ -39,6 +39,7 @@ clap = { version = "4.5.17", features = ["derive"] } cipher = { path = "./cipher" } dirs = "5.0.1" data = { path = "./data" } +shellexpand = "3.1.0" figment = { version = "0.10.19", features = ["yaml", "test"] } fhe_rs = { package = "fhe", git = "https://github.com/gnosisguild/fhe.rs", version = "0.1.0-beta.7" } fhe-traits = { git = "https://github.com/gnosisguild/fhe.rs", version = "0.1.0-beta.7" } @@ -53,6 +54,7 @@ rand = "0.8.5" serde = { version = "1.0.208", features = ["derive"] } sled = "0.34.7" sha2 = "0.10.8" +tempfile = "3.14.0" tokio = { version = "1.38", features = ["full"] } tracing = "0.1.37" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/packages/ciphernode/config/Cargo.toml b/packages/ciphernode/config/Cargo.toml index a694ceb5..93a4d74f 100644 --- a/packages/ciphernode/config/Cargo.toml +++ b/packages/ciphernode/config/Cargo.toml @@ -9,3 +9,8 @@ anyhow = { workspace = true } serde = { workspace = true } figment = { workspace = true } alloy = { workspace = true } +shellexpand = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } + diff --git a/packages/ciphernode/config/src/app_config.rs b/packages/ciphernode/config/src/app_config.rs index 49f7d0ac..9caa5da7 100644 --- a/packages/ciphernode/config/src/app_config.rs +++ b/packages/ciphernode/config/src/app_config.rs @@ -10,6 +10,8 @@ use std::{ path::{Path, PathBuf}, }; +use crate::yaml::load_yaml_with_env; + #[derive(Debug, Deserialize, Serialize, PartialEq)] #[serde(untagged)] pub enum Contract { @@ -202,8 +204,10 @@ pub fn load_config(config_file: Option<&str>) -> Result { defaults.config_file = file.into(); } + let with_envs = load_yaml_with_env(&defaults.config_file())?; + let config = Figment::from(Serialized::defaults(&defaults)) - .merge(Yaml::file(defaults.config_file())) + .merge(Yaml::string(&with_envs)) .extract()?; Ok(config) @@ -445,4 +449,62 @@ chains: Ok(()) }); } + + #[test] + fn test_config_env_vars() { + Jail::expect_with(|jail| { + let home = format!("{}", jail.directory().to_string_lossy()); + jail.set_env("HOME", &home); + jail.set_env("XDG_CONFIG_HOME", &format!("{}/.config", home)); + jail.set_env("TEST_RPC_URL_PORT", "8545"); + jail.set_env("TEST_USERNAME", "envUser"); + jail.set_env("TEST_PASSWORD", "envPassword"); + jail.set_env( + "TEST_CONTRACT_ADDRESS", + "0x1234567890123456789012345678901234567890", + ); + + let filename = format!("{}/.config/enclave/config.yaml", home); + let filedir = format!("{}/.config/enclave", home); + jail.create_dir(filedir)?; + jail.create_file( + filename, + r#" +chains: + - name: "hardhat" + rpc_url: "ws://test-endpoint:${TEST_RPC_URL_PORT}" + rpc_auth: + type: "Basic" + credentials: + username: "${TEST_USERNAME}" + password: "${TEST_PASSWORD}" + contracts: + enclave: "${TEST_CONTRACT_ADDRESS}" + ciphernode_registry: + address: "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" + deploy_block: 1764352873645 + filter_registry: "0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9" +"#, + )?; + + let config: AppConfig = load_config(None).map_err(|err| err.to_string())?; + let chain = config.chains().first().unwrap(); + + // Test that environment variables are properly substituted + assert_eq!(chain.rpc_url, "ws://test-endpoint:8545"); + assert_eq!( + chain.rpc_auth, + RpcAuth::Basic { + username: "envUser".to_string(), + password: "envPassword".to_string(), + } + ); + assert_eq!( + chain.contracts.enclave.address(), + "0x1234567890123456789012345678901234567890" + ); + + Ok(()) + }); + } } diff --git a/packages/ciphernode/config/src/lib.rs b/packages/ciphernode/config/src/lib.rs index ba182a1b..83e06ce9 100644 --- a/packages/ciphernode/config/src/lib.rs +++ b/packages/ciphernode/config/src/lib.rs @@ -1,2 +1,3 @@ mod app_config; +mod yaml; pub use app_config::*; diff --git a/packages/ciphernode/config/src/yaml.rs b/packages/ciphernode/config/src/yaml.rs new file mode 100644 index 00000000..92da78e1 --- /dev/null +++ b/packages/ciphernode/config/src/yaml.rs @@ -0,0 +1,51 @@ +use anyhow::Result; +use std::{fs, path::PathBuf}; + +pub fn load_yaml_with_env(file_path: &PathBuf) -> Result { + // Read the file content to string + let content = match fs::read_to_string(file_path) { + Ok(val) => val, + Err(_) => "".to_string(), + }; + + // Collect environment variables and perform substitution + Ok(shellexpand::env(&content)?.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::env; + use std::fs::File; + use std::io::Write; + use tempfile::tempdir; + + #[test] + fn test_yaml_env_substitution() -> Result<()> { + // Create a temporary directory and file + let dir = tempdir()?; + let file_path = dir.path().join("test.yaml"); + let mut file = File::create(&file_path)?; + + // Write test YAML content + writeln!( + file, + "database:\n url: $MY_DATABASE_URL\n password: ${{MY_DB_PASSWORD}}" + )?; + + // Set environment variables + env::set_var("MY_DATABASE_URL", "postgres://localhost:5432"); + env::set_var("MY_DB_PASSWORD", "secret123"); + + // Test the function + let processed = load_yaml_with_env(&file_path)?; + + env::remove_var("MY_DATABASE_URL"); + env::remove_var("MY_DB_PASSWORD"); + + assert!(processed.contains("postgres://localhost:5432")); + assert!(processed.contains("secret123")); + + Ok(()) + } +}