diff --git a/.gitignore b/.gitignore index 81aa761..6577bd0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ /*.csv /.vscode .DS_Store -/*.sh \ No newline at end of file +/*.sh +/*.json \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 4d06b38..4cec145 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,9 +82,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "autocfg" @@ -307,47 +307,88 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", + "futures-sink", ] [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-macro" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", + "futures-sink", "futures-task", "memchr", "pin-project-lite", @@ -412,20 +453,32 @@ checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "honey-health" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", - "chrono", "clap", "colored", "dotenv", "glob", + "honeycomb-client", + "serde", + "serde_yaml", + "strsim", + "tokio", +] + +[[package]] +name = "honeycomb-client" +version = "0.1.1" +source = "git+https://github.com/jerbly/honeycomb-client?tag=v0.1.1#ea0c432e93cf7cb7cc12bf358a2bbdb03029604f" +dependencies = [ + "anyhow", + "chrono", + "futures", "openssl", "reqwest", "serde", "serde_json", - "serde_yaml", - "strsim", ] [[package]] @@ -602,6 +655,16 @@ version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.20" @@ -726,9 +789,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.1.6+3.1.4" +version = "300.2.1+3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085" +checksum = "3fe476c29791a5ca0d1273c697e96085bbabbbea2ef7afd5617e78a4b40332d3" dependencies = [ "cc", ] @@ -746,6 +809,29 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + [[package]] name = "percent-encoding" version = "2.3.0" @@ -772,18 +858,18 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -799,9 +885,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ "base64", "bytes", @@ -869,6 +955,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "2.9.2" @@ -894,18 +986,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.192" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.192" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", @@ -914,9 +1006,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -948,6 +1040,15 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + [[package]] name = "slab" version = "0.4.9" @@ -957,6 +1058,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallvec" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" + [[package]] name = "socket2" version = "0.4.10" @@ -985,9 +1092,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "syn" -version = "2.0.39" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -1045,20 +1152,34 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.34.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", "libc", "mio", "num_cpus", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2 0.5.5", + "tokio-macros", "windows-sys", ] +[[package]] +name = "tokio-macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-native-tls" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 2659125..6466ac2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "honey-health" -version = "0.3.2" +version = "0.3.3" edition = "2021" authors = ["Jeremy Blythe "] @@ -8,14 +8,12 @@ authors = ["Jeremy Blythe "] [dependencies] anyhow = "1.0.72" -chrono = { version = "0.4.26", features = ["serde"] } clap = { version = "4.4.8", features = ["derive"] } colored = "2.0.4" dotenv = "0.15.0" glob = "0.3.1" -openssl = { version = "0.10.61", features = ["vendored"] } -reqwest = { version = "0.11.18", features = ["blocking", "json"] } +honeycomb-client = { git = "https://github.com/jerbly/honeycomb-client", tag = "v0.1.1" } serde = { version = "1.0.183", features = ["derive"] } -serde_json = "1.0.104" serde_yaml = "0.9.25" strsim = "0.10.0" +tokio = { version = "1.35.1", features = ["full"] } diff --git a/src/honeycomb.rs b/src/honeycomb.rs deleted file mode 100644 index bf85071..0000000 --- a/src/honeycomb.rs +++ /dev/null @@ -1,69 +0,0 @@ -use std::env; - -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; - -pub struct HoneyComb { - pub api_key: String, -} -const URL: &str = "https://api.honeycomb.io/1/"; -const HONEYCOMB_API_KEY: &str = "HONEYCOMB_API_KEY"; - -#[derive(Debug, Deserialize)] -pub struct Dataset { - pub slug: String, - pub last_written_at: Option>, -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct Column { - pub id: String, - pub key_name: String, - pub r#type: String, - pub description: String, - pub hidden: bool, - pub last_written: DateTime, -} - -impl HoneyComb { - pub fn new() -> Self { - Self { - api_key: env::var(HONEYCOMB_API_KEY) - .unwrap_or_else(|_| panic!("Environment variable {} not found", HONEYCOMB_API_KEY)), - } - } - pub fn list_all_datasets(&self) -> anyhow::Result> { - let client = reqwest::blocking::Client::new(); - let response = client - .get(format!("{}datasets", URL)) - .header("X-Honeycomb-Team", &self.api_key) - .send()?; - - let text = response.text()?; - - match serde_json::from_str::>(&text) { - Ok(datasets) => Ok(datasets), - Err(_) => { - println!("Invalid JSON data: {}", text); - Err(anyhow::anyhow!("Invalid JSON data")) - } - } - } - pub fn list_all_columns(&self, dataset_slug: &str) -> anyhow::Result> { - let client = reqwest::blocking::Client::new(); - let response = client - .get(format!("{}columns/{}", URL, dataset_slug)) - .header("X-Honeycomb-Team", &self.api_key) - .send()?; - - let text = response.text()?; - - match serde_json::from_str::>(&text) { - Ok(columns) => Ok(columns), - Err(_) => { - println!("Invalid JSON data: {}", text); - Err(anyhow::anyhow!("Invalid JSON data")) - } - } - } -} diff --git a/src/main.rs b/src/main.rs index f7b2285..968caf9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,3 @@ -mod honeycomb; mod semconv; use std::{ @@ -9,10 +8,9 @@ use std::{ }; use anyhow::{Context, Ok}; -use chrono::Utc; use clap::Parser; use colored::Colorize; -use honeycomb::{Column, HoneyComb}; +use honeycomb_client::honeycomb::Column; use semconv::{SemanticConventions, Suggestion}; // For each dataset get all the columns and put them in a map of column_name -> ColumnUsage @@ -81,7 +79,7 @@ struct ColumnUsageMap { } impl ColumnUsageMap { - fn new( + async fn new( root_dirs: &[String], include_datasets: Option>, max_last_written_days: usize, @@ -94,65 +92,51 @@ impl ColumnUsageMap { dataset_health: vec![], semconv: sc, }; - let hc = HoneyComb::new(); - let now = Utc::now(); - let inc_datasets = match include_datasets { - Some(d) => d, - None => HashSet::new(), + let hc = match honeycomb_client::get_honeycomb(&["columns", "createDatasets"]).await? { + Some(hclient) => hclient, + None => { + anyhow::bail!("API key does not have required access"); + } }; - let mut datasets = hc - .list_all_datasets()? - .iter() - .filter_map(|d| { - if (now - d.last_written_at.unwrap_or(Utc::now())).num_days() - < max_last_written_days as i64 - { - if inc_datasets.is_empty() || inc_datasets.contains(&d.slug) { - Some(d.slug.clone()) - } else { - None - } - } else { - None - } - }) - .collect::>(); - datasets.sort(); - cm.datasets = datasets; + + let dataset_slugs = hc + .get_dataset_slugs(max_last_written_days as i64, include_datasets) + .await?; + + cm.datasets = dataset_slugs; eprint!("Reading {} datasets ", cm.datasets.len()); - for (dataset_num, dataset_slug) in cm.datasets.iter().enumerate() { - //println!("Reading dataset: {}", dataset_slug); + let mut dataset_num = 0; + hc.process_datasets_columns(max_last_written_days as i64, &cm.datasets, |_, columns| { eprint!("."); - let columns = hc.list_all_columns(dataset_slug)?; let mut dataset_health = DatasetHealth::new(); for column in columns { - let duration = now - column.last_written; - if duration.num_days() < max_last_written_days as i64 { - let health: Suggestion; - if let Some(cu) = cm.map.get_mut(&column.key_name) { - cu.datasets[dataset_num] = true; - health = cu.suggestion.clone(); - } else { - let key_name = column.key_name.clone(); - let suggestion = cm.semconv.get_suggestion(&key_name); - let cu = ColumnUsage::new( - column, - suggestion.clone(), - cm.datasets.len(), - dataset_num, - ); - cm.map.insert(key_name, cu); - health = suggestion; - } - match health { - Suggestion::Matching => dataset_health.matching += 1, - Suggestion::Missing(_) => dataset_health.missing += 1, - _ => dataset_health.bad += 1, - } + let health: Suggestion; + if let Some(cu) = cm.map.get_mut(&column.key_name) { + cu.datasets[dataset_num] = true; + health = cu.suggestion.clone(); + } else { + let key_name = column.key_name.clone(); + let suggestion = cm.semconv.get_suggestion(&key_name); + let cu = ColumnUsage::new( + column, + suggestion.clone(), + cm.datasets.len(), + dataset_num, + ); + cm.map.insert(key_name, cu); + health = suggestion; + } + match health { + Suggestion::Matching => dataset_health.matching += 1, + Suggestion::Missing(_) => dataset_health.missing += 1, + _ => dataset_health.bad += 1, } } cm.dataset_health.push(dataset_health); - } + dataset_num += 1; + }) + .await?; + eprintln!(); Ok(cm) } @@ -302,7 +286,8 @@ struct Args { last_written_days: usize, } -fn main() -> anyhow::Result<()> { +#[tokio::main] +async fn main() -> anyhow::Result<()> { dotenv::dotenv().ok(); let args = Args::parse(); let mut root_dirs = vec![]; @@ -319,7 +304,7 @@ fn main() -> anyhow::Result<()> { ); } let include_datasets = args.dataset.map(HashSet::from_iter); - let cm = ColumnUsageMap::new(&root_dirs, include_datasets, args.last_written_days)?; + let cm = ColumnUsageMap::new(&root_dirs, include_datasets, args.last_written_days).await?; if cm.datasets.is_empty() { println!("No datasets found"); return Ok(());