From 420b4c0579d0c1bcfe1f3c9137cdc38fa254d905 Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Wed, 5 Apr 2023 13:48:43 -0700 Subject: [PATCH 01/30] chore: updating signature for linux build --- VERSION.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.json b/VERSION.json index 4214a983c..f09b4b5cb 100644 --- a/VERSION.json +++ b/VERSION.json @@ -12,7 +12,7 @@ "url": "https://github.com/spyglass-search/spyglass/releases/download/v2023.4.1/Spyglass_universal.app.tar.gz" }, "linux-x86_64": { - "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYmdRb044enZ3SEFWaDdsQzhVYU52RDIzRmFVK1g5TGxGRW83VTZySFAvWjBGMW9ZMkRIeXhldEhWTnF3cGQzRnpINkJGaWxBdTZvdEkwUHlTUGJkUkFNPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjgwNjU1MTQ2CWZpbGU6c3B5Z2xhc3NfMjMuNC4xX2FtZDY0LkFwcEltYWdlLnRhci5negpJemd2d0VoZEJMUDJRKzVPeHJFalJjZUhoRzdHbUdvd21aVEpGSXBWRjI0ZURyZW9YK3VTVW1oVmJvMGdkWmpTdjBZWGZ4amhlbC9kU1ZoUzJSVmhCZz09Cg==", + "signature": "dW50cnVzdGVkIGNvbW1lbnQ6IHNpZ25hdHVyZSBmcm9tIHRhdXJpIHNlY3JldCBrZXkKUlVSRk9WYXdDUXphYnM2VDRla2k2VnBVY29zUlZ1b0k4ZkhWVGU1Q2tmMEpIQ0tFQ3RxUGEyRkFBZ3RtaUxLUGpqTThrcENaYnhEZE5OenorZHA3N2srb2QyZ0ZhTHhhZFFnPQp0cnVzdGVkIGNvbW1lbnQ6IHRpbWVzdGFtcDoxNjgwNzI1NjA3CWZpbGU6c3B5Z2xhc3NfMjMuNC4xX2FtZDY0LkFwcEltYWdlLnRhci5negp1MDZ2K3o3T2RNeUhpRHcyVXF6dnNxemo4dWZQMDNpT2R6TnhGOGdUSk5LNytWY0RSSzRNSThyVkV3VmphWGM4UlNlMm1mRnJGY1hhOGs5SW55MXlDUT09Cg==", "url": "https://github.com/spyglass-search/spyglass/releases/download/v2023.4.1/spyglass_23.4.1_amd64.AppImage.tar.gz" }, "windows-x86_64": { From 50f79c4f4bbfb16953083697c853d5ad4b252d5d Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Thu, 6 Apr 2023 14:47:08 -0700 Subject: [PATCH 02/30] setup a default for `restart_required` for older plugin configs (#428) --- crates/shared/src/config.rs | 2 +- crates/shared/src/form.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/shared/src/config.rs b/crates/shared/src/config.rs index 07fc9c872..03f87cd63 100644 --- a/crates/shared/src/config.rs +++ b/crates/shared/src/config.rs @@ -368,7 +368,7 @@ impl Config { settings.insert(plugin_config.name.clone(), config.clone()); } - Err(error) => log::error!("Error loading plugin config {:?}", error), + Err(error) => log::warn!("Error loading plugin config {:?}", error), } } } diff --git a/crates/shared/src/form.rs b/crates/shared/src/form.rs index 4faad7eca..faa5b7eee 100644 --- a/crates/shared/src/form.rs +++ b/crates/shared/src/form.rs @@ -111,6 +111,7 @@ pub struct SettingOpts { pub label: String, pub value: String, pub form_type: FormType, - pub restart_required: bool, pub help_text: Option, + #[serde(default)] + pub restart_required: bool, } From 9a5e7a0ed43c55588b9b2c4a9529b5443c20df4c Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Fri, 7 Apr 2023 16:57:33 -0700 Subject: [PATCH 03/30] add clang to list of of linux deps (for whisper-rs compilation) & ignore (#429) any dev models --- Makefile | 3 ++- assets/.gitignore | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 assets/.gitignore diff --git a/Makefile b/Makefile index 503b583c3..b11cf7dbd 100644 --- a/Makefile +++ b/Makefile @@ -99,7 +99,8 @@ setup-dev-linux: libayatana-appindicator3-dev \ librsvg2-dev \ cmake \ - libsdl2-dev + libsdl2-dev \ + clang run-client-dev: cargo tauri dev diff --git a/assets/.gitignore b/assets/.gitignore new file mode 100644 index 000000000..604f0f2cf --- /dev/null +++ b/assets/.gitignore @@ -0,0 +1 @@ +models From 84e703ca653016b29958f8b4dfdfb61413cd7bf2 Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Mon, 10 Apr 2023 18:23:38 -0700 Subject: [PATCH 04/30] feature: load local lens archive (#430) * move debug binary main into crates/spyglass/binaries/debug * update debug fmt for ReadonlySearcher * cargo fmt * binaries -> bin to be more rusty * warn on invalid plugin manifest * suppress tantivy logging in debug binary * add option to keep/delete lens archive after processing * cargo fmt --- .gitignore | 2 +- crates/spyglass/Cargo.toml | 2 +- crates/spyglass/bin/debug/src/main.rs | 229 ++++++++++++++++++ crates/spyglass/src/debug/main.rs | 196 --------------- .../spyglass/src/pipeline/cache_pipeline.rs | 22 +- crates/spyglass/src/pipeline/mod.rs | 9 +- crates/spyglass/src/plugin/mod.rs | 2 +- crates/spyglass/src/search/mod.rs | 2 +- 8 files changed, 259 insertions(+), 205 deletions(-) create mode 100644 crates/spyglass/bin/debug/src/main.rs delete mode 100644 crates/spyglass/src/debug/main.rs diff --git a/.gitignore b/.gitignore index 52d14cee3..e7485a492 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,4 @@ target **/target node_modules dist -binaries \ No newline at end of file +binaries diff --git a/crates/spyglass/Cargo.toml b/crates/spyglass/Cargo.toml index b63ae2076..a0b4054fd 100644 --- a/crates/spyglass/Cargo.toml +++ b/crates/spyglass/Cargo.toml @@ -96,4 +96,4 @@ path = "src/main.rs" [[bin]] name = "spyglass-debug" -path = "src/debug/main.rs" \ No newline at end of file +path = "bin/debug/src/main.rs" \ No newline at end of file diff --git a/crates/spyglass/bin/debug/src/main.rs b/crates/spyglass/bin/debug/src/main.rs new file mode 100644 index 000000000..21ad4298d --- /dev/null +++ b/crates/spyglass/bin/debug/src/main.rs @@ -0,0 +1,229 @@ +use anyhow::anyhow; +use clap::{Parser, Subcommand}; +use entities::models::{self, indexed_document::DocumentIdentifier}; +use libspyglass::state::AppState; +use ron::ser::PrettyConfig; +use shared::config::Config; +use spyglass_plugin::DocumentQuery; +use std::{path::PathBuf, process::ExitCode}; +use tracing_log::LogTracer; +use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter}; + +use libspyglass::pipeline::cache_pipeline::process_update; +use libspyglass::search::{self, IndexPath, QueryStats, ReadonlySearcher, Searcher}; + +#[cfg(debug_assertions)] +const LOG_LEVEL: &str = "spyglassdebug=DEBUG"; +#[cfg(debug_assertions)] +const LIBSPYGLASS_LEVEL: &str = "libspyglass=DEBUG"; + +#[cfg(not(debug_assertions))] +const LOG_LEVEL: &str = "spyglassdebug=INFO"; +#[cfg(not(debug_assertions))] +const LIBSPYGLASS_LEVEL: &str = "libspyglass=INFO"; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +pub struct CdxCli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Outputs crawl details for a crawl ID + CrawlDetails { + crawl_task_id: i64, + }, + /// Outputs document metadata & content for a document ID + GetDocumentDetails { + id_or_url: String, + }, + GetDocumentQueryExplanation { + id_or_url: String, + query: String, + }, + /// Load a local lens archive into the index + LoadArchive { + name: String, + archive_path: PathBuf, + }, +} + +#[tokio::main] +async fn main() -> anyhow::Result { + let subscriber = tracing_subscriber::registry() + .with( + EnvFilter::from_default_env() + .add_directive(LOG_LEVEL.parse().expect("Invalid log filter")) + .add_directive("tantivy=WARN".parse().expect("Invalid EnvFilter")) + .add_directive(LIBSPYGLASS_LEVEL.parse().expect("invalid log filter")), + ) + .with(fmt::Layer::new().with_writer(std::io::stdout)); + tracing::subscriber::set_global_default(subscriber).expect("Unable to set a global subscriber"); + let _ = LogTracer::init(); + + let cli = CdxCli::parse(); + let config = Config::new(); + + match cli.command { + Command::CrawlDetails { crawl_task_id } => { + let db = models::create_connection(&config, false).await?; + let num_progress = models::crawl_queue::num_tasks_in_progress(&db) + .await + .unwrap_or_default(); + let task_details = models::crawl_queue::get_task_details(crawl_task_id, &db).await; + + println!("## Task Details ##"); + println!("Task Processing: {}", num_progress); + match task_details { + Ok(Some((task, tags))) => { + println!( + "Crawl Task: {}", + ron::ser::to_string_pretty(&task, PrettyConfig::new()).unwrap_or_default() + ); + println!( + "Tags: {}", + ron::ser::to_string_pretty(&tags, PrettyConfig::new()).unwrap_or_default() + ); + } + Ok(None) => { + println!("No task found for id {}", crawl_task_id); + } + Err(err) => { + println!("Error accessing task details {:?}", err); + } + } + } + Command::GetDocumentDetails { id_or_url } => { + let db = models::create_connection(&config, false).await?; + + let identifier = if id_or_url.contains("://") { + DocumentIdentifier::Url(&id_or_url) + } else { + DocumentIdentifier::DocId(&id_or_url) + }; + + let doc_details = + models::indexed_document::get_document_details(&db, identifier).await?; + + println!("## Document Details ##"); + match doc_details { + Some((doc, tags)) => { + println!( + "Document: {}", + ron::ser::to_string_pretty(&doc, PrettyConfig::new()).unwrap_or_default() + ); + println!( + "Tags: {}", + ron::ser::to_string_pretty(&tags, PrettyConfig::new()).unwrap_or_default() + ); + let index = + ReadonlySearcher::with_index(&IndexPath::LocalPath(config.index_dir())) + .expect("Unable to open index."); + + let docs = ReadonlySearcher::search_by_query( + &db, + &index, + &DocumentQuery { + urls: Some(vec![doc.url.clone()]), + ..Default::default() + }, + ) + .await; + println!("### Indexed Document ###"); + if docs.is_empty() { + println!("No indexed document for url {:?}", &doc.url); + } else { + for (_score, doc_addr) in docs { + if let Ok(Ok(doc)) = index + .reader + .searcher() + .doc(doc_addr) + .map(|doc| search::document_to_struct(&doc)) + { + println!( + "Indexed Document: {}", + ron::ser::to_string_pretty(&doc, PrettyConfig::new()) + .unwrap_or_default() + ); + } else { + println!("Error accessing Doc at address {:?}", doc_addr); + } + } + } + } + None => println!("No document found for identifier: {}", id_or_url), + } + } + Command::GetDocumentQueryExplanation { id_or_url, query } => { + let db = models::create_connection(&config, false).await?; + + let doc_query = if id_or_url.contains("://") { + DocumentQuery { + urls: Some(vec![id_or_url.clone()]), + ..Default::default() + } + } else { + DocumentQuery { + ids: Some(vec![id_or_url.clone()]), + ..Default::default() + } + }; + + let index = ReadonlySearcher::with_index(&IndexPath::LocalPath(config.index_dir())) + .expect("Unable to open index."); + + let docs = ReadonlySearcher::search_by_query(&db, &index, &doc_query).await; + + if docs.is_empty() { + println!("No indexed document for url {:?}", id_or_url); + } else { + for (_score, doc_addr) in docs { + let mut stats = QueryStats::default(); + let explain = ReadonlySearcher::explain_search_with_lens( + &db, + doc_addr, + &vec![], + &index, + query.as_str(), + &mut stats, + ) + .await; + match explain { + Some(explanation) => { + println!( + "Query \"{:?}\" for document {:?} \n {:?}", + query, id_or_url, explanation + ); + } + None => { + println!("Could not get score for document"); + } + } + } + } + } + Command::LoadArchive { name, archive_path } => { + if !archive_path.exists() { + eprintln!("{} does not exist!", archive_path.display()); + return Err(anyhow!("ARCHIVE_PATH does not exist")); + } + + let config = Config::new(); + let state = AppState::new(&config).await; + + let lens = shared::config::LensConfig { + author: "spyglass-search".into(), + name: name.clone(), + label: name, + ..Default::default() + }; + + process_update(state.clone(), &lens, archive_path, true).await; + let _ = Searcher::save(&state).await; + } + } + + Ok(ExitCode::SUCCESS) +} diff --git a/crates/spyglass/src/debug/main.rs b/crates/spyglass/src/debug/main.rs deleted file mode 100644 index 6a613e758..000000000 --- a/crates/spyglass/src/debug/main.rs +++ /dev/null @@ -1,196 +0,0 @@ -use clap::{Parser, Subcommand}; -use entities::models::{self, indexed_document::DocumentIdentifier}; -use libspyglass::search::{self, IndexPath, QueryStats, ReadonlySearcher}; -use ron::ser::PrettyConfig; -use shared::config::Config; -use spyglass_plugin::DocumentQuery; -use std::process::ExitCode; -use tracing_log::LogTracer; -use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter}; - -const LOG_LEVEL: tracing::Level = tracing::Level::INFO; - -#[cfg(debug_assertions)] -const LIB_LOG_LEVEL: &str = "spyglassdebug=DEBUG"; - -#[cfg(not(debug_assertions))] -const LIB_LOG_LEVEL: &str = "spyglassdebug=INFO"; - -#[derive(Parser)] -#[command(author, version, about, long_about = None)] -pub struct CdxCli { - #[command(subcommand)] - command: Option, -} - -#[derive(Subcommand)] -enum Command { - CrawlDetails { crawl_task_id: i64 }, - GetDocumentDetails { id_or_url: String }, - GetDocumentQueryExplanation { id_or_url: String, query: String }, -} - -#[tokio::main] -async fn main() -> anyhow::Result { - let subscriber = tracing_subscriber::registry() - .with( - EnvFilter::from_default_env() - .add_directive(LOG_LEVEL.into()) - .add_directive(LIB_LOG_LEVEL.parse().expect("invalid log filter")), - ) - .with(fmt::Layer::new().with_writer(std::io::stdout)); - tracing::subscriber::set_global_default(subscriber).expect("Unable to set a global subscriber"); - let _ = LogTracer::init(); - - let cli = CdxCli::parse(); - let config = Config::new(); - - if let Some(command) = cli.command { - match command { - Command::CrawlDetails { crawl_task_id } => { - let db = models::create_connection(&config, false).await?; - let num_progress = models::crawl_queue::num_tasks_in_progress(&db) - .await - .unwrap_or_default(); - let task_details = models::crawl_queue::get_task_details(crawl_task_id, &db).await; - - println!("## Task Details ##"); - println!("Task Processing: {}", num_progress); - match task_details { - Ok(Some((task, tags))) => { - println!( - "Crawl Task: {}", - ron::ser::to_string_pretty(&task, PrettyConfig::new()) - .unwrap_or_default() - ); - println!( - "Tags: {}", - ron::ser::to_string_pretty(&tags, PrettyConfig::new()) - .unwrap_or_default() - ); - } - Ok(None) => { - println!("No task found for id {}", crawl_task_id); - } - Err(err) => { - println!("Error accessing task details {:?}", err); - } - } - } - Command::GetDocumentDetails { id_or_url } => { - let db = models::create_connection(&config, false).await?; - - let identifier = if id_or_url.contains("://") { - DocumentIdentifier::Url(&id_or_url) - } else { - DocumentIdentifier::DocId(&id_or_url) - }; - - let doc_details = - models::indexed_document::get_document_details(&db, identifier).await?; - - println!("## Document Details ##"); - match doc_details { - Some((doc, tags)) => { - println!( - "Document: {}", - ron::ser::to_string_pretty(&doc, PrettyConfig::new()) - .unwrap_or_default() - ); - println!( - "Tags: {}", - ron::ser::to_string_pretty(&tags, PrettyConfig::new()) - .unwrap_or_default() - ); - let index = - ReadonlySearcher::with_index(&IndexPath::LocalPath(config.index_dir())) - .expect("Unable to open index."); - - let docs = ReadonlySearcher::search_by_query( - &db, - &index, - &DocumentQuery { - urls: Some(vec![doc.url.clone()]), - ..Default::default() - }, - ) - .await; - println!("### Indexed Document ###"); - if docs.is_empty() { - println!("No indexed document for url {:?}", &doc.url); - } else { - for (_score, doc_addr) in docs { - if let Ok(Ok(doc)) = index - .reader - .searcher() - .doc(doc_addr) - .map(|doc| search::document_to_struct(&doc)) - { - println!( - "Indexed Document: {}", - ron::ser::to_string_pretty(&doc, PrettyConfig::new()) - .unwrap_or_default() - ); - } else { - println!("Error accessing Doc at address {:?}", doc_addr); - } - } - } - } - None => println!("No document found for identifier: {}", id_or_url), - } - } - Command::GetDocumentQueryExplanation { id_or_url, query } => { - let db = models::create_connection(&config, false).await?; - - let doc_query = if id_or_url.contains("://") { - DocumentQuery { - urls: Some(vec![id_or_url.clone()]), - ..Default::default() - } - } else { - DocumentQuery { - ids: Some(vec![id_or_url.clone()]), - ..Default::default() - } - }; - - let index = ReadonlySearcher::with_index(&IndexPath::LocalPath(config.index_dir())) - .expect("Unable to open index."); - - let docs = ReadonlySearcher::search_by_query(&db, &index, &doc_query).await; - - if docs.is_empty() { - println!("No indexed document for url {:?}", id_or_url); - } else { - for (_score, doc_addr) in docs { - let mut stats = QueryStats::default(); - let explain = ReadonlySearcher::explain_search_with_lens( - &db, - doc_addr, - &vec![], - &index, - query.as_str(), - &mut stats, - ) - .await; - match explain { - Some(explanation) => { - println!( - "Query \"{:?}\" for document {:?} \n {:?}", - query, id_or_url, explanation - ); - } - None => { - println!("Could not get score for document"); - } - } - } - } - } - } - Ok(ExitCode::SUCCESS) - } else { - Ok(ExitCode::FAILURE) - } -} diff --git a/crates/spyglass/src/pipeline/cache_pipeline.rs b/crates/spyglass/src/pipeline/cache_pipeline.rs index c1fe97c29..83b0f0738 100644 --- a/crates/spyglass/src/pipeline/cache_pipeline.rs +++ b/crates/spyglass/src/pipeline/cache_pipeline.rs @@ -79,12 +79,21 @@ pub async fn process_update_warc(state: AppState, cache_path: PathBuf) { /// processes the cache for a lens. The cache is streamed in from the provided path /// and processed. After the process is complete the cache is deleted -pub async fn process_update(state: AppState, lens: &LensConfig, cache_path: PathBuf) { +pub async fn process_update( + state: AppState, + lens: &LensConfig, + cache_path: PathBuf, + keep_archive: bool, +) { let now = Instant::now(); + let mut total_processed = 0; + let records = archive::read_parsed(&cache_path); if let Ok(mut record_iter) = records { let mut record_list: Vec = Vec::new(); for record in record_iter.by_ref() { + total_processed += 1; + record_list.push(record); if record_list.len() >= 5000 { if let Err(err) = documents::process_records(&state, lens, &mut record_list).await { @@ -102,8 +111,15 @@ pub async fn process_update(state: AppState, lens: &LensConfig, cache_path: Path } // attempt to remove processed cache file - let _ = cache::delete_cache(&cache_path); - log::debug!("Processing Cache Took: {:?}", now.elapsed().as_millis()); + if !keep_archive { + let _ = cache::delete_cache(&cache_path); + } + + log::debug!( + "Processed {} records in {:?}ms", + total_processed, + now.elapsed().as_millis() + ); state .publish_event(&spyglass_rpc::RpcEvent { event_type: spyglass_rpc::RpcEventType::LensInstalled, diff --git a/crates/spyglass/src/pipeline/mod.rs b/crates/spyglass/src/pipeline/mod.rs index f8586d9e9..b5603f16e 100644 --- a/crates/spyglass/src/pipeline/mod.rs +++ b/crates/spyglass/src/pipeline/mod.rs @@ -129,8 +129,13 @@ pub async fn initialize_pipelines( } PipelineCommand::ProcessCache(lens, cache_file) => { if let Some(lens_config) = app_state.lenses.get(&lens) { - cache_pipeline::process_update(app_state.clone(), &lens_config, cache_file) - .await; + cache_pipeline::process_update( + app_state.clone(), + &lens_config, + cache_file, + false, + ) + .await; } } } diff --git a/crates/spyglass/src/plugin/mod.rs b/crates/spyglass/src/plugin/mod.rs index 79a80ef35..b9ff349aa 100644 --- a/crates/spyglass/src/plugin/mod.rs +++ b/crates/spyglass/src/plugin/mod.rs @@ -258,7 +258,7 @@ pub async fn plugin_event_loop( }, ); } - Err(e) => log::error!("Unable to init plugin <{}>: {}", plugin.name, e), + Err(e) => log::warn!("Unable to init plugin <{}>: {}", plugin.name, e), } } Some(PluginCommand::QueueIntervalCheck) => { diff --git a/crates/spyglass/src/search/mod.rs b/crates/spyglass/src/search/mod.rs index 8af1714a0..67a0d7760 100644 --- a/crates/spyglass/src/search/mod.rs +++ b/crates/spyglass/src/search/mod.rs @@ -72,7 +72,7 @@ impl Debug for Searcher { impl Debug for ReadonlySearcher { fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { - f.debug_struct("Searcher") + f.debug_struct("ReadonlySearcher") .field("index", &self.index) .finish() } From ae7ddf421181bc67bce07a1e0b8c919dc0aaf108 Mon Sep 17 00:00:00 2001 From: travolin Date: Thu, 13 Apr 2023 16:03:12 -0700 Subject: [PATCH 05/30] Add support to use postgres database or sqlite database (#431) --------- Co-authored-by: Joel Bredeson --- crates/entities/Cargo.toml | 8 +- crates/entities/src/lib.rs | 4 +- crates/entities/src/models/bootstrap_queue.rs | 24 +++ crates/entities/src/models/connection.rs | 24 +++ crates/entities/src/models/crawl_queue.rs | 45 +++- crates/entities/src/models/crawl_tag.rs | 24 +++ crates/entities/src/models/document_tag.rs | 24 +++ crates/entities/src/models/fetch_history.rs | 24 +++ .../entities/src/models/indexed_document.rs | 35 +++- crates/entities/src/models/lens.rs | 45 ++++ crates/entities/src/models/link.rs | 24 +++ crates/entities/src/models/mod.rs | 29 ++- crates/entities/src/models/processed_files.rs | 28 ++- crates/entities/src/models/resource_rule.rs | 24 +++ crates/entities/src/models/tag.rs | 28 ++- .../src/m20220505_000001_create_table.rs | 197 ++++++++++++------ ...0508_000001_lens_and_crawl_queue_update.rs | 50 +++-- .../m20220522_000001_bootstrap_queue_table.rs | 41 ++-- .../src/m20221023_000001_connection_table.rs | 61 ++++-- ...221107_000001_recreate_connection_table.rs | 66 ++++-- .../src/m20221109_add_tags_table.rs | 90 +++++--- ...221116_000001_add_connection_constraint.rs | 2 +- ...1123_000001_add_document_tag_constraint.rs | 2 +- ...124_000001_add_tags_for_existing_lenses.rs | 10 +- .../m20221210_000001_add_crawl_tags_table.rs | 58 ++++-- .../m20230104_000001_add_column_n_index.rs | 6 +- .../m20230112_000001_migrate_search_schema.rs | 2 +- .../src/m20230126_000001_create_file_table.rs | 46 ++-- .../src/m20230201_000001_add_tag_index.rs | 2 +- ...30203_000001_add_indexed_document_index.rs | 2 +- crates/spyglass/src/search/mod.rs | 60 ++++++ 31 files changed, 840 insertions(+), 245 deletions(-) diff --git a/crates/entities/Cargo.toml b/crates/entities/Cargo.toml index a677b6dc0..ddbe71823 100644 --- a/crates/entities/Cargo.toml +++ b/crates/entities/Cargo.toml @@ -10,7 +10,7 @@ anyhow = "1.0" chrono = { version = "0.4", features = ["serde"] } log = "0.4" regex = "1" -sea-orm = { version = "0.11", features = ["macros", "sqlx-sqlite", "runtime-tokio-rustls", "with-chrono", "with-json"], default-features = false } +sea-orm = { version = "0.11", features = ["macros", "sqlx-sqlite", "sqlx-postgres", "runtime-tokio-rustls", "with-chrono", "with-json"], default-features = false } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" shared = { path = "../shared" } @@ -23,4 +23,8 @@ tokio = { version = "1", features = ["full"] } url = "2.2" [dev-dependencies] -ron = "0.8" \ No newline at end of file +ron = "0.8" + +[lib] +name = "entities" +path = "src/lib.rs" \ No newline at end of file diff --git a/crates/entities/src/lib.rs b/crates/entities/src/lib.rs index 1cb86dade..fe589643b 100644 --- a/crates/entities/src/lib.rs +++ b/crates/entities/src/lib.rs @@ -4,7 +4,7 @@ pub mod schema; pub mod test; pub use sea_orm; -use sea_orm::{DatabaseConnection, DbBackend, DbErr, FromQueryResult, Statement}; +use sea_orm::{ConnectionTrait, DatabaseConnection, DbErr, FromQueryResult, Statement}; use shared::response::LibraryStats; pub const BATCH_SIZE: usize = 3000; @@ -20,7 +20,7 @@ pub async fn get_library_stats( db: &DatabaseConnection, ) -> Result, DbErr> { let counts = CountByStatus::find_by_statement(Statement::from_string( - DbBackend::Sqlite, + db.get_database_backend(), r#" SELECT count(*) as "count", tags.value as "name", status diff --git a/crates/entities/src/models/bootstrap_queue.rs b/crates/entities/src/models/bootstrap_queue.rs index 8813053e0..e684237b6 100644 --- a/crates/entities/src/models/bootstrap_queue.rs +++ b/crates/entities/src/models/bootstrap_queue.rs @@ -91,3 +91,27 @@ pub async fn dequeue( Ok(()) } + +// Helper method to copy the table from one database to another +pub async fn copy_table( + from: &DatabaseConnection, + to: &DatabaseConnection, +) -> anyhow::Result<(), sea_orm::DbErr> { + let mut pages = Entity::find().paginate(from, 1000); + Entity::delete_many().exec(to).await?; + while let Ok(Some(pages)) = pages.fetch_and_next().await { + let active_model = pages + .into_iter() + .map(|model| model.into()) + .collect::>(); + Entity::insert_many(active_model) + .on_conflict( + sea_orm::sea_query::OnConflict::columns(vec![Column::Id]) + .do_nothing() + .to_owned(), + ) + .exec(to) + .await?; + } + Ok(()) +} diff --git a/crates/entities/src/models/connection.rs b/crates/entities/src/models/connection.rs index 501a4df57..2a949a912 100644 --- a/crates/entities/src/models/connection.rs +++ b/crates/entities/src/models/connection.rs @@ -186,6 +186,30 @@ pub async fn set_sync_status( Ok(()) } +// Helper method to copy the table from one database to another +pub async fn copy_table( + from: &DatabaseConnection, + to: &DatabaseConnection, +) -> anyhow::Result<(), sea_orm::DbErr> { + let mut pages = Entity::find().paginate(from, 1000); + Entity::delete_many().exec(to).await?; + while let Ok(Some(pages)) = pages.fetch_and_next().await { + let active_model = pages + .into_iter() + .map(|model| model.into()) + .collect::>(); + Entity::insert_many(active_model) + .on_conflict( + sea_orm::sea_query::OnConflict::columns(vec![Column::Id]) + .do_nothing() + .to_owned(), + ) + .exec(to) + .await?; + } + Ok(()) +} + #[cfg(test)] mod test { use super::ActiveModel; diff --git a/crates/entities/src/models/crawl_queue.rs b/crates/entities/src/models/crawl_queue.rs index 2c6a42b39..d82a01116 100644 --- a/crates/entities/src/models/crawl_queue.rs +++ b/crates/entities/src/models/crawl_queue.rs @@ -2,8 +2,7 @@ use regex::{RegexSet, RegexSetBuilder}; use sea_orm::entity::prelude::*; use sea_orm::sea_query::{OnConflict, Query, SqliteQueryBuilder}; use sea_orm::{ - sea_query, ConnectionTrait, DatabaseBackend, DbBackend, FromQueryResult, InsertResult, - QueryTrait, Set, Statement, + sea_query, ConnectionTrait, FromQueryResult, InsertResult, QueryTrait, Set, Statement, }; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; @@ -215,9 +214,9 @@ pub async fn num_queued( Ok(res) } -fn gen_dequeue_sql(user_settings: &UserSettings) -> Statement { +fn gen_dequeue_sql(db: &DatabaseConnection, user_settings: &UserSettings) -> Statement { Statement::from_sql_and_values( - DbBackend::Sqlite, + db.get_database_backend(), include_str!("sql/dequeue.sqlx"), vec![ user_settings.domain_crawl_limit.value().into(), @@ -313,7 +312,7 @@ pub async fn dequeue( } else { // Otherwise, grab a URL off the stack & send it back. Entity::find() - .from_raw_sql(gen_dequeue_sql(user_settings)) + .from_raw_sql(gen_dequeue_sql(db, user_settings)) .one(db) .await? } @@ -647,7 +646,7 @@ pub async fn enqueue_all( .build(SqliteQueryBuilder); let values: Vec = values.iter().map(|x| x.to_owned()).collect(); - let statement = Statement::from_sql_and_values(DbBackend::Sqlite, &sql, values); + let statement = Statement::from_sql_and_values(db.get_database_backend(), &sql, values); if let Err(err) = db.execute(statement).await { log::warn!("insert_many error: {err}"); } else if !overrides.tags.is_empty() { @@ -920,7 +919,7 @@ pub async fn find_by_lens( name: &str, ) -> Result, sea_orm::DbErr> { CrawlTaskId::find_by_statement(Statement::from_sql_and_values( - DatabaseBackend::Sqlite, + db.get_database_backend(), r#" SELECT crawl_queue.id @@ -958,6 +957,30 @@ pub async fn get_task_details( Ok(None) } +// Helper method to copy the table from one database to another +pub async fn copy_table( + from: &DatabaseConnection, + to: &DatabaseConnection, +) -> anyhow::Result<(), sea_orm::DbErr> { + let mut pages = Entity::find().paginate(from, 1000); + Entity::delete_many().exec(to).await?; + while let Ok(Some(pages)) = pages.fetch_and_next().await { + let active_model = pages + .into_iter() + .map(|model| model.into()) + .collect::>(); + Entity::insert_many(active_model) + .on_conflict( + sea_orm::sea_query::OnConflict::columns(vec![Column::Id]) + .do_nothing() + .to_owned(), + ) + .exec(to) + .await?; + } + Ok(()) +} + #[cfg(test)] mod test { use sea_orm::prelude::*; @@ -997,10 +1020,12 @@ mod test { assert_eq!(res.url, url); } - #[test] - fn test_priority_sql() { + #[tokio::test] + async fn test_priority_sql() { + let db = setup_test_db().await; + let settings = UserSettings::default(); - let sql = gen_dequeue_sql(&settings); + let sql = gen_dequeue_sql(&db, &settings); assert_eq!( sql.to_string(), "WITH\nindexed AS (\n SELECT\n domain,\n count(*) as count\n FROM indexed_document\n GROUP BY domain\n),\ninflight AS (\n SELECT\n domain,\n count(*) as count\n FROM crawl_queue\n WHERE status = \"Processing\"\n GROUP BY domain\n)\nSELECT\n cq.*\nFROM crawl_queue cq\nLEFT JOIN indexed ON indexed.domain = cq.domain\nLEFT JOIN inflight ON inflight.domain = cq.domain\nWHERE\n COALESCE(indexed.count, 0) < 500000 AND\n COALESCE(inflight.count, 0) < 2 AND\n status = \"Queued\" and\n url not like \"file%\"\nORDER BY\n cq.updated_at ASC" diff --git a/crates/entities/src/models/crawl_tag.rs b/crates/entities/src/models/crawl_tag.rs index 67b75853c..701777547 100644 --- a/crates/entities/src/models/crawl_tag.rs +++ b/crates/entities/src/models/crawl_tag.rs @@ -51,3 +51,27 @@ impl ActiveModelBehavior for ActiveModel { Ok(self) } } + +// Helper method to copy the table from one database to another +pub async fn copy_table( + from: &DatabaseConnection, + to: &DatabaseConnection, +) -> anyhow::Result<(), sea_orm::DbErr> { + let mut pages = Entity::find().paginate(from, 1000); + Entity::delete_many().exec(to).await?; + while let Ok(Some(pages)) = pages.fetch_and_next().await { + let active_model = pages + .into_iter() + .map(|model| model.into()) + .collect::>(); + Entity::insert_many(active_model) + .on_conflict( + sea_orm::sea_query::OnConflict::columns(vec![Column::Id]) + .do_nothing() + .to_owned(), + ) + .exec(to) + .await?; + } + Ok(()) +} diff --git a/crates/entities/src/models/document_tag.rs b/crates/entities/src/models/document_tag.rs index 03212c396..a920d08c2 100644 --- a/crates/entities/src/models/document_tag.rs +++ b/crates/entities/src/models/document_tag.rs @@ -51,3 +51,27 @@ impl ActiveModelBehavior for ActiveModel { Ok(self) } } + +// Helper method to copy the table from one database to another +pub async fn copy_table( + from: &DatabaseConnection, + to: &DatabaseConnection, +) -> anyhow::Result<(), sea_orm::DbErr> { + let mut pages = Entity::find().paginate(from, 1000); + Entity::delete_many().exec(to).await?; + while let Ok(Some(pages)) = pages.fetch_and_next().await { + let active_model = pages + .into_iter() + .map(|model| model.into()) + .collect::>(); + Entity::insert_many(active_model) + .on_conflict( + sea_orm::sea_query::OnConflict::columns(vec![Column::Id]) + .do_nothing() + .to_owned(), + ) + .exec(to) + .await?; + } + Ok(()) +} diff --git a/crates/entities/src/models/fetch_history.rs b/crates/entities/src/models/fetch_history.rs index f64f508da..986cd0823 100644 --- a/crates/entities/src/models/fetch_history.rs +++ b/crates/entities/src/models/fetch_history.rs @@ -115,6 +115,30 @@ pub async fn upsert( } } +// Helper method to copy the table from one database to another +pub async fn copy_table( + from: &DatabaseConnection, + to: &DatabaseConnection, +) -> anyhow::Result<(), sea_orm::DbErr> { + let mut pages = Entity::find().paginate(from, 1000); + Entity::delete_many().exec(to).await?; + while let Ok(Some(pages)) = pages.fetch_and_next().await { + let active_model = pages + .into_iter() + .map(|model| model.into()) + .collect::>(); + Entity::insert_many(active_model) + .on_conflict( + sea_orm::sea_query::OnConflict::columns(vec![Column::Id]) + .do_nothing() + .to_owned(), + ) + .exec(to) + .await?; + } + Ok(()) +} + #[cfg(test)] mod test { use sea_orm::prelude::*; diff --git a/crates/entities/src/models/indexed_document.rs b/crates/entities/src/models/indexed_document.rs index 49d85418c..605c88ea6 100644 --- a/crates/entities/src/models/indexed_document.rs +++ b/crates/entities/src/models/indexed_document.rs @@ -6,8 +6,7 @@ use crate::BATCH_SIZE; use sea_orm::entity::prelude::*; use sea_orm::sea_query::OnConflict; use sea_orm::{ - ConnectionTrait, DatabaseBackend, FromQueryResult, InsertResult, QuerySelect, QueryTrait, Set, - Statement, + ConnectionTrait, FromQueryResult, InsertResult, QuerySelect, QueryTrait, Set, Statement, }; use serde::Serialize; @@ -244,7 +243,7 @@ pub async fn insert_tags_for_docs_by_id( .do_nothing() .to_owned(), ) - .build(DatabaseBackend::Sqlite); + .build(db.get_database_backend()); if let Err(err) = db.execute(query.clone()).await { log::error!("Unable to execute: {} due to {}", query.to_string(), err); @@ -372,7 +371,7 @@ pub async fn find_by_lens( name: &str, ) -> Result, sea_orm::DbErr> { IndexedDocumentId::find_by_statement(Statement::from_sql_and_values( - DatabaseBackend::Sqlite, + db.get_database_backend(), r#" SELECT indexed_document.id, @@ -400,7 +399,7 @@ pub async fn find_by_doc_ids( .join(","); IndexedDocumentId::find_by_statement(Statement::from_string( - DatabaseBackend::Sqlite, + db.get_database_backend(), format!( r#" SELECT @@ -428,7 +427,7 @@ pub async fn get_tag_ids_by_doc_id( id: &str, ) -> Result, sea_orm::DbErr> { IndexedDocumentTagId::find_by_statement(Statement::from_sql_and_values( - DatabaseBackend::Sqlite, + db.get_database_backend(), r#" SELECT document_tag.tag_id as id @@ -467,6 +466,30 @@ pub async fn get_document_details( Ok(None) } +// Helper method to copy the table from one database to another +pub async fn copy_table( + from: &DatabaseConnection, + to: &DatabaseConnection, +) -> anyhow::Result<(), sea_orm::DbErr> { + let mut pages = Entity::find().paginate(from, 1000); + Entity::delete_many().exec(to).await?; + while let Ok(Some(pages)) = pages.fetch_and_next().await { + let active_model = pages + .into_iter() + .map(|model| model.into()) + .collect::>(); + Entity::insert_many(active_model) + .on_conflict( + sea_orm::sea_query::OnConflict::columns(vec![Column::Id]) + .do_nothing() + .to_owned(), + ) + .exec(to) + .await?; + } + Ok(()) +} + #[cfg(test)] mod test { use std::collections::HashMap; diff --git a/crates/entities/src/models/lens.rs b/crates/entities/src/models/lens.rs index 29c40dd32..a0e38ae37 100644 --- a/crates/entities/src/models/lens.rs +++ b/crates/entities/src/models/lens.rs @@ -1,7 +1,9 @@ use sea_orm::entity::prelude::*; use sea_orm::sea_query; use sea_orm::DeleteResult; +use sea_orm::FromQueryResult; use sea_orm::Set; +use sea_orm::{ConnectionTrait, Statement}; use serde::Serialize; use shared::config::LensConfig; @@ -231,6 +233,49 @@ pub async fn install_or_update( Ok((true, new_db_entry)) } +/// Represents the tag id that is associated with a document +#[derive(Debug, FromQueryResult)] +pub struct LensName { + pub name: String, +} + +/// Helper method used to get all of the lens names +pub async fn get_lens_names(db: &C) -> Result, DbErr> +where + C: ConnectionTrait, +{ + LensName::find_by_statement(Statement::from_string( + db.get_database_backend(), + String::from(r#"SELECT name FROM lens"#), + )) + .all(db) + .await +} + +// Helper method to copy the table from one database to another +pub async fn copy_table( + from: &DatabaseConnection, + to: &DatabaseConnection, +) -> anyhow::Result<(), sea_orm::DbErr> { + let mut pages = Entity::find().paginate(from, 1000); + Entity::delete_many().exec(to).await?; + while let Ok(Some(pages)) = pages.fetch_and_next().await { + let active_model = pages + .into_iter() + .map(|model| model.into()) + .collect::>(); + Entity::insert_many(active_model) + .on_conflict( + sea_orm::sea_query::OnConflict::columns(vec![Column::Id]) + .do_nothing() + .to_owned(), + ) + .exec(to) + .await?; + } + Ok(()) +} + #[cfg(test)] mod test { use super::{add_or_enable, Entity}; diff --git a/crates/entities/src/models/link.rs b/crates/entities/src/models/link.rs index 4ccf91584..990659b99 100644 --- a/crates/entities/src/models/link.rs +++ b/crates/entities/src/models/link.rs @@ -50,3 +50,27 @@ pub async fn save_link( Ok(()) } + +// Helper method to copy the table from one database to another +pub async fn copy_table( + from: &DatabaseConnection, + to: &DatabaseConnection, +) -> anyhow::Result<(), sea_orm::DbErr> { + let mut pages = Entity::find().paginate(from, 1000); + Entity::delete_many().exec(to).await?; + while let Ok(Some(pages)) = pages.fetch_and_next().await { + let active_model = pages + .into_iter() + .map(|model| model.into()) + .collect::>(); + Entity::insert_many(active_model) + .on_conflict( + sea_orm::sea_query::OnConflict::columns(vec![Column::Id]) + .do_nothing() + .to_owned(), + ) + .exec(to) + .await?; + } + Ok(()) +} diff --git a/crates/entities/src/models/mod.rs b/crates/entities/src/models/mod.rs index ab7e0da6b..204245f50 100644 --- a/crates/entities/src/models/mod.rs +++ b/crates/entities/src/models/mod.rs @@ -16,6 +16,8 @@ pub mod tag; use shared::config::Config; +/// Creates a connection based on the passed in +/// configuration pub async fn create_connection( config: &Config, is_test: bool, @@ -33,9 +35,14 @@ pub async fn create_connection( ) }; + create_connection_by_uri(&db_uri).await +} + +/// Creates a connection based on the database uri +pub async fn create_connection_by_uri(db_uri: &str) -> anyhow::Result { // See https://www.sea-ql.org/SeaORM/docs/install-and-config/connection // for more connection options - let mut opt = ConnectOptions::new(db_uri); + let mut opt = ConnectOptions::new(db_uri.to_owned()); opt.max_connections(10) .min_connections(2) .sqlx_logging(false); @@ -43,6 +50,26 @@ pub async fn create_connection( Ok(Database::connect(opt).await?) } +// Helper method used to copy all tables from one database to another. +// Note that the destination database will have all content deleted. +pub async fn copy_all_tables( + from: &DatabaseConnection, + to: &DatabaseConnection, +) -> anyhow::Result<(), sea_orm::DbErr> { + bootstrap_queue::copy_table(from, to).await?; + connection::copy_table(from, to).await?; + crawl_queue::copy_table(from, to).await?; + document_tag::copy_table(from, to).await?; + fetch_history::copy_table(from, to).await?; + indexed_document::copy_table(from, to).await?; + lens::copy_table(from, to).await?; + link::copy_table(from, to).await?; + processed_files::copy_table(from, to).await?; + resource_rule::copy_table(from, to).await?; + tag::copy_table(from, to).await?; + Ok(()) +} + #[cfg(test)] mod test { use crate::models::create_connection; diff --git a/crates/entities/src/models/processed_files.rs b/crates/entities/src/models/processed_files.rs index 76bde18ff..46a57b3a2 100644 --- a/crates/entities/src/models/processed_files.rs +++ b/crates/entities/src/models/processed_files.rs @@ -1,5 +1,5 @@ use sea_orm::entity::prelude::*; -use sea_orm::{DatabaseBackend, FromQueryResult, Set, Statement}; +use sea_orm::{FromQueryResult, Set, Statement}; use serde::Serialize; use crate::BATCH_SIZE; @@ -91,7 +91,7 @@ pub async fn get_files_to_recrawl( ) -> Result, DbErr> { let ext_filter = format!("%.{ext}"); let urls = FileUrls::find_by_statement(Statement::from_sql_and_values( - DatabaseBackend::Sqlite, + db.get_database_backend(), r#" with possible as ( select url @@ -111,3 +111,27 @@ pub async fn get_files_to_recrawl( Err(err) => Err(err), } } + +// Helper method to copy the table from one database to another +pub async fn copy_table( + from: &DatabaseConnection, + to: &DatabaseConnection, +) -> anyhow::Result<(), sea_orm::DbErr> { + let mut pages = Entity::find().paginate(from, 1000); + Entity::delete_many().exec(to).await?; + while let Ok(Some(pages)) = pages.fetch_and_next().await { + let active_model = pages + .into_iter() + .map(|model| model.into()) + .collect::>(); + Entity::insert_many(active_model) + .on_conflict( + sea_orm::sea_query::OnConflict::columns(vec![Column::Id]) + .do_nothing() + .to_owned(), + ) + .exec(to) + .await?; + } + Ok(()) +} diff --git a/crates/entities/src/models/resource_rule.rs b/crates/entities/src/models/resource_rule.rs index f628eb4de..a1678f2a0 100644 --- a/crates/entities/src/models/resource_rule.rs +++ b/crates/entities/src/models/resource_rule.rs @@ -46,6 +46,30 @@ impl ActiveModelBehavior for ActiveModel { } } +// Helper method to copy the table from one database to another +pub async fn copy_table( + from: &DatabaseConnection, + to: &DatabaseConnection, +) -> anyhow::Result<(), sea_orm::DbErr> { + let mut pages = Entity::find().paginate(from, 1000); + Entity::delete_many().exec(to).await?; + while let Ok(Some(pages)) = pages.fetch_and_next().await { + let active_model = pages + .into_iter() + .map(|model| model.into()) + .collect::>(); + Entity::insert_many(active_model) + .on_conflict( + sea_orm::sea_query::OnConflict::columns(vec![Column::Id]) + .do_nothing() + .to_owned(), + ) + .exec(to) + .await?; + } + Ok(()) +} + #[cfg(test)] mod test { use sea_orm::prelude::*; diff --git a/crates/entities/src/models/tag.rs b/crates/entities/src/models/tag.rs index 2fa11a6d8..04204e735 100644 --- a/crates/entities/src/models/tag.rs +++ b/crates/entities/src/models/tag.rs @@ -198,8 +198,8 @@ where .do_nothing() .to_owned(), ) - .exec_with_returning(db) - .await; + .exec_without_returning(db) + .await?; let tag = Entity::find() .filter(Column::Label.eq(label.to_string())) @@ -320,6 +320,30 @@ pub async fn get_tags_by_value( find.all(db).await } +// Helper method to copy the table from one database to another +pub async fn copy_table( + from: &DatabaseConnection, + to: &DatabaseConnection, +) -> anyhow::Result<(), sea_orm::DbErr> { + let mut pages = Entity::find().paginate(from, 1000); + Entity::delete_many().exec(to).await?; + while let Ok(Some(pages)) = pages.fetch_and_next().await { + let active_model = pages + .into_iter() + .map(|model| model.into()) + .collect::>(); + Entity::insert_many(active_model) + .on_conflict( + sea_orm::sea_query::OnConflict::columns(vec![Column::Id]) + .do_nothing() + .to_owned(), + ) + .exec(to) + .await?; + } + Ok(()) +} + #[cfg(test)] mod test { use crate::models::tag; diff --git a/crates/migrations/src/m20220505_000001_create_table.rs b/crates/migrations/src/m20220505_000001_create_table.rs index 56b0e16f7..2c16022ee 100644 --- a/crates/migrations/src/m20220505_000001_create_table.rs +++ b/crates/migrations/src/m20220505_000001_create_table.rs @@ -1,4 +1,4 @@ -use entities::sea_orm::{ConnectionTrait, Statement}; +use entities::sea_orm::{ConnectionTrait, DbBackend, Statement}; use sea_orm_migration::prelude::*; pub struct Migration; @@ -12,71 +12,136 @@ impl MigrationName for Migration { #[async_trait::async_trait] impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - let crawl_queue = r#" - CREATE TABLE IF NOT EXISTS "crawl_queue" ( - "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, - "domain" text NOT NULL, - "url" text NOT NULL UNIQUE, - "status" text NOT NULL, - "num_retries" integer NOT NULL DEFAULT 0, - "force_crawl" integer NOT NULL DEFAULT FALSE, - "created_at" text NOT NULL, - "updated_at" text NOT NULL);"#; - - let fetch_history = r#" - CREATE TABLE IF NOT EXISTS "fetch_history" ( - "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, - "protocol" text NOT NULL, - "domain" text NOT NULL, - "path" text NOT NULL, - "hash" text, - "status" integer NOT NULL, - "no_index" integer NOT NULL DEFAULT FALSE, - "created_at" text NOT NULL, - "updated_at" text NOT NULL );"#; - - let indexed_document = r#" - CREATE TABLE IF NOT EXISTS "indexed_document" ( - "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, - "domain" text NOT NULL, - "url" text NOT NULL, - "doc_id" text NOT NULL, - "created_at" text NOT NULL, - "updated_at" text NOT NULL ); - "#; - - let resource_rules = r#" - CREATE TABLE IF NOT EXISTS "resource_rules" ( - "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, - "domain" text NOT NULL, - "rule" text NOT NULL, - "no_index" integer NOT NULL, - "allow_crawl" integer NOT NULL, - "created_at" text NOT NULL, - "updated_at" text NOT NULL );"#; - - let link = r#" - CREATE TABLE IF NOT EXISTS "link" ( - "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, - "src_domain" text NOT NULL, - "src_url" text NOT NULL, - "dst_domain" text NOT NULL, - "dst_url" text NOT NULL );"#; - - for sql in &[ - crawl_queue, - fetch_history, - indexed_document, - resource_rules, - link, - ] { - manager - .get_connection() - .execute(Statement::from_string( - manager.get_database_backend(), - sql.to_owned().to_string(), - )) - .await?; + let sql_list = if manager.get_database_backend() == DbBackend::Sqlite { + let crawl_queue = r#" + CREATE TABLE IF NOT EXISTS "crawl_queue" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "domain" text NOT NULL, + "url" text NOT NULL UNIQUE, + "status" text NOT NULL, + "num_retries" integer NOT NULL DEFAULT 0, + "force_crawl" integer NOT NULL DEFAULT FALSE, + "created_at" text NOT NULL, + "updated_at" text NOT NULL);"#; + + let fetch_history = r#" + CREATE TABLE IF NOT EXISTS "fetch_history" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "protocol" text NOT NULL, + "domain" text NOT NULL, + "path" text NOT NULL, + "hash" text, + "status" integer NOT NULL, + "no_index" integer NOT NULL DEFAULT FALSE, + "created_at" text NOT NULL, + "updated_at" text NOT NULL );"#; + + let indexed_document = r#" + CREATE TABLE IF NOT EXISTS "indexed_document" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "domain" text NOT NULL, + "url" text NOT NULL, + "doc_id" text NOT NULL, + "created_at" text NOT NULL, + "updated_at" text NOT NULL );"#; + + let resource_rules = r#" + CREATE TABLE IF NOT EXISTS "resource_rules" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "domain" text NOT NULL, + "rule" text NOT NULL, + "no_index" integer NOT NULL, + "allow_crawl" integer NOT NULL, + "created_at" text NOT NULL, + "updated_at" text NOT NULL );"#; + + let link = r#" + CREATE TABLE IF NOT EXISTS "link" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "src_domain" text NOT NULL, + "src_url" text NOT NULL, + "dst_domain" text NOT NULL, + "dst_url" text NOT NULL );"#; + + Some([ + crawl_queue, + fetch_history, + indexed_document, + resource_rules, + link, + ]) + } else if manager.get_database_backend() == DbBackend::Postgres { + let crawl_queue = r#" + CREATE TABLE IF NOT EXISTS "crawl_queue" ( + "id" BIGSERIAL PRIMARY KEY, + "domain" text NOT NULL, + "url" text NOT NULL UNIQUE, + "status" text NOT NULL, + "num_retries" integer NOT NULL DEFAULT 0, + "force_crawl" boolean NOT NULL DEFAULT FALSE, + "created_at" TIMESTAMPTZ NOT NULL, + "updated_at" TIMESTAMPTZ NOT NULL);"#; + + let fetch_history = r#" + CREATE TABLE IF NOT EXISTS "fetch_history" ( + "id" BIGSERIAL PRIMARY KEY, + "protocol" text NOT NULL, + "domain" text NOT NULL, + "path" text NOT NULL, + "hash" text, + "status" integer NOT NULL, + "no_index" boolean NOT NULL DEFAULT FALSE, + "created_at" TIMESTAMPTZ NOT NULL, + "updated_at" TIMESTAMPTZ NOT NULL );"#; + + let indexed_document = r#" + CREATE TABLE IF NOT EXISTS "indexed_document" ( + "id" BIGSERIAL PRIMARY KEY, + "domain" text NOT NULL, + "url" text NOT NULL, + "doc_id" text NOT NULL, + "created_at" TIMESTAMPTZ NOT NULL, + "updated_at" TIMESTAMPTZ NOT NULL );"#; + + let resource_rules = r#" + CREATE TABLE IF NOT EXISTS "resource_rules" ( + "id" BIGSERIAL PRIMARY KEY, + "domain" text NOT NULL, + "rule" text NOT NULL, + "no_index" integer NOT NULL, + "allow_crawl" integer NOT NULL, + "created_at" TIMESTAMPTZ NOT NULL, + "updated_at" TIMESTAMPTZ NOT NULL );"#; + + let link = r#" + CREATE TABLE IF NOT EXISTS "link" ( + "id" BIGSERIAL PRIMARY KEY, + "src_domain" text NOT NULL, + "src_url" text NOT NULL, + "dst_domain" text NOT NULL, + "dst_url" text NOT NULL );"#; + + Some([ + crawl_queue, + fetch_history, + indexed_document, + resource_rules, + link, + ]) + } else { + None + }; + + if let Some(sql_list) = sql_list { + for sql in sql_list { + manager + .get_connection() + .execute(Statement::from_string( + manager.get_database_backend(), + sql.to_owned().to_string(), + )) + .await?; + } } Ok(()) diff --git a/crates/migrations/src/m20220508_000001_lens_and_crawl_queue_update.rs b/crates/migrations/src/m20220508_000001_lens_and_crawl_queue_update.rs index a5565c394..70fa30cdd 100644 --- a/crates/migrations/src/m20220508_000001_lens_and_crawl_queue_update.rs +++ b/crates/migrations/src/m20220508_000001_lens_and_crawl_queue_update.rs @@ -1,6 +1,6 @@ use entities::{ models::crawl_queue, - sea_orm::{ConnectionTrait, Statement}, + sea_orm::{ConnectionTrait, DbBackend, Statement}, }; use sea_orm_migration::prelude::*; @@ -15,22 +15,40 @@ impl MigrationName for Migration { #[async_trait::async_trait] impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - let lens_table = r#" - CREATE TABLE IF NOT EXISTS "lens" ( - "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, - "name" text NOT NULL UNIQUE, - "author" text NOT NULL, - "description" text NOT NULL, - "version" text NOT NULL);"#; + let lens_table = if manager.get_database_backend() == DbBackend::Sqlite { + Some( + r#" + CREATE TABLE IF NOT EXISTS "lens" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "name" text NOT NULL UNIQUE, + "author" text NOT NULL, + "description" text NOT NULL, + "version" text NOT NULL);"#, + ) + } else if manager.get_database_backend() == DbBackend::Postgres { + Some( + r#" + CREATE TABLE IF NOT EXISTS "lens" ( + "id" BIGSERIAL PRIMARY KEY, + "name" text NOT NULL UNIQUE, + "author" text NOT NULL, + "description" text NOT NULL, + "version" text NOT NULL);"#, + ) + } else { + None + }; - // Create lens table - manager - .get_connection() - .execute(Statement::from_string( - manager.get_database_backend(), - lens_table.to_owned().to_string(), - )) - .await?; + if let Some(lens_table) = lens_table { + // Create lens table + manager + .get_connection() + .execute(Statement::from_string( + manager.get_database_backend(), + lens_table.to_owned().to_string(), + )) + .await?; + } // Add crawl_type column manager diff --git a/crates/migrations/src/m20220522_000001_bootstrap_queue_table.rs b/crates/migrations/src/m20220522_000001_bootstrap_queue_table.rs index edde55631..08518dae3 100644 --- a/crates/migrations/src/m20220522_000001_bootstrap_queue_table.rs +++ b/crates/migrations/src/m20220522_000001_bootstrap_queue_table.rs @@ -1,4 +1,4 @@ -use entities::sea_orm::{ConnectionTrait, Statement}; +use entities::sea_orm::{ConnectionTrait, DbBackend, Statement}; use sea_orm_migration::prelude::*; pub struct Migration; @@ -12,22 +12,39 @@ impl MigrationName for Migration { #[async_trait::async_trait] impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - let new_table = r#" - CREATE TABLE IF NOT EXISTS "bootstrap_queue" ( + let new_table = if manager.get_database_backend() == DbBackend::Sqlite { + Some( + r#"CREATE TABLE IF NOT EXISTS "bootstrap_queue" ( "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "seed_url" text NOT NULL UNIQUE, "count" integer NOT NULL DEFAULT 0, "created_at" text NOT NULL, - "updated_at" text NOT NULL);"#; + "updated_at" text NOT NULL);"#, + ) + } else if manager.get_database_backend() == DbBackend::Postgres { + Some( + r#"CREATE TABLE IF NOT EXISTS "bootstrap_queue" ( + "id" BIGSERIAL PRIMARY KEY, + "seed_url" text NOT NULL UNIQUE, + "count" integer NOT NULL DEFAULT 0, + "created_at" TIMESTAMPTZ NOT NULL, + "updated_at" TIMESTAMPTZ NOT NULL);"#, + ) + } else { + None + }; + + if let Some(new_table) = new_table { + // Create lens table + manager + .get_connection() + .execute(Statement::from_string( + manager.get_database_backend(), + new_table.to_owned().to_string(), + )) + .await?; + } - // Create lens table - manager - .get_connection() - .execute(Statement::from_string( - manager.get_database_backend(), - new_table.to_owned().to_string(), - )) - .await?; Ok(()) } diff --git a/crates/migrations/src/m20221023_000001_connection_table.rs b/crates/migrations/src/m20221023_000001_connection_table.rs index 75cea99f6..01a2a929f 100644 --- a/crates/migrations/src/m20221023_000001_connection_table.rs +++ b/crates/migrations/src/m20221023_000001_connection_table.rs @@ -1,6 +1,6 @@ use crate::sea_orm::Statement; use sea_orm_migration::prelude::*; -use sea_orm_migration::sea_orm::ConnectionTrait; +use sea_orm_migration::sea_orm::{ConnectionTrait, DbBackend}; pub struct Migration; impl MigrationName for Migration { @@ -12,26 +12,47 @@ impl MigrationName for Migration { #[async_trait::async_trait] impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - // Add connection table - let new_table = r#" - CREATE TABLE IF NOT EXISTS "connections" ( - "id" text NOT NULL PRIMARY KEY, - "access_token" text NOT NULL, - "refresh_token" text, - "scopes" text NOT NULL, - "expires_in" integer, - "granted_at" text NOT NULL, - "created_at" text NOT NULL, - "updated_at" text NOT NULL);"#; + let new_table = if manager.get_database_backend() == DbBackend::Sqlite { + Some( + r#" + CREATE TABLE IF NOT EXISTS "connections" ( + "id" text NOT NULL PRIMARY KEY, + "access_token" text NOT NULL, + "refresh_token" text, + "scopes" text NOT NULL, + "expires_in" integer, + "granted_at" text NOT NULL, + "created_at" text NOT NULL, + "updated_at" text NOT NULL);"#, + ) + } else if manager.get_database_backend() == DbBackend::Postgres { + Some( + r#" + CREATE TABLE IF NOT EXISTS "connections" ( + "id" text NOT NULL PRIMARY KEY, + "access_token" text NOT NULL, + "refresh_token" text, + "scopes" text NOT NULL, + "expires_in" integer, + "granted_at" TIMESTAMPTZ NOT NULL, + "created_at" TIMESTAMPTZ NOT NULL, + "updated_at" TIMESTAMPTZ NOT NULL);"#, + ) + } else { + None + }; + + if let Some(new_table) = new_table { + // Create lens table + manager + .get_connection() + .execute(Statement::from_string( + manager.get_database_backend(), + new_table.to_owned().to_string(), + )) + .await?; + } - // Create lens table - manager - .get_connection() - .execute(Statement::from_string( - manager.get_database_backend(), - new_table.to_owned().to_string(), - )) - .await?; Ok(()) } diff --git a/crates/migrations/src/m20221107_000001_recreate_connection_table.rs b/crates/migrations/src/m20221107_000001_recreate_connection_table.rs index d5ac6ddbf..c90cde8e5 100644 --- a/crates/migrations/src/m20221107_000001_recreate_connection_table.rs +++ b/crates/migrations/src/m20221107_000001_recreate_connection_table.rs @@ -1,6 +1,6 @@ use crate::sea_orm::Statement; use sea_orm_migration::prelude::*; -use sea_orm_migration::sea_orm::ConnectionTrait; +use sea_orm_migration::sea_orm::{ConnectionTrait, DbBackend}; pub struct Migration; @@ -22,28 +22,50 @@ impl MigrationTrait for Migration { )) .await?; - // Add account column - let new_table = r#" - CREATE TABLE IF NOT EXISTS "connections" ( - "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, - "api_id" text NOT NULL, - "account" text NOT NULL, - "access_token" text NOT NULL, - "refresh_token" text, - "scopes" text NOT NULL, - "expires_in" integer, - "granted_at" text NOT NULL, - "created_at" text NOT NULL, - "updated_at" text NOT NULL);"#; + let new_table = if manager.get_database_backend() == DbBackend::Sqlite { + Some( + r#" + CREATE TABLE IF NOT EXISTS "connections" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "api_id" text NOT NULL, + "account" text NOT NULL, + "access_token" text NOT NULL, + "refresh_token" text, + "scopes" text NOT NULL, + "expires_in" integer, + "granted_at" text NOT NULL, + "created_at" text NOT NULL, + "updated_at" text NOT NULL);"#, + ) + } else if manager.get_database_backend() == DbBackend::Postgres { + Some( + r#" + CREATE TABLE IF NOT EXISTS "connections" ( + "id" BIGSERIAL PRIMARY KEY, + "api_id" text NOT NULL, + "account" text NOT NULL, + "access_token" text NOT NULL, + "refresh_token" text, + "scopes" text NOT NULL, + "expires_in" integer, + "granted_at" TIMESTAMPTZ NOT NULL, + "created_at" TIMESTAMPTZ NOT NULL, + "updated_at" TIMESTAMPTZ NOT NULL);"#, + ) + } else { + None + }; - // Create lens table - manager - .get_connection() - .execute(Statement::from_string( - manager.get_database_backend(), - new_table.to_owned().to_string(), - )) - .await?; + if let Some(new_table) = new_table { + // Create lens table + manager + .get_connection() + .execute(Statement::from_string( + manager.get_database_backend(), + new_table.to_owned().to_string(), + )) + .await?; + } Ok(()) } diff --git a/crates/migrations/src/m20221109_add_tags_table.rs b/crates/migrations/src/m20221109_add_tags_table.rs index a5f6aa4ef..0c4f777a3 100644 --- a/crates/migrations/src/m20221109_add_tags_table.rs +++ b/crates/migrations/src/m20221109_add_tags_table.rs @@ -1,6 +1,6 @@ use crate::sea_orm::Statement; use sea_orm_migration::prelude::*; -use sea_orm_migration::sea_orm::ConnectionTrait; +use sea_orm_migration::sea_orm::{ConnectionTrait, DbBackend}; pub struct Migration; @@ -13,39 +13,61 @@ impl MigrationName for Migration { #[async_trait::async_trait] impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - // Add tags table - manager - .get_connection() - .execute(Statement::from_string( - manager.get_database_backend(), - r#"CREATE TABLE IF NOT EXISTS "tags" ( - "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, - "label" text NOT NULL, - "value" text, - "created_at" text NOT NULL, - "updated_at" text NOT NULL - );"# - .to_string(), - )) - .await?; + let tables = if manager.get_database_backend() == DbBackend::Sqlite { + let tags = r#"CREATE TABLE IF NOT EXISTS "tags" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "label" text NOT NULL, + "value" text, + "created_at" text NOT NULL, + "updated_at" text NOT NULL + );"#; + let doc_tag = r#"CREATE TABLE IF NOT EXISTS "document_tag" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "indexed_document_id" integer NOT NULL, + "tag_id" integer NOT NULL, + "created_at" text NOT NULL, + "updated_at" text NOT NULL, + FOREIGN KEY(indexed_document_id) REFERENCES indexed_document(id), + FOREIGN KEY(tag_id) REFERENCES tags(id) + );"#; + Some([tags, doc_tag]) + } else if manager.get_database_backend() == DbBackend::Postgres { + let tags = r#"CREATE TABLE IF NOT EXISTS "tags" ( + "id" BIGSERIAL PRIMARY KEY, + "label" text NOT NULL, + "value" text, + "created_at" TIMESTAMPTZ NOT NULL, + "updated_at" TIMESTAMPTZ NOT NULL + );"#; + let doc_tag = r#"CREATE TABLE IF NOT EXISTS "document_tag" ( + "id" BIGSERIAL PRIMARY KEY, + "indexed_document_id" integer NOT NULL, + "tag_id" integer NOT NULL, + "created_at" TIMESTAMPTZ NOT NULL, + "updated_at" TIMESTAMPTZ NOT NULL, + CONSTRAINT fk_tag_id + FOREIGN KEY(tag_id) + REFERENCES tags(id), + CONSTRAINT fk_indexed_document_id + FOREIGN KEY(indexed_document_id) + REFERENCES indexed_document(id) + );"#; + Some([tags, doc_tag]) + } else { + None + }; - // Add through table - manager - .get_connection() - .execute(Statement::from_string( - manager.get_database_backend(), - r#"CREATE TABLE IF NOT EXISTS "document_tag" ( - "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, - "indexed_document_id" integer NOT NULL, - "tag_id" integer NOT NULL, - "created_at" text NOT NULL, - "updated_at" text NOT NULL, - FOREIGN KEY(indexed_document_id) REFERENCES indexed_document(id), - FOREIGN KEY(tag_id) REFERENCES tags(id) - );"# - .to_string(), - )) - .await?; + if let Some(tables) = tables { + for tbl in tables { + manager + .get_connection() + .execute(Statement::from_string( + manager.get_database_backend(), + tbl.to_string(), + )) + .await?; + } + } // Create index on (label, value). Should only every be one instance of a // tag with (label, value). @@ -53,7 +75,7 @@ impl MigrationTrait for Migration { .get_connection() .execute(Statement::from_string( manager.get_database_backend(), - "CREATE UNIQUE INDEX IF NOT EXISTS `idx-tag-label-value` ON `tags` (`label`, `value`);" + "CREATE UNIQUE INDEX IF NOT EXISTS \"idx-tag-label-value\" ON \"tags\" (\"label\", \"value\");" .to_string(), )) .await?; diff --git a/crates/migrations/src/m20221116_000001_add_connection_constraint.rs b/crates/migrations/src/m20221116_000001_add_connection_constraint.rs index e3ddc3959..765f75b94 100644 --- a/crates/migrations/src/m20221116_000001_add_connection_constraint.rs +++ b/crates/migrations/src/m20221116_000001_add_connection_constraint.rs @@ -19,7 +19,7 @@ impl MigrationTrait for Migration { .get_connection() .execute(Statement::from_string( manager.get_database_backend(), - "CREATE UNIQUE INDEX IF NOT EXISTS `idx-connections-api-id-account` ON `connections` (`api_id`, `account`);" + "CREATE UNIQUE INDEX IF NOT EXISTS \"idx-connections-api-id-account\" ON \"connections\" (\"api_id\", \"account\");" .to_string(), )) .await?; diff --git a/crates/migrations/src/m20221123_000001_add_document_tag_constraint.rs b/crates/migrations/src/m20221123_000001_add_document_tag_constraint.rs index 42950600e..8adeee8e9 100644 --- a/crates/migrations/src/m20221123_000001_add_document_tag_constraint.rs +++ b/crates/migrations/src/m20221123_000001_add_document_tag_constraint.rs @@ -28,7 +28,7 @@ impl MigrationTrait for Migration { .get_connection() .execute(Statement::from_string( manager.get_database_backend(), - "CREATE UNIQUE INDEX IF NOT EXISTS `idx-document-tag-doc-id-tag-id` ON `document_tag` (`indexed_document_id`, `tag_id`);" + "CREATE UNIQUE INDEX IF NOT EXISTS \"idx-document-tag-doc-id-tag-id\" ON \"document_tag\" (\"indexed_document_id\", \"tag_id\");" .to_string(), )) .await?; diff --git a/crates/migrations/src/m20221124_000001_add_tags_for_existing_lenses.rs b/crates/migrations/src/m20221124_000001_add_tags_for_existing_lenses.rs index 2eb720fab..96cce988d 100644 --- a/crates/migrations/src/m20221124_000001_add_tags_for_existing_lenses.rs +++ b/crates/migrations/src/m20221124_000001_add_tags_for_existing_lenses.rs @@ -1,7 +1,6 @@ use entities::{ models::{ - document_tag, indexed_document, - lens::{self, LensType}, + document_tag, indexed_document, lens, tag::{get_or_create, TagType}, }, sea_orm::{ @@ -107,12 +106,7 @@ impl MigrationTrait for Migration { let db = manager.get_connection(); // Loop through lenses - let lenses = lens::Entity::find() - .filter(lens::Column::IsEnabled.eq(true)) - .filter(lens::Column::LensType.eq(LensType::Simple)) - .all(db) - .await - .unwrap_or_default(); + let lenses = lens::get_lens_names(db).await?; let lens_dir = config.lenses_dir(); diff --git a/crates/migrations/src/m20221210_000001_add_crawl_tags_table.rs b/crates/migrations/src/m20221210_000001_add_crawl_tags_table.rs index bbeebcb01..7c0caa883 100644 --- a/crates/migrations/src/m20221210_000001_add_crawl_tags_table.rs +++ b/crates/migrations/src/m20221210_000001_add_crawl_tags_table.rs @@ -1,12 +1,11 @@ use entities::{ models::{ - crawl_queue, crawl_tag, - lens::{self, LensType}, + crawl_queue, crawl_tag, lens, tag::{get_or_create, TagType}, }, sea_orm::{ - ColumnTrait, ConnectionTrait, DatabaseTransaction, EntityTrait, QueryFilter, Set, - Statement, TransactionTrait, + ColumnTrait, ConnectionTrait, DatabaseTransaction, DbBackend, EntityTrait, QueryFilter, + Set, Statement, TransactionTrait, }, BATCH_SIZE, }; @@ -103,12 +102,13 @@ async fn add_tags_for_lens(db: &DatabaseTransaction, conf: &LensConfig) { #[async_trait::async_trait] impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - // Add crawl_tag table & idx - manager - .get_connection() - .execute(Statement::from_string( - manager.get_database_backend(), - r#"CREATE TABLE IF NOT EXISTS "crawl_tag" ( + if manager.get_database_backend() == DbBackend::Sqlite { + // Add crawl_tag table & idx + manager + .get_connection() + .execute(Statement::from_string( + manager.get_database_backend(), + r#"CREATE TABLE IF NOT EXISTS "crawl_tag" ( "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "crawl_queue_id" integer NOT NULL, "tag_id" integer NOT NULL, @@ -117,15 +117,38 @@ impl MigrationTrait for Migration { FOREIGN KEY(crawl_queue_id) REFERENCES crawl_queue(id), FOREIGN KEY(tag_id) REFERENCES tags(id) );"# - .to_string(), - )) - .await?; + .to_string(), + )) + .await?; + } else if manager.get_database_backend() == DbBackend::Postgres { + // Add crawl_tag table & idx + manager + .get_connection() + .execute(Statement::from_string( + manager.get_database_backend(), + r#"CREATE TABLE IF NOT EXISTS "crawl_tag" ( + "id" BIGSERIAL PRIMARY KEY, + "crawl_queue_id" integer NOT NULL, + "tag_id" integer NOT NULL, + "created_at" TIMESTAMPTZ NOT NULL, + "updated_at" TIMESTAMPTZ NOT NULL, + CONSTRAINT fk_crawl_queue_id + FOREIGN KEY(crawl_queue_id) + REFERENCES crawl_queue(id), + CONSTRAINT fk_tag_id + FOREIGN KEY(tag_id) + REFERENCES tags(id) + );"# + .to_string(), + )) + .await?; + } manager .get_connection() .execute(Statement::from_string( manager.get_database_backend(), - "CREATE UNIQUE INDEX IF NOT EXISTS `idx-crawl-tag-doc-id-tag-id` ON `crawl_tag` (`crawl_queue_id`, `tag_id`);" + "CREATE UNIQUE INDEX IF NOT EXISTS \"idx-crawl-tag-doc-id-tag-id\" ON \"crawl_tag\" (\"crawl_queue_id\", \"tag_id\");" .to_string(), )) .await?; @@ -134,12 +157,7 @@ impl MigrationTrait for Migration { let db = manager.get_connection(); // Loop through lenses - let lenses = lens::Entity::find() - .filter(lens::Column::IsEnabled.eq(true)) - .filter(lens::Column::LensType.eq(LensType::Simple)) - .all(db) - .await - .unwrap_or_default(); + let lenses = lens::get_lens_names(db).await?; let lens_dir = config.lenses_dir(); diff --git a/crates/migrations/src/m20230104_000001_add_column_n_index.rs b/crates/migrations/src/m20230104_000001_add_column_n_index.rs index 15e3c39e8..adb5ee347 100644 --- a/crates/migrations/src/m20230104_000001_add_column_n_index.rs +++ b/crates/migrations/src/m20230104_000001_add_column_n_index.rs @@ -45,7 +45,7 @@ impl MigrationTrait for Migration { .get_connection() .execute(Statement::from_string( manager.get_database_backend(), - "CREATE INDEX `tmp-idx-indexed_document-url` ON `indexed_document` (`url`);" + "CREATE INDEX \"tmp-idx-indexed_document-url\" ON \"indexed_document\" (\"url\");" .to_string(), )) .await?; @@ -98,7 +98,7 @@ impl MigrationTrait for Migration { .get_connection() .execute(Statement::from_string( manager.get_database_backend(), - "DROP INDEX `tmp-idx-indexed_document-url`;".to_string(), + "DROP INDEX \"tmp-idx-indexed_document-url\";".to_string(), )) .await?; @@ -109,7 +109,7 @@ impl MigrationTrait for Migration { .get_connection() .execute(Statement::from_string( manager.get_database_backend(), - "CREATE UNIQUE INDEX `idx-indexed_document-url` ON `indexed_document` (`url`);" + "CREATE UNIQUE INDEX \"idx-indexed_document-url\" ON \"indexed_document\" (\"url\");" .to_string(), )) .await; diff --git a/crates/migrations/src/m20230112_000001_migrate_search_schema.rs b/crates/migrations/src/m20230112_000001_migrate_search_schema.rs index aaaf53a79..c4b0ddcc7 100644 --- a/crates/migrations/src/m20230112_000001_migrate_search_schema.rs +++ b/crates/migrations/src/m20230112_000001_migrate_search_schema.rs @@ -144,7 +144,7 @@ impl MigrationTrait for Migration { .get_connection() .execute(Statement::from_string( manager.get_database_backend(), - "CREATE INDEX IF NOT EXISTS `idx-document_tag-indexed_document_id` ON `document_tag` (`indexed_document_id`);" + "CREATE INDEX IF NOT EXISTS \"idx-document_tag-indexed_document_id\" ON \"document_tag\" (\"indexed_document_id\");" .to_string(), )) .await?; diff --git a/crates/migrations/src/m20230126_000001_create_file_table.rs b/crates/migrations/src/m20230126_000001_create_file_table.rs index cd2de020c..989bb8d61 100644 --- a/crates/migrations/src/m20230126_000001_create_file_table.rs +++ b/crates/migrations/src/m20230126_000001_create_file_table.rs @@ -1,4 +1,4 @@ -use entities::sea_orm::{ConnectionTrait, Statement}; +use entities::sea_orm::{ConnectionTrait, DbBackend, Statement}; use sea_orm_migration::prelude::*; pub struct Migration; @@ -12,20 +12,38 @@ impl MigrationName for Migration { #[async_trait::async_trait] impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { - let processed_files = r#" - CREATE TABLE IF NOT EXISTS "processed_files" ( - "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, - "file_path" text NOT NULL UNIQUE, - "created_at" text NOT NULL, - "last_modified" text NOT NULL);"#; + let processed_files = if manager.get_database_backend() == DbBackend::Sqlite { + Some( + r#" + CREATE TABLE IF NOT EXISTS "processed_files" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "file_path" text NOT NULL UNIQUE, + "created_at" text NOT NULL, + "last_modified" text NOT NULL);"#, + ) + } else if manager.get_database_backend() == DbBackend::Postgres { + Some( + r#" + CREATE TABLE IF NOT EXISTS "processed_files" ( + "id" BIGSERIAL PRIMARY KEY, + "file_path" text NOT NULL UNIQUE, + "created_at" TIMESTAMPTZ NOT NULL, + "last_modified" TIMESTAMPTZ NOT NULL);"#, + ) + } else { + None + }; + + if let Some(processed_files) = processed_files { + manager + .get_connection() + .execute(Statement::from_string( + manager.get_database_backend(), + processed_files.to_owned().to_string(), + )) + .await?; + } - manager - .get_connection() - .execute(Statement::from_string( - manager.get_database_backend(), - processed_files.to_owned().to_string(), - )) - .await?; Ok(()) } diff --git a/crates/migrations/src/m20230201_000001_add_tag_index.rs b/crates/migrations/src/m20230201_000001_add_tag_index.rs index bcfbbfc0a..99e009398 100644 --- a/crates/migrations/src/m20230201_000001_add_tag_index.rs +++ b/crates/migrations/src/m20230201_000001_add_tag_index.rs @@ -17,7 +17,7 @@ impl MigrationTrait for Migration { .get_connection() .execute(Statement::from_string( manager.get_database_backend(), - "CREATE INDEX IF NOT EXISTS `idx-tag-value` ON `tags` (`value`);".to_string(), + "CREATE INDEX IF NOT EXISTS \"idx-tag-value\" ON \"tags\" (\"value\");".to_string(), )) .await?; diff --git a/crates/migrations/src/m20230203_000001_add_indexed_document_index.rs b/crates/migrations/src/m20230203_000001_add_indexed_document_index.rs index 566f69d94..7baf7f195 100644 --- a/crates/migrations/src/m20230203_000001_add_indexed_document_index.rs +++ b/crates/migrations/src/m20230203_000001_add_indexed_document_index.rs @@ -17,7 +17,7 @@ impl MigrationTrait for Migration { .get_connection() .execute(Statement::from_string( manager.get_database_backend(), - "CREATE INDEX IF NOT EXISTS `idx-indexed_document-doc_id` ON `indexed_document` (`doc_id`);".to_string(), + "CREATE INDEX IF NOT EXISTS \"idx-indexed_document-doc_id\" ON \"indexed_document\" (\"doc_id\");".to_string(), )) .await?; diff --git a/crates/spyglass/src/search/mod.rs b/crates/spyglass/src/search/mod.rs index 67a0d7760..089490e0a 100644 --- a/crates/spyglass/src/search/mod.rs +++ b/crates/spyglass/src/search/mod.rs @@ -538,6 +538,66 @@ impl ReadonlySearcher { } None } + + pub async fn search_with_lens( + db: &DatabaseConnection, + applied_lenses: &Vec, + searcher: &ReadonlySearcher, + query_string: &str, + stats: &mut QueryStats, + ) -> Vec { + let start_timer = Instant::now(); + + let mut tag_boosts = HashSet::new(); + let favorite_boost = if let Ok(Some(favorited)) = tag::Entity::find() + .filter(tag::Column::Label.eq(tag::TagType::Favorited.to_string())) + .one(db) + .await + { + Some(favorited.id) + } else { + None + }; + + let tag_checks = get_tag_checks(db, query_string).await.unwrap_or_default(); + tag_boosts.extend(tag_checks); + + let index = &searcher.index; + let reader = &searcher.reader; + let fields = DocFields::as_fields(); + let searcher = reader.searcher(); + let tokenizers = index.tokenizers().clone(); + let query = build_query( + index.schema(), + tokenizers, + fields, + query_string, + applied_lenses, + tag_boosts.into_iter(), + favorite_boost, + stats, + ); + + let collector = TopDocs::with_limit(5); + + let top_docs = searcher + .search(&query, &collector) + .expect("Unable to execute query"); + + log::debug!( + "query `{}` returned {} results from {} docs in {} ms", + query_string, + top_docs.len(), + searcher.num_docs(), + Instant::now().duration_since(start_timer).as_millis() + ); + + top_docs + .into_iter() + // Filter out negative scores + .filter(|(score, _)| *score > 0.0) + .collect() + } } // Helper method used to get the list of tag ids that should be included in the search From 2440fc2445f6391c494c9999a4f313de016c521d Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Thu, 13 Apr 2023 22:41:48 -0700 Subject: [PATCH 06/30] separate out reusable frontend components & scaffolding a web client (#432) * splitting out reusable UI components * scaffolding out a web-app * cleaning up ui-components split * added sidebar, setting up search area * setup basic console logging & add search/enter events * setup basic deploy cmd * forgot to remove tooltips mod * moving tag to ui-components --- Cargo.lock | 55 +- Cargo.toml | 7 +- apps/web/Cargo.toml | 23 + apps/web/Makefile | 5 + apps/web/Trunk.toml | 4 + apps/web/index.html | 9 + apps/web/package-lock.json | 1731 +++++++++++++++++ apps/web/package.json | 10 + apps/web/public/main.css | 1 + apps/web/src-css/main.css | 38 + apps/web/src/main.rs | 47 + apps/web/src/pages/mod.rs | 77 + apps/web/src/pages/search.rs | 88 + apps/web/tailwind.config.js | 54 + crates/client/Cargo.toml | 3 +- crates/client/src/components/btn.rs | 71 +- .../client/src/components/forms/pathlist.rs | 8 +- crates/client/src/components/mod.rs | 4 +- crates/client/src/components/result.rs | 8 +- crates/client/src/pages/admin.rs | 4 +- crates/client/src/pages/connection_manager.rs | 3 +- crates/client/src/pages/discover.rs | 4 +- crates/client/src/pages/library.rs | 2 +- crates/client/src/pages/plugin_manager.rs | 3 +- crates/client/src/pages/search.rs | 2 +- crates/client/src/pages/settings.rs | 7 +- crates/client/src/pages/startup.rs | 2 +- crates/client/src/pages/updater.rs | 3 +- .../client/src/pages/wizard/indexing_help.rs | 3 +- crates/client/src/pages/wizard/mod.rs | 3 +- crates/shared/src/event.rs | 4 +- crates/ui-components/Cargo.toml | 10 + crates/ui-components/src/btn.rs | 163 ++ .../src}/icons/browser.rs | 0 .../src/icons/mod.rs} | 9 + crates/ui-components/src/lib.rs | 4 + .../components => ui-components/src}/tag.rs | 0 .../src}/tooltip.rs | 0 38 files changed, 2367 insertions(+), 102 deletions(-) create mode 100644 apps/web/Cargo.toml create mode 100644 apps/web/Makefile create mode 100644 apps/web/Trunk.toml create mode 100644 apps/web/index.html create mode 100644 apps/web/package-lock.json create mode 100644 apps/web/package.json create mode 100644 apps/web/public/main.css create mode 100644 apps/web/src-css/main.css create mode 100644 apps/web/src/main.rs create mode 100644 apps/web/src/pages/mod.rs create mode 100644 apps/web/src/pages/search.rs create mode 100644 apps/web/tailwind.config.js create mode 100644 crates/ui-components/Cargo.toml create mode 100644 crates/ui-components/src/btn.rs rename crates/{client/src/components => ui-components/src}/icons/browser.rs (100%) rename crates/{client/src/components/icons.rs => ui-components/src/icons/mod.rs} (98%) create mode 100644 crates/ui-components/src/lib.rs rename crates/{client/src/components => ui-components/src}/tag.rs (100%) rename crates/{client/src/components => ui-components/src}/tooltip.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 29c2a4fec..94566d15c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1036,6 +1036,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "console_log" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8aed40e4edbf4d3b4431ab260b63fdc40f5780a4766824329ea0f1eefe3c0f" +dependencies = [ + "log", + "web-sys", +] + [[package]] name = "const_fn" version = "0.4.9" @@ -2431,7 +2441,7 @@ dependencies = [ "gloo-events", "gloo-utils", "serde", - "serde-wasm-bindgen", + "serde-wasm-bindgen 0.4.5", "serde_urlencoded", "thiserror", "wasm-bindgen", @@ -6162,6 +6172,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3b143e2833c57ab9ad3ea280d21fd34e285a42837aeb0ee301f4f41890fa00e" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_bytes" version = "0.11.9" @@ -6652,11 +6673,12 @@ dependencies = [ "log", "num-format", "serde", - "serde-wasm-bindgen", + "serde-wasm-bindgen 0.5.0", "serde_json", "shared", "strum 0.24.1", "strum_macros 0.24.3", + "ui-components", "url", "wasm-bindgen", "wasm-bindgen-futures", @@ -8291,6 +8313,14 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" +[[package]] +name = "ui-components" +version = "0.1.0" +dependencies = [ + "wasm-bindgen-futures", + "yew", +] + [[package]] name = "uname" version = "0.1.1" @@ -8964,6 +8994,27 @@ dependencies = [ "wast", ] +[[package]] +name = "web" +version = "0.1.0" +dependencies = [ + "console_log", + "gloo", + "log", + "markdown", + "serde-wasm-bindgen 0.5.0", + "serde_json", + "shared", + "strum 0.24.1", + "strum_macros 0.24.3", + "ui-components", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "yew", + "yew-router", +] + [[package]] name = "web-sys" version = "0.3.61" diff --git a/Cargo.toml b/Cargo.toml index a0564e409..5b97c6ae6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,12 @@ members = [ "crates/shared", "crates/spyglass", "crates/tauri", - # Publically published crates + "crates/ui-components", + + # Clients + "apps/web", + + # Public published crates "crates/spyglass-plugin", "crates/spyglass-lens", "crates/spyglass-rpc", diff --git a/apps/web/Cargo.toml b/apps/web/Cargo.toml new file mode 100644 index 000000000..25095d04e --- /dev/null +++ b/apps/web/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "web" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +console_log = "1.0" +"shared" = { path = "../../crates/shared" } +gloo = "0.8.0" +log = "0.4" +serde_json = "1.0" +serde-wasm-bindgen = "0.5" +strum = "0.24" +strum_macros = "0.24" +ui-components = { path = "../../crates/ui-components" } +wasm-bindgen = "0.2.83" +wasm-bindgen-futures = "0.4.33" +web-sys = { version = "0.3.60", features = ["Navigator", "VisibilityState"] } +yew = { version = "0.20.0", features = ["csr"] } +yew-router = "0.17" +markdown = "1.0.0-alpha.7" \ No newline at end of file diff --git a/apps/web/Makefile b/apps/web/Makefile new file mode 100644 index 000000000..f0a4519e4 --- /dev/null +++ b/apps/web/Makefile @@ -0,0 +1,5 @@ +.PHONY: deploy + +deploy: + trunk build --release + aws s3 cp --recursive dist s3://app.spyglass.fyi diff --git a/apps/web/Trunk.toml b/apps/web/Trunk.toml new file mode 100644 index 000000000..0292379e0 --- /dev/null +++ b/apps/web/Trunk.toml @@ -0,0 +1,4 @@ + +[build] +target = "index.html" +dist = "dist" \ No newline at end of file diff --git a/apps/web/index.html b/apps/web/index.html new file mode 100644 index 000000000..5599bd9b8 --- /dev/null +++ b/apps/web/index.html @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/apps/web/package-lock.json b/apps/web/package-lock.json new file mode 100644 index 000000000..58a5c88c5 --- /dev/null +++ b/apps/web/package-lock.json @@ -0,0 +1,1731 @@ +{ + "name": "web", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "devDependencies": { + "@tailwindcss/forms": "^0.5.2", + "tailwindcss": "^3.0.24" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", + "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==", + "dev": true, + "dependencies": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.18", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz", + "integrity": "sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "3.1.0", + "@jridgewell/sourcemap-codec": "1.4.14" + } + }, + "node_modules/@jridgewell/trace-mapping/node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", + "dev": true + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@tailwindcss/forms": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/forms/-/forms-0.5.3.tgz", + "integrity": "sha512-y5mb86JUoiUgBjY/o6FJSFZSEttfb3Q5gllE4xoKjAAD+vBrnIhE4dViwUuow3va8mpH4s9jyUbUbrRGoRdc2Q==", + "dev": true, + "dependencies": { + "mini-svg-data-uri": "^1.2.3" + }, + "peerDependencies": { + "tailwindcss": ">=3.0.0 || >= 3.0.0-alpha.1" + } + }, + "node_modules/any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", + "dev": true + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "dev": true + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "dependencies": { + "fill-range": "^7.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/camelcase-css": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz", + "integrity": "sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + ], + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chokidar/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "dev": true, + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/didyoumean": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", + "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==", + "dev": true + }, + "node_modules/dlv": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", + "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", + "dev": true + }, + "node_modules/fast-glob": { + "version": "3.2.12", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", + "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-glob/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/fastq": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", + "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", + "dev": true, + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "node_modules/glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-core-module": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.0.tgz", + "integrity": "sha512-RECHCBCd/viahWmwj6enj19sKbHfJrddi/6cBDsNTKbNq0f7VeaUkBo60BqzvPqo/W54ChS62Z5qyun7cfOMqQ==", + "dev": true, + "dependencies": { + "has": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/jiti": { + "version": "1.18.2", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.18.2.tgz", + "integrity": "sha512-QAdOptna2NYiSSpv0O/BwoHBSmz4YhpzJHyi+fnMRTXFjp7B8i/YG5Z8IfusxB1ufjcD2Sre1F3R+nX3fvy7gg==", + "dev": true, + "bin": { + "jiti": "bin/jiti.js" + } + }, + "node_modules/lilconfig": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz", + "integrity": "sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "dev": true, + "dependencies": { + "braces": "^3.0.2", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mini-svg-data-uri": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/mini-svg-data-uri/-/mini-svg-data-uri-1.4.4.tgz", + "integrity": "sha512-r9deDe9p5FJUPZAk3A59wGH7Ii9YrjjWw0jmw/liSbHl2CHiyXj6FcDXDu2K3TjVAXqiJdaw3xxwlZZr9E6nHg==", + "dev": true, + "bin": { + "mini-svg-data-uri": "cli.js" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mz": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "dev": true, + "dependencies": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true + }, + "node_modules/picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pirates": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.5.tgz", + "integrity": "sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/postcss": { + "version": "8.4.21", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.21.tgz", + "integrity": "sha512-tP7u/Sn/dVxK2NnruI4H9BG+x+Wxz6oeZ1cJ8P6G/PZY0IKk4k/63TDsQf2kQq3+qoJeLm2kIBUNlZe3zgb4Zg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + } + ], + "dependencies": { + "nanoid": "^3.3.4", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-import": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/postcss-import/-/postcss-import-14.1.0.tgz", + "integrity": "sha512-flwI+Vgm4SElObFVPpTIT7SU7R3qk2L7PyduMcokiaVKuWv9d/U+Gm/QAd8NDLuykTWTkcrjOeD2Pp1rMeBTGw==", + "dev": true, + "dependencies": { + "postcss-value-parser": "^4.0.0", + "read-cache": "^1.0.0", + "resolve": "^1.1.7" + }, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "postcss": "^8.0.0" + } + }, + "node_modules/postcss-js": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/postcss-js/-/postcss-js-4.0.1.tgz", + "integrity": "sha512-dDLF8pEO191hJMtlHFPRa8xsizHaM82MLfNkUHdUtVEV3tgTp5oj+8qbEqYM57SLfc74KSbw//4SeJma2LRVIw==", + "dev": true, + "dependencies": { + "camelcase-css": "^2.0.1" + }, + "engines": { + "node": "^12 || ^14 || >= 16" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + "peerDependencies": { + "postcss": "^8.4.21" + } + }, + "node_modules/postcss-load-config": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-3.1.4.tgz", + "integrity": "sha512-6DiM4E7v4coTE4uzA8U//WhtPwyhiim3eyjEMFCnUpzbrkK9wJHgKDT2mR+HbtSrd/NubVaYTOpSpjUl8NQeRg==", + "dev": true, + "dependencies": { + "lilconfig": "^2.0.5", + "yaml": "^1.10.2" + }, + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + "peerDependencies": { + "postcss": ">=8.0.9", + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "postcss": { + "optional": true + }, + "ts-node": { + "optional": true + } + } + }, + "node_modules/postcss-nested": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/postcss-nested/-/postcss-nested-6.0.0.tgz", + "integrity": "sha512-0DkamqrPcmkBDsLn+vQDIrtkSbNkv5AD/M322ySo9kqFkCIYklym2xEmWkwo+Y3/qZo34tzEPNUw4y7yMCdv5w==", + "dev": true, + "dependencies": { + "postcss-selector-parser": "^6.0.10" + }, + "engines": { + "node": ">=12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + "peerDependencies": { + "postcss": "^8.2.14" + } + }, + "node_modules/postcss-selector-parser": { + "version": "6.0.11", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.11.tgz", + "integrity": "sha512-zbARubNdogI9j7WY4nQJBiNqQf3sLS3wCP4WfOidu+p28LofJqDH1tcXypGrcmMHhDk2t9wGhCsYe/+szLTy1g==", + "dev": true, + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/quick-lru": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", + "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/read-cache": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", + "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", + "dev": true, + "dependencies": { + "pify": "^2.3.0" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/resolve": { + "version": "1.22.2", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz", + "integrity": "sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==", + "dev": true, + "dependencies": { + "is-core-module": "^2.11.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true, + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/source-map-js": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", + "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/sucrase": { + "version": "3.32.0", + "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.32.0.tgz", + "integrity": "sha512-ydQOU34rpSyj2TGyz4D2p8rbktIOZ8QY9s+DGLvFU1i5pWJE8vkpruCjGCMHsdXwnD7JDcS+noSwM/a7zyNFDQ==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.2", + "commander": "^4.0.0", + "glob": "7.1.6", + "lines-and-columns": "^1.1.6", + "mz": "^2.7.0", + "pirates": "^4.0.1", + "ts-interface-checker": "^0.1.9" + }, + "bin": { + "sucrase": "bin/sucrase", + "sucrase-node": "bin/sucrase-node" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/tailwindcss": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.3.1.tgz", + "integrity": "sha512-Vkiouc41d4CEq0ujXl6oiGFQ7bA3WEhUZdTgXAhtKxSy49OmKs8rEfQmupsfF0IGW8fv2iQkp1EVUuapCFrZ9g==", + "dev": true, + "dependencies": { + "arg": "^5.0.2", + "chokidar": "^3.5.3", + "color-name": "^1.1.4", + "didyoumean": "^1.2.2", + "dlv": "^1.1.3", + "fast-glob": "^3.2.12", + "glob-parent": "^6.0.2", + "is-glob": "^4.0.3", + "jiti": "^1.17.2", + "lilconfig": "^2.0.6", + "micromatch": "^4.0.5", + "normalize-path": "^3.0.0", + "object-hash": "^3.0.0", + "picocolors": "^1.0.0", + "postcss": "^8.0.9", + "postcss-import": "^14.1.0", + "postcss-js": "^4.0.0", + "postcss-load-config": "^3.1.4", + "postcss-nested": "6.0.0", + "postcss-selector-parser": "^6.0.11", + "postcss-value-parser": "^4.2.0", + "quick-lru": "^5.1.1", + "resolve": "^1.22.1", + "sucrase": "^3.29.0" + }, + "bin": { + "tailwind": "lib/cli.js", + "tailwindcss": "lib/cli.js" + }, + "engines": { + "node": ">=12.13.0" + }, + "peerDependencies": { + "postcss": "^8.0.9" + } + }, + "node_modules/thenify": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", + "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", + "dev": true, + "dependencies": { + "any-promise": "^1.0.0" + } + }, + "node_modules/thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", + "dev": true, + "dependencies": { + "thenify": ">= 3.1.0 < 4" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/ts-interface-checker": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", + "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", + "dev": true + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "node_modules/yaml": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", + "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "dev": true, + "engines": { + "node": ">= 6" + } + } + }, + "dependencies": { + "@jridgewell/gen-mapping": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", + "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==", + "dev": true, + "requires": { + "@jridgewell/set-array": "^1.0.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.9" + } + }, + "@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "dev": true + }, + "@jridgewell/set-array": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", + "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "dev": true + }, + "@jridgewell/sourcemap-codec": { + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", + "dev": true + }, + "@jridgewell/trace-mapping": { + "version": "0.3.18", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz", + "integrity": "sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==", + "dev": true, + "requires": { + "@jridgewell/resolve-uri": "3.1.0", + "@jridgewell/sourcemap-codec": "1.4.14" + }, + "dependencies": { + "@jridgewell/sourcemap-codec": { + "version": "1.4.14", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", + "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==", + "dev": true + } + } + }, + "@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + } + }, + "@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true + }, + "@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "requires": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + } + }, + "@tailwindcss/forms": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/forms/-/forms-0.5.3.tgz", + "integrity": "sha512-y5mb86JUoiUgBjY/o6FJSFZSEttfb3Q5gllE4xoKjAAD+vBrnIhE4dViwUuow3va8mpH4s9jyUbUbrRGoRdc2Q==", + "dev": true, + "requires": { + "mini-svg-data-uri": "^1.2.3" + } + }, + "any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==", + "dev": true + }, + "anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "requires": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + } + }, + "arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "dev": true + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "binary-extensions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dev": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "requires": { + "fill-range": "^7.0.1" + } + }, + "camelcase-css": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz", + "integrity": "sha512-QOSvevhslijgYwRx6Rv7zKdMF8lbRmx+uQGx2+vDc+KI/eBnsy9kit5aj23AgGu3pa4t9AgwbnXWqS+iOY+2aA==", + "dev": true + }, + "chokidar": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", + "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", + "dev": true, + "requires": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "fsevents": "~2.3.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "dependencies": { + "glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + } + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "dev": true + }, + "didyoumean": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", + "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==", + "dev": true + }, + "dlv": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/dlv/-/dlv-1.1.3.tgz", + "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", + "dev": true + }, + "fast-glob": { + "version": "3.2.12", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", + "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.4" + }, + "dependencies": { + "glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + } + } + }, + "fastq": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", + "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", + "dev": true, + "requires": { + "reusify": "^1.0.4" + } + }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true + }, + "fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "optional": true + }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "requires": { + "is-glob": "^4.0.3" + } + }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "requires": { + "function-bind": "^1.1.1" + } + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "requires": { + "binary-extensions": "^2.0.0" + } + }, + "is-core-module": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.0.tgz", + "integrity": "sha512-RECHCBCd/viahWmwj6enj19sKbHfJrddi/6cBDsNTKbNq0f7VeaUkBo60BqzvPqo/W54ChS62Z5qyun7cfOMqQ==", + "dev": true, + "requires": { + "has": "^1.0.3" + } + }, + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true + }, + "is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "requires": { + "is-extglob": "^2.1.1" + } + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true + }, + "jiti": { + "version": "1.18.2", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.18.2.tgz", + "integrity": "sha512-QAdOptna2NYiSSpv0O/BwoHBSmz4YhpzJHyi+fnMRTXFjp7B8i/YG5Z8IfusxB1ufjcD2Sre1F3R+nX3fvy7gg==", + "dev": true + }, + "lilconfig": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-2.1.0.tgz", + "integrity": "sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==", + "dev": true + }, + "lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true + }, + "merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true + }, + "micromatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", + "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "dev": true, + "requires": { + "braces": "^3.0.2", + "picomatch": "^2.3.1" + } + }, + "mini-svg-data-uri": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/mini-svg-data-uri/-/mini-svg-data-uri-1.4.4.tgz", + "integrity": "sha512-r9deDe9p5FJUPZAk3A59wGH7Ii9YrjjWw0jmw/liSbHl2CHiyXj6FcDXDu2K3TjVAXqiJdaw3xxwlZZr9E6nHg==", + "dev": true + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "mz": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "dev": true, + "requires": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, + "nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "dev": true + }, + "normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true + }, + "object-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", + "dev": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true + }, + "path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true + }, + "picocolors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", + "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", + "dev": true + }, + "picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true + }, + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "dev": true + }, + "pirates": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.5.tgz", + "integrity": "sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ==", + "dev": true + }, + "postcss": { + "version": "8.4.21", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.21.tgz", + "integrity": "sha512-tP7u/Sn/dVxK2NnruI4H9BG+x+Wxz6oeZ1cJ8P6G/PZY0IKk4k/63TDsQf2kQq3+qoJeLm2kIBUNlZe3zgb4Zg==", + "dev": true, + "requires": { + "nanoid": "^3.3.4", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + } + }, + "postcss-import": { + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/postcss-import/-/postcss-import-14.1.0.tgz", + "integrity": "sha512-flwI+Vgm4SElObFVPpTIT7SU7R3qk2L7PyduMcokiaVKuWv9d/U+Gm/QAd8NDLuykTWTkcrjOeD2Pp1rMeBTGw==", + "dev": true, + "requires": { + "postcss-value-parser": "^4.0.0", + "read-cache": "^1.0.0", + "resolve": "^1.1.7" + } + }, + "postcss-js": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/postcss-js/-/postcss-js-4.0.1.tgz", + "integrity": "sha512-dDLF8pEO191hJMtlHFPRa8xsizHaM82MLfNkUHdUtVEV3tgTp5oj+8qbEqYM57SLfc74KSbw//4SeJma2LRVIw==", + "dev": true, + "requires": { + "camelcase-css": "^2.0.1" + } + }, + "postcss-load-config": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-3.1.4.tgz", + "integrity": "sha512-6DiM4E7v4coTE4uzA8U//WhtPwyhiim3eyjEMFCnUpzbrkK9wJHgKDT2mR+HbtSrd/NubVaYTOpSpjUl8NQeRg==", + "dev": true, + "requires": { + "lilconfig": "^2.0.5", + "yaml": "^1.10.2" + } + }, + "postcss-nested": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/postcss-nested/-/postcss-nested-6.0.0.tgz", + "integrity": "sha512-0DkamqrPcmkBDsLn+vQDIrtkSbNkv5AD/M322ySo9kqFkCIYklym2xEmWkwo+Y3/qZo34tzEPNUw4y7yMCdv5w==", + "dev": true, + "requires": { + "postcss-selector-parser": "^6.0.10" + } + }, + "postcss-selector-parser": { + "version": "6.0.11", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.11.tgz", + "integrity": "sha512-zbARubNdogI9j7WY4nQJBiNqQf3sLS3wCP4WfOidu+p28LofJqDH1tcXypGrcmMHhDk2t9wGhCsYe/+szLTy1g==", + "dev": true, + "requires": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + } + }, + "postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true + }, + "queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true + }, + "quick-lru": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", + "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", + "dev": true + }, + "read-cache": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", + "integrity": "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA==", + "dev": true, + "requires": { + "pify": "^2.3.0" + } + }, + "readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "requires": { + "picomatch": "^2.2.1" + } + }, + "resolve": { + "version": "1.22.2", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz", + "integrity": "sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==", + "dev": true, + "requires": { + "is-core-module": "^2.11.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + } + }, + "reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true + }, + "run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "requires": { + "queue-microtask": "^1.2.2" + } + }, + "source-map-js": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", + "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "dev": true + }, + "sucrase": { + "version": "3.32.0", + "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.32.0.tgz", + "integrity": "sha512-ydQOU34rpSyj2TGyz4D2p8rbktIOZ8QY9s+DGLvFU1i5pWJE8vkpruCjGCMHsdXwnD7JDcS+noSwM/a7zyNFDQ==", + "dev": true, + "requires": { + "@jridgewell/gen-mapping": "^0.3.2", + "commander": "^4.0.0", + "glob": "7.1.6", + "lines-and-columns": "^1.1.6", + "mz": "^2.7.0", + "pirates": "^4.0.1", + "ts-interface-checker": "^0.1.9" + } + }, + "supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true + }, + "tailwindcss": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.3.1.tgz", + "integrity": "sha512-Vkiouc41d4CEq0ujXl6oiGFQ7bA3WEhUZdTgXAhtKxSy49OmKs8rEfQmupsfF0IGW8fv2iQkp1EVUuapCFrZ9g==", + "dev": true, + "requires": { + "arg": "^5.0.2", + "chokidar": "^3.5.3", + "color-name": "^1.1.4", + "didyoumean": "^1.2.2", + "dlv": "^1.1.3", + "fast-glob": "^3.2.12", + "glob-parent": "^6.0.2", + "is-glob": "^4.0.3", + "jiti": "^1.17.2", + "lilconfig": "^2.0.6", + "micromatch": "^4.0.5", + "normalize-path": "^3.0.0", + "object-hash": "^3.0.0", + "picocolors": "^1.0.0", + "postcss": "^8.0.9", + "postcss-import": "^14.1.0", + "postcss-js": "^4.0.0", + "postcss-load-config": "^3.1.4", + "postcss-nested": "6.0.0", + "postcss-selector-parser": "^6.0.11", + "postcss-value-parser": "^4.2.0", + "quick-lru": "^5.1.1", + "resolve": "^1.22.1", + "sucrase": "^3.29.0" + } + }, + "thenify": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", + "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", + "dev": true, + "requires": { + "any-promise": "^1.0.0" + } + }, + "thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==", + "dev": true, + "requires": { + "thenify": ">= 3.1.0 < 4" + } + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "requires": { + "is-number": "^7.0.0" + } + }, + "ts-interface-checker": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz", + "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==", + "dev": true + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + }, + "yaml": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", + "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "dev": true + } + } +} diff --git a/apps/web/package.json b/apps/web/package.json new file mode 100644 index 000000000..5112da096 --- /dev/null +++ b/apps/web/package.json @@ -0,0 +1,10 @@ +{ + "scripts": { + "build": "tailwindcss -i src-css/main.css -o ./public/main.css --minify", + "watch": "tailwindcss -i src-css/main.css -o ./public/main.css --minify --watch" + }, + "devDependencies": { + "@tailwindcss/forms": "^0.5.2", + "tailwindcss": "^3.0.24" + } +} diff --git a/apps/web/public/main.css b/apps/web/public/main.css new file mode 100644 index 000000000..b5ed38ab9 --- /dev/null +++ b/apps/web/public/main.css @@ -0,0 +1 @@ +/*! tailwindcss v3.3.1 | MIT License | https://tailwindcss.com*/*,:after,:before{box-sizing:border-box;border:0 solid #e5e7eb}:after,:before{--tw-content:""}html{line-height:1.5;-webkit-text-size-adjust:100%;-moz-tab-size:4;-o-tab-size:4;tab-size:4;font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol,Noto Color Emoji;font-feature-settings:normal;font-variation-settings:normal}body{margin:0;line-height:inherit}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,pre,samp{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:initial}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}button,input,optgroup,select,textarea{font-family:inherit;font-size:100%;font-weight:inherit;line-height:inherit;color:inherit;margin:0;padding:0}button,select{text-transform:none}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button;background-color:initial;background-image:none}:-moz-focusring{outline:auto}:-moz-ui-invalid{box-shadow:none}progress{vertical-align:initial}::-webkit-inner-spin-button,::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}summary{display:list-item}blockquote,dd,dl,figure,h1,h2,h3,h4,h5,h6,hr,p,pre{margin:0}fieldset{margin:0}fieldset,legend{padding:0}menu,ol,ul{list-style:none;margin:0;padding:0}textarea{resize:vertical}input::-moz-placeholder,textarea::-moz-placeholder{opacity:1;color:#9ca3af}input::placeholder,textarea::placeholder{opacity:1;color:#9ca3af}[role=button],button{cursor:pointer}:disabled{cursor:default}audio,canvas,embed,iframe,img,object,svg,video{display:block;vertical-align:middle}img,video{max-width:100%;height:auto}[hidden]{display:none}*,::backdrop,:after,:before{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:#3b82f680;--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: }.absolute{position:absolute}.relative{position:relative}.sticky{position:sticky}.top-0{top:0}.left-0{left:0}.z-50{z-index:50}.z-40{z-index:40}.order-1{order:1}.mx-auto{margin-left:auto;margin-right:auto}.-ml-16{margin-left:-4rem}.-mt-2{margin-top:-.5rem}.mb-2{margin-bottom:.5rem}.mb-\[-4px\]{margin-bottom:-4px}.ml-auto{margin-left:auto}.mr-1{margin-right:.25rem}.mt-2{margin-top:.5rem}.mb-6{margin-bottom:1.5rem}.mr-2{margin-right:.5rem}.inline-block{display:inline-block}.inline{display:inline}.flex{display:flex}.contents{display:contents}.hidden{display:none}.h-4{height:1rem}.h-5{height:1.25rem}.h-6{height:1.5rem}.h-\[48px\]{height:48px}.h-screen{height:100vh}.h-8{height:2rem}.h-full{height:100%}.max-h-\[640px\]{max-height:640px}.min-h-\[128px\]{min-height:128px}.w-2{width:.5rem}.w-4{width:1rem}.w-5{width:1.25rem}.w-6{width:1.5rem}.w-\[48px\]{width:48px}.w-48{width:12rem}.w-full{width:100%}.flex-1{flex:1 1 0%}.flex-none{flex:none}.grow{flex-grow:1}@keyframes fade-in{0%{opacity:0;transform:translateY(10px)}to{opacity:1;transform:translateY(0)}}.animate-fade-in{animation:fade-in .5s ease-out}@keyframes pulse{50%{opacity:.5}}.animate-pulse{animation:pulse 2s cubic-bezier(.4,0,.6,1) infinite}@keyframes spin{to{transform:rotate(1turn)}}.animate-spin{animation:spin 1s linear infinite}.cursor-help{cursor:help}.cursor-pointer{cursor:pointer}.resize{resize:both}.flex-row{flex-direction:row}.flex-col{flex-direction:column}.flex-nowrap{flex-wrap:nowrap}.place-content-end{place-content:end}.items-center{align-items:center}.gap-1{gap:.25rem}.gap-2{gap:.5rem}.gap-4{gap:1rem}.gap-8{gap:2rem}.overflow-hidden{overflow:hidden}.overflow-y-auto{overflow-y:auto}.overflow-x-hidden{overflow-x:hidden}.overflow-y-scroll{overflow-y:scroll}.rounded{border-radius:.25rem}.rounded-full{border-radius:9999px}.rounded-md{border-radius:.375rem}.rounded-xl{border-radius:.75rem}.border{border-width:1px}.border-2{border-width:2px}.border-b-2{border-bottom-width:2px}.border-t{border-top-width:1px}.border-t-2{border-top-width:2px}.border-cyan-600{--tw-border-opacity:1;border-color:rgb(8 145 178/var(--tw-border-opacity))}.border-neutral-600{--tw-border-opacity:1;border-color:rgb(82 82 82/var(--tw-border-opacity))}.border-red-700{--tw-border-opacity:1;border-color:rgb(185 28 28/var(--tw-border-opacity))}.border-neutral-900{--tw-border-opacity:1;border-color:rgb(23 23 23/var(--tw-border-opacity))}.border-t-neutral-700{--tw-border-opacity:1;border-top-color:rgb(64 64 64/var(--tw-border-opacity))}.bg-cyan-600{--tw-bg-opacity:1;background-color:rgb(8 145 178/var(--tw-bg-opacity))}.bg-cyan-900{--tw-bg-opacity:1;background-color:rgb(22 78 99/var(--tw-bg-opacity))}.bg-green-700{--tw-bg-opacity:1;background-color:rgb(21 128 61/var(--tw-bg-opacity))}.bg-neutral-700{--tw-bg-opacity:1;background-color:rgb(64 64 64/var(--tw-bg-opacity))}.bg-neutral-800{--tw-bg-opacity:1;background-color:rgb(38 38 38/var(--tw-bg-opacity))}.bg-neutral-900{--tw-bg-opacity:1;background-color:rgb(23 23 23/var(--tw-bg-opacity))}.bg-transparent{background-color:initial}.bg-stone-900{--tw-bg-opacity:1;background-color:rgb(28 25 23/var(--tw-bg-opacity))}.bg-cyan-700{--tw-bg-opacity:1;background-color:rgb(14 116 144/var(--tw-bg-opacity))}.p-4{padding:1rem}.p-2{padding:.5rem}.p-0{padding:0}.px-2{padding-left:.5rem;padding-right:.5rem}.px-3{padding-left:.75rem;padding-right:.75rem}.px-4{padding-left:1rem;padding-right:1rem}.py-1{padding-top:.25rem;padding-bottom:.25rem}.py-2{padding-top:.5rem;padding-bottom:.5rem}.py-4{padding-top:1rem;padding-bottom:1rem}.px-8{padding-left:2rem;padding-right:2rem}.py-3{padding-top:.75rem;padding-bottom:.75rem}.px-1{padding-left:.25rem;padding-right:.25rem}.py-0{padding-top:0;padding-bottom:0}.py-0\.5{padding-top:.125rem;padding-bottom:.125rem}.pb-2{padding-bottom:.5rem}.pl-3{padding-left:.75rem}.text-left{text-align:left}.text-center{text-align:center}.text-right{text-align:right}.text-base{font-size:.875rem}.text-lg{font-size:1rem}.text-sm{font-size:.75rem}.text-xl{font-size:1.125rem}.text-xs{font-size:.625rem}.text-5xl{font-size:2rem}.text-2xl{font-size:1.25rem}.font-semibold{font-weight:600}.font-bold{font-weight:700}.uppercase{text-transform:uppercase}.leading-5{line-height:1.25rem}.text-cyan-500{--tw-text-opacity:1;color:rgb(6 182 212/var(--tw-text-opacity))}.text-neutral-400{--tw-text-opacity:1;color:rgb(163 163 163/var(--tw-text-opacity))}.text-neutral-500{--tw-text-opacity:1;color:rgb(115 115 115/var(--tw-text-opacity))}.text-red-500{--tw-text-opacity:1;color:rgb(239 68 68/var(--tw-text-opacity))}.text-stone-400{--tw-text-opacity:1;color:rgb(168 162 158/var(--tw-text-opacity))}.text-white{--tw-text-opacity:1;color:rgb(255 255 255/var(--tw-text-opacity))}.text-yellow-400{--tw-text-opacity:1;color:rgb(250 204 21/var(--tw-text-opacity))}.text-gray-500{--tw-text-opacity:1;color:rgb(107 114 128/var(--tw-text-opacity))}.text-cyan-600{--tw-text-opacity:1;color:rgb(8 145 178/var(--tw-text-opacity))}.text-yellow-500{--tw-text-opacity:1;color:rgb(234 179 8/var(--tw-text-opacity))}.underline{text-decoration-line:underline}.placeholder-neutral-300::-moz-placeholder{--tw-placeholder-opacity:1;color:rgb(212 212 212/var(--tw-placeholder-opacity))}.placeholder-neutral-300::placeholder{--tw-placeholder-opacity:1;color:rgb(212 212 212/var(--tw-placeholder-opacity))}.placeholder-neutral-600::-moz-placeholder{--tw-placeholder-opacity:1;color:rgb(82 82 82/var(--tw-placeholder-opacity))}.placeholder-neutral-600::placeholder{--tw-placeholder-opacity:1;color:rgb(82 82 82/var(--tw-placeholder-opacity))}.caret-white{caret-color:#fff}.outline-none{outline:2px solid #0000;outline-offset:2px}.filter{filter:var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow)}input:checked~.dot{transform:translateX(100%);background-color:#48bb78}*{scrollbar-width:thin;scrollbar-color:#404040 #171717}::-webkit-scrollbar{width:10px}::-webkit-scrollbar-track{background:#171717}::-webkit-scrollbar-thumb{background:#404040;border-radius:100vh;border:2px solid #171717}::-webkit-scrollbar-thumb:hover{background:#164e63}mark{color:#fff;background-color:rgb(14 116 144/var(--tw-bg-opacity));padding-left:.125rem;padding-right:.125rem}.hover\:bg-cyan-600:hover{--tw-bg-opacity:1;background-color:rgb(8 145 178/var(--tw-bg-opacity))}.hover\:bg-green-900:hover{--tw-bg-opacity:1;background-color:rgb(20 83 45/var(--tw-bg-opacity))}.hover\:bg-neutral-600:hover{--tw-bg-opacity:1;background-color:rgb(82 82 82/var(--tw-bg-opacity))}.hover\:bg-red-700:hover{--tw-bg-opacity:1;background-color:rgb(185 28 28/var(--tw-bg-opacity))}.hover\:bg-neutral-700:hover{--tw-bg-opacity:1;background-color:rgb(64 64 64/var(--tw-bg-opacity))}.hover\:bg-cyan-900:hover{--tw-bg-opacity:1;background-color:rgb(22 78 99/var(--tw-bg-opacity))}.hover\:bg-cyan-800:hover{--tw-bg-opacity:1;background-color:rgb(21 94 117/var(--tw-bg-opacity))}.hover\:text-white:hover{--tw-text-opacity:1;color:rgb(255 255 255/var(--tw-text-opacity))}.focus\:outline-none:focus{outline:2px solid #0000;outline-offset:2px}.active\:bg-neutral-700:active{--tw-bg-opacity:1;background-color:rgb(64 64 64/var(--tw-bg-opacity))}.active\:outline-none:active{outline:2px solid #0000;outline-offset:2px}.group:hover .group-hover\:block{display:block}.group:hover .group-hover\:text-neutral-400{--tw-text-opacity:1;color:rgb(163 163 163/var(--tw-text-opacity))} \ No newline at end of file diff --git a/apps/web/src-css/main.css b/apps/web/src-css/main.css new file mode 100644 index 000000000..afbaecb1d --- /dev/null +++ b/apps/web/src-css/main.css @@ -0,0 +1,38 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +input:checked ~ .dot { + transform: translateX(100%); + background-color: #48bb78; +} + +* { + scrollbar-width: thin; + scrollbar-color: #404040 #171717; +} + +*::-webkit-scrollbar { + width: 10px; +} + +*::-webkit-scrollbar-track { + background: #171717; +} + +*::-webkit-scrollbar-thumb { + background: #404040; + border-radius: 100vh; + border: 2px solid #171717; +} + +*::-webkit-scrollbar-thumb:hover { + background: #164e63; +} + +mark { + color: rgb(255 255 255); + background-color: rgb(14 116 144 / var(--tw-bg-opacity)); + padding-left: 0.125rem; + padding-right: 0.125rem; +} \ No newline at end of file diff --git a/apps/web/src/main.rs b/apps/web/src/main.rs new file mode 100644 index 000000000..98a1330ad --- /dev/null +++ b/apps/web/src/main.rs @@ -0,0 +1,47 @@ +use wasm_bindgen::{prelude::Closure, JsValue}; +use yew::prelude::*; +use yew_router::prelude::*; + +mod pages; +use pages::AppPage; + +#[derive(Clone, Routable, PartialEq)] +pub enum Route { + #[at("/")] + Start, + #[at("/result")] + Result, + #[at("/library")] + MyLibrary, + #[not_found] + #[at("/404")] + NotFound, +} + +fn switch(routes: Route) -> Html { + if routes == Route::NotFound { + html! {
{"Not Found!"}
} + } else { + html! { + + } + } +} + +pub async fn listen(_event_name: &str, _cb: &Closure) -> Result { + Ok(JsValue::NULL) +} + +#[function_component] +fn App() -> Html { + html! { + + render={switch} /> + + } +} + +fn main() { + let _ = console_log::init_with_level(log::Level::Debug); + yew::Renderer::::new().render(); +} diff --git a/apps/web/src/pages/mod.rs b/apps/web/src/pages/mod.rs new file mode 100644 index 000000000..c4b3ac3ce --- /dev/null +++ b/apps/web/src/pages/mod.rs @@ -0,0 +1,77 @@ +use ui_components::icons; +use yew::prelude::*; +use yew_router::prelude::Link; + +use crate::Route; +pub mod search; + +#[derive(PartialEq, Properties)] +pub struct NavLinkProps { + tab: Route, + children: Children, + current: Route, +} + +#[function_component(NavLink)] +pub fn nav_link(props: &NavLinkProps) -> Html { + let link_styles = classes!( + "flex-row", + "flex", + "hover:bg-neutral-700", + "items-center", + "p-2", + "rounded", + "w-full", + (props.current == props.tab).then_some(Some("bg-neutral-700")) + ); + + html! { + classes={link_styles} to={props.tab.clone()}> + {props.children.clone()} + > + } +} + +#[derive(Properties, PartialEq)] +pub struct AppPageProps { + pub tab: Route, +} + +#[function_component] +pub fn AppPage(props: &AppPageProps) -> Html { + html! { +
+
+
+
+ {"Spyglass"} +
+
    +
  • + + + {"My Library"} + +
    • +
+
+
+
+ {"Searches"} +
+
    +
  • + + + {"Search"} + +
    • +
+
+
+
+ +
+
+ } +} diff --git a/apps/web/src/pages/search.rs b/apps/web/src/pages/search.rs new file mode 100644 index 000000000..b9edd796d --- /dev/null +++ b/apps/web/src/pages/search.rs @@ -0,0 +1,88 @@ +use std::str::FromStr; + +use shared::keyboard::KeyCode; +use ui_components::btn::{Btn, BtnType}; +use web_sys::HtmlInputElement; +use yew::prelude::*; + +#[derive(Clone, Debug)] +pub enum Msg { + HandleKeyboardEvent(KeyboardEvent), + HandleSearch, +} + +pub struct SearchPage { + search_wrapper_ref: NodeRef, + search_input_ref: NodeRef, + status_msg: Option, +} + +impl Component for SearchPage { + type Message = Msg; + type Properties = (); + + fn create(_ctx: &yew::Context) -> Self { + Self { + search_input_ref: Default::default(), + search_wrapper_ref: Default::default(), + status_msg: None, + } + } + + fn update(&mut self, ctx: &yew::Context, msg: Self::Message) -> bool { + let link = ctx.link(); + match msg { + Msg::HandleKeyboardEvent(event) => { + let key = event.key(); + if let Ok(code) = KeyCode::from_str(&key.to_uppercase()) { + if code == KeyCode::Enter { + log::info!("key-code: {code}"); + link.send_message(Msg::HandleSearch); + } + } + } + Msg::HandleSearch => { + let query = self + .search_input_ref + .cast::() + .map(|x| x.value()); + + log::info!("handling search! {:?}", query); + if let Some(query) = query { + self.status_msg = Some(format!("searching: {query}")); + } + } + } + false + } + + fn view(&self, ctx: &yew::Context) -> yew::Html { + let link = ctx.link(); + + html! { +
+
+ + + {"Search"} + +
+
+ {self.status_msg.clone().unwrap_or_else(|| "how to guide?".into())} +
+
+ } + } +} diff --git a/apps/web/tailwind.config.js b/apps/web/tailwind.config.js new file mode 100644 index 000000000..7cac5cbe1 --- /dev/null +++ b/apps/web/tailwind.config.js @@ -0,0 +1,54 @@ +module.exports = { + content: [ + "./src/**/*.{html,js,rs}", + "./*.html", + "../../crates/ui-components/**/*.rs" + ], + theme: { + minWidth: { + "4": "1rem", + "5": "1.25rem" + }, + extend: { + keyframes: { + "fade-in": { + "0%": { + opacity: "0", + transform: "translateY(10px)", + }, + "100%": { + opacity: "1", + transform: "translateY(0)", + }, + }, + wiggle: { + "0%, 100%": { transform: "rotate(-6deg)" }, + "50%": { transform: "rotate(6deg)" }, + } + }, + animation: { + "fade-in": "fade-in 0.5s ease-out", + "wiggle-short": "wiggle 1s ease-in-out 10", + "wiggle": "wiggle 1s ease-in-out infinite", + } + }, + fontSize: { + // 10px + xs: "0.625rem", + // 12px + sm: "0.75rem", + // 14px + base: "0.875rem", + // 16px + lg: "1rem", + xl: "1.125rem", + "2xl": "1.25rem", + "3xl": "1.5rem", + "4xl": "1.875rem", + "5xl": "2rem", + } + }, + plugins: [ + require("@tailwindcss/forms")({ strategy: "class" }) + ], +} diff --git a/crates/client/Cargo.toml b/crates/client/Cargo.toml index b54579f1d..36a8eb2e1 100644 --- a/crates/client/Cargo.toml +++ b/crates/client/Cargo.toml @@ -13,7 +13,7 @@ handlebars = "4.3.6" num-format = { version = "0.4", default-features = false } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -serde-wasm-bindgen = "0.4" +serde-wasm-bindgen = "0.5" "shared" = { path = "../shared" } strum = "0.24" strum_macros = "0.24" @@ -23,4 +23,5 @@ wasm-logger = "0.2.0" web-sys = { version = "0.3.60", features = ["Navigator", "VisibilityState"] } yew = { version = "0.20", features = ["csr"] } yew-router = "0.17" +ui-components = { path = "../ui-components" } url = "2.2.2" \ No newline at end of file diff --git a/crates/client/src/components/btn.rs b/crates/client/src/components/btn.rs index 3d037902c..e997543aa 100644 --- a/crates/client/src/components/btn.rs +++ b/crates/client/src/components/btn.rs @@ -2,76 +2,7 @@ use shared::event::OpenResultParams; use wasm_bindgen_futures::spawn_local; use yew::prelude::*; -use crate::{ - components::{icons, tooltip::Tooltip}, - tauri_invoke, -}; - -#[derive(Properties, PartialEq, Eq)] -pub struct DeleteButtonProps { - pub doc_id: String, -} - -#[function_component(DeleteButton)] -pub fn delete_btn(props: &DeleteButtonProps) -> Html { - let onclick = { - let doc_id = props.doc_id.clone(); - Callback::from(move |e: MouseEvent| { - e.prevent_default(); - e.stop_immediate_propagation(); - - let doc_id = doc_id.clone(); - spawn_local(async move { - let _ = crate::delete_doc(doc_id.clone()).await; - }); - }) - }; - - html! { - - } -} - -#[derive(Properties, PartialEq)] -pub struct RecrawlButtonProps { - pub domain: String, - pub onrecrawl: Option>, -} - -#[function_component(RecrawlButton)] -pub fn recrawl_button(props: &RecrawlButtonProps) -> Html { - let onclick = { - let domain = props.domain.clone(); - let callback = props.onrecrawl.clone(); - - Callback::from(move |me| { - let domain = domain.clone(); - let callback = callback.clone(); - - spawn_local(async move { - let _ = crate::recrawl_domain(domain.clone()).await; - }); - - if let Some(callback) = callback { - callback.emit(me); - } - }) - }; - - html! { - - } -} +use crate::tauri_invoke; #[allow(dead_code)] #[derive(Clone, PartialEq, Eq)] diff --git a/crates/client/src/components/forms/pathlist.rs b/crates/client/src/components/forms/pathlist.rs index ad36fe9ff..6386bfb7d 100644 --- a/crates/client/src/components/forms/pathlist.rs +++ b/crates/client/src/components/forms/pathlist.rs @@ -45,7 +45,9 @@ impl Component for PathField { let link = link.clone(); spawn_local(async move { let cb = Closure::wrap(Box::new(move |payload: JsValue| { - if let Ok(res) = serde_wasm_bindgen::from_value::(payload) { + if let Ok(res) = + serde_wasm_bindgen::from_value::>(payload) + { link.send_message(PathMsg::UpdatePath( Path::new(&res.payload).to_path_buf(), )); @@ -176,7 +178,9 @@ impl Component for PathList { let link = link.clone(); spawn_local(async move { let cb = Closure::wrap(Box::new(move |payload: JsValue| { - if let Ok(res) = serde_wasm_bindgen::from_value::(payload) { + if let Ok(res) = + serde_wasm_bindgen::from_value::>(payload) + { link.send_message(Msg::AddPath(Path::new(&res.payload).to_path_buf())); } }) as Box); diff --git a/crates/client/src/components/mod.rs b/crates/client/src/components/mod.rs index 208b0c446..b9a90fe16 100644 --- a/crates/client/src/components/mod.rs +++ b/crates/client/src/components/mod.rs @@ -1,12 +1,10 @@ pub mod btn; pub mod forms; -pub mod icons; pub mod lens; pub mod result; -pub mod tag; -pub mod tooltip; pub mod user_action_list; use shared::keyboard::ModifiersState; +use ui_components::icons; use yew::{prelude::*, virtual_dom::AttrValue}; use crate::utils::{self, OsName}; diff --git a/crates/client/src/components/result.rs b/crates/client/src/components/result.rs index d665a1872..4047a8ec7 100644 --- a/crates/client/src/components/result.rs +++ b/crates/client/src/components/result.rs @@ -3,13 +3,13 @@ use url::Url; use yew::{platform::spawn_local, prelude::*}; use yew_router::Routable; -use super::{ - btn, icons, - tag::{Tag, TagIcon}, -}; +use super::btn; + use crate::{pages::Tab, tauri_invoke, Route}; use shared::response::{LensResult, SearchResult}; use shared::{constants::FEEDBACK_FORM, event}; +use ui_components::icons; +use ui_components::tag::{Tag, TagIcon}; #[derive(Properties, PartialEq)] pub struct SearchResultProps { diff --git a/crates/client/src/pages/admin.rs b/crates/client/src/pages/admin.rs index e83151692..c98aa10df 100644 --- a/crates/client/src/pages/admin.rs +++ b/crates/client/src/pages/admin.rs @@ -1,11 +1,11 @@ use strum_macros::{Display, EnumString}; +use ui_components::icons; use wasm_bindgen::prelude::*; use wasm_bindgen_futures::spawn_local; use yew::{classes, prelude::*, Children}; use yew_router::components::Link; use yew_router::hooks::use_navigator; -use crate::components::icons; use crate::{listen, pages, Route}; use shared::event::{ClientEvent, ListenPayload}; @@ -68,7 +68,7 @@ pub fn settings_page(props: &SettingsPageProps) -> Html { spawn_local(async move { let cb = Closure::wrap(Box::new(move |payload: JsValue| { - if let Ok(payload) = serde_wasm_bindgen::from_value::(payload) { + if let Ok(payload) = serde_wasm_bindgen::from_value::>(payload) { match payload.payload.as_str() { "/settings/discover" => history.push(&Route::SettingsPage { tab: pages::Tab::Discover, diff --git a/crates/client/src/pages/connection_manager.rs b/crates/client/src/pages/connection_manager.rs index 3639448b7..fff073c58 100644 --- a/crates/client/src/pages/connection_manager.rs +++ b/crates/client/src/pages/connection_manager.rs @@ -5,6 +5,7 @@ use gloo::timers::callback::Interval; use shared::event::{AuthorizeConnectionParams, ClientEvent, ClientInvoke, ResyncConnectionParams}; use shared::response::{ListConnectionResult, SupportedConnection, UserConnection}; +use ui_components::icons; use wasm_bindgen::prelude::*; use wasm_bindgen_futures::spawn_local; use yew::prelude::*; @@ -12,7 +13,7 @@ use yew::prelude::*; use crate::components::{ btn, btn::{BtnSize, BtnType}, - icons, Header, + Header, }; use crate::utils::RequestState; use crate::{listen, tauri_invoke}; diff --git a/crates/client/src/pages/discover.rs b/crates/client/src/pages/discover.rs index 75f5448b3..f094de230 100644 --- a/crates/client/src/pages/discover.rs +++ b/crates/client/src/pages/discover.rs @@ -1,13 +1,13 @@ use shared::event::{ClientEvent, ClientInvoke, InstallLensParams}; use shared::response::{InstallableLens, LensResult}; use std::collections::{HashMap, HashSet}; +use ui_components::icons; use wasm_bindgen::prelude::*; use wasm_bindgen_futures::spawn_local; use web_sys::{HtmlElement, HtmlInputElement}; use yew::prelude::*; -use crate::components::lens::LensEvent; -use crate::components::{icons, lens::LibraryLens}; +use crate::components::lens::{LensEvent, LibraryLens}; use crate::invoke; use crate::utils::RequestState; diff --git a/crates/client/src/pages/library.rs b/crates/client/src/pages/library.rs index f5eaf24d9..8179e8e20 100644 --- a/crates/client/src/pages/library.rs +++ b/crates/client/src/pages/library.rs @@ -6,7 +6,6 @@ use yew::prelude::*; use crate::components::{ btn::Btn, - icons, lens::{LensEvent, LibraryLens}, Header, }; @@ -15,6 +14,7 @@ use crate::{invoke, listen, tauri_invoke}; use shared::event::ClientInvoke; use shared::event::{ClientEvent, UninstallLensParams}; use shared::response::LensResult; +use ui_components::icons; async fn fetch_user_installed_lenses() -> Option> { match invoke(ClientInvoke::ListInstalledLenses.as_ref(), JsValue::NULL).await { diff --git a/crates/client/src/pages/plugin_manager.rs b/crates/client/src/pages/plugin_manager.rs index ae4e46dda..c9490b69f 100644 --- a/crates/client/src/pages/plugin_manager.rs +++ b/crates/client/src/pages/plugin_manager.rs @@ -7,9 +7,10 @@ use yew::prelude::*; use shared::event::ClientInvoke; use shared::response::PluginResult; +use ui_components::icons; use crate::components::forms::Toggle; -use crate::components::{icons, Header}; +use crate::components::Header; use crate::utils::RequestState; use crate::{invoke, listen, tauri_invoke}; diff --git a/crates/client/src/pages/search.rs b/crates/client/src/pages/search.rs index f8668df90..f03e76a98 100644 --- a/crates/client/src/pages/search.rs +++ b/crates/client/src/pages/search.rs @@ -13,10 +13,10 @@ use shared::{ event::{ClientEvent, ClientInvoke, OpenResultParams}, response::{self, SearchMeta, SearchResult, SearchResults}, }; +use ui_components::icons; use crate::components::user_action_list::{self, ActionListBtn, ActionsList, DEFAULT_ACTION_LABEL}; use crate::components::{ - icons, result::{FeedbackResult, LensResultItem, SearchResultItem}, KeyComponent, SelectedLens, }; diff --git a/crates/client/src/pages/settings.rs b/crates/client/src/pages/settings.rs index 723a11889..195c49e06 100644 --- a/crates/client/src/pages/settings.rs +++ b/crates/client/src/pages/settings.rs @@ -3,13 +3,10 @@ use wasm_bindgen_futures::spawn_local; use yew::prelude::*; use crate::components::forms::{FormElement, SettingChangeEvent}; -use crate::{ - components::{btn, icons}, - save_user_settings, tauri_invoke, - utils::RequestState, -}; +use crate::{components::btn, save_user_settings, tauri_invoke, utils::RequestState}; use shared::event::ClientInvoke; use shared::form::SettingOpts; +use ui_components::icons; #[derive(Clone)] pub enum Msg { diff --git a/crates/client/src/pages/startup.rs b/crates/client/src/pages/startup.rs index 91d603bba..3231a6db2 100644 --- a/crates/client/src/pages/startup.rs +++ b/crates/client/src/pages/startup.rs @@ -4,8 +4,8 @@ use wasm_bindgen_futures::spawn_local; use yew::prelude::*; use shared::event::ClientInvoke; +use ui_components::icons; -use crate::components::icons; use crate::invoke; pub struct StartupPage { diff --git a/crates/client/src/pages/updater.rs b/crates/client/src/pages/updater.rs index bf8995e3c..f2e09f976 100644 --- a/crates/client/src/pages/updater.rs +++ b/crates/client/src/pages/updater.rs @@ -3,9 +3,10 @@ use wasm_bindgen::JsValue; use wasm_bindgen_futures::spawn_local; use yew::prelude::*; -use crate::components::{btn::Btn, icons, Header}; +use crate::components::{btn::Btn, Header}; use crate::invoke; use shared::event::ClientInvoke; +use ui_components::icons; // Random gif for your viewing pleasure. const UPDATE_GIFS: [&str; 5] = [ diff --git a/crates/client/src/pages/wizard/indexing_help.rs b/crates/client/src/pages/wizard/indexing_help.rs index 8043ad4f9..4ee9eab8d 100644 --- a/crates/client/src/pages/wizard/indexing_help.rs +++ b/crates/client/src/pages/wizard/indexing_help.rs @@ -1,9 +1,9 @@ use super::btn; +use crate::components::forms; use crate::components::{ btn::{BtnAlign, BtnSize}, forms::SettingChangeEvent, }; -use crate::components::{forms, icons}; use crate::tauri_invoke; use shared::{ constants::{CHROME_EXT_LINK, FIREFOX_EXT_LINK}, @@ -11,6 +11,7 @@ use shared::{ form::{FormType, SettingOpts}, response::DefaultIndices, }; +use ui_components::icons; use yew::platform::spawn_local; use yew::prelude::*; use yew::virtual_dom::VNode; diff --git a/crates/client/src/pages/wizard/mod.rs b/crates/client/src/pages/wizard/mod.rs index f1c0ad09e..91907743b 100644 --- a/crates/client/src/pages/wizard/mod.rs +++ b/crates/client/src/pages/wizard/mod.rs @@ -3,9 +3,10 @@ use wasm_bindgen_futures::spawn_local; use yew::prelude::*; use yew_router::hooks::use_navigator; -use crate::components::{btn, forms::SettingChangeEvent, icons}; +use crate::components::{btn, forms::SettingChangeEvent}; use crate::{tauri_invoke, Route}; use shared::event::{ClientInvoke, WizardFinishedParams}; +use ui_components::icons; mod display_searchbar; mod indexing_help; diff --git a/crates/shared/src/event.rs b/crates/shared/src/event.rs index 279eb4ba5..2fe87b61e 100644 --- a/crates/shared/src/event.rs +++ b/crates/shared/src/event.rs @@ -2,8 +2,8 @@ use serde::{Deserialize, Serialize}; use strum_macros::{AsRefStr, Display}; #[derive(Clone, Debug, Deserialize)] -pub struct ListenPayload { - pub payload: String, +pub struct ListenPayload { + pub payload: T, } #[derive(AsRefStr, Display)] diff --git a/crates/ui-components/Cargo.toml b/crates/ui-components/Cargo.toml new file mode 100644 index 000000000..c28cf8b87 --- /dev/null +++ b/crates/ui-components/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "ui-components" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +wasm-bindgen-futures = "0.4.33" +yew = { version = "0.20", features = ["csr"] } \ No newline at end of file diff --git a/crates/ui-components/src/btn.rs b/crates/ui-components/src/btn.rs new file mode 100644 index 000000000..b095dfb68 --- /dev/null +++ b/crates/ui-components/src/btn.rs @@ -0,0 +1,163 @@ +use yew::prelude::*; + +#[allow(dead_code)] +#[derive(Clone, PartialEq, Eq)] +pub enum BtnAlign { + Left, + Right, + Center, +} + +impl Default for BtnAlign { + fn default() -> Self { + Self::Center + } +} + +#[derive(Clone, PartialEq, Eq)] +pub enum BtnType { + Default, + Danger, + Success, + Primary, +} + +impl Default for BtnType { + fn default() -> Self { + Self::Default + } +} + +#[allow(dead_code)] +#[derive(PartialEq, Eq)] +pub enum BtnSize { + Xs, + Sm, + Base, + Lg, + Xl, +} + +impl Default for BtnSize { + fn default() -> Self { + Self::Base + } +} + +#[derive(Properties, PartialEq)] +pub struct DefaultBtnProps { + #[prop_or_default] + pub _type: BtnType, + #[prop_or_default] + pub size: BtnSize, + #[prop_or_default] + pub align: BtnAlign, + #[prop_or_default] + pub onclick: Callback, + #[prop_or_default] + pub disabled: bool, + #[prop_or_default] + pub children: Children, + #[prop_or_default] + pub href: Option, + #[prop_or_default] + pub classes: Classes, +} + +#[function_component(Btn)] +pub fn default_button(props: &DefaultBtnProps) -> Html { + let mut colors = match props._type { + BtnType::Default => classes!( + "border-neutral-600", + "border", + "hover:bg-neutral-600", + "active:bg-neutral-700", + "text-white", + ), + BtnType::Danger => classes!( + "border", + "border-red-700", + "hover:bg-red-700", + "text-red-500", + "hover:text-white" + ), + BtnType::Success => classes!("bg-green-700", "hover:bg-green-900"), + BtnType::Primary => classes!("bg-cyan-600", "hover:bg-cyan-800"), + }; + + if props.disabled { + colors.push("text-stone-400"); + } + + let sizes = match props.size { + BtnSize::Xs => classes!("text-xs", "px-2", "py-1"), + BtnSize::Sm => classes!("text-sm", "px-2", "py-1"), + BtnSize::Base => classes!("text-base", "px-3", "py-2"), + BtnSize::Lg => classes!("text-lg", "px-3", "py-2"), + BtnSize::Xl => classes!("text-xl", "px-4", "py-4"), + }; + + let styles = classes!( + props.classes.clone(), + colors, + sizes, + "cursor-pointer", + "flex-row", + "flex", + "font-semibold", + "items-center", + "leading-5", + "rounded-md", + ); + + let is_confirmed = use_state(|| false); + + let confirmed_state = is_confirmed.clone(); + let prop_onclick = props.onclick.clone(); + let btn_type = props._type.clone(); + + let handle_onclick = Callback::from(move |evt| { + // Handle confirmation for danger buttons + if btn_type == BtnType::Danger { + if *confirmed_state { + prop_onclick.emit(evt); + } else { + confirmed_state.set(true); + } + } else { + prop_onclick.emit(evt); + } + }); + + let label = if props._type == BtnType::Danger && *is_confirmed { + Children::new(vec![html! { <>{"⚠️ Click to confirm"} }]) + } else { + props.children.clone() + }; + + let mut label_styles = classes!("flex", "flex-row", "gap-1", "items-center",); + + match &props.align { + BtnAlign::Left => {} + BtnAlign::Right => label_styles.push("ml-auto"), + BtnAlign::Center => label_styles.push("mx-auto"), + } + + if props.href.is_none() { + html! { + + } + } else { + html! { + +
+ {label} +
+ + } + } +} diff --git a/crates/client/src/components/icons/browser.rs b/crates/ui-components/src/icons/browser.rs similarity index 100% rename from crates/client/src/components/icons/browser.rs rename to crates/ui-components/src/icons/browser.rs diff --git a/crates/client/src/components/icons.rs b/crates/ui-components/src/icons/mod.rs similarity index 98% rename from crates/client/src/components/icons.rs rename to crates/ui-components/src/icons/mod.rs index c5381d10b..7dd661cd8 100644 --- a/crates/client/src/components/icons.rs +++ b/crates/ui-components/src/icons/mod.rs @@ -369,6 +369,15 @@ pub fn down_arrow(props: &IconProps) -> Html { } } +#[function_component(Warning)] +pub fn warning_icon(props: &IconProps) -> Html { + html! { + + + + } +} + #[derive(Properties, PartialEq)] pub struct FileExtIconProps { pub ext: String, diff --git a/crates/ui-components/src/lib.rs b/crates/ui-components/src/lib.rs new file mode 100644 index 000000000..3f6e6a7b6 --- /dev/null +++ b/crates/ui-components/src/lib.rs @@ -0,0 +1,4 @@ +pub mod btn; +pub mod icons; +pub mod tag; +pub mod tooltip; diff --git a/crates/client/src/components/tag.rs b/crates/ui-components/src/tag.rs similarity index 100% rename from crates/client/src/components/tag.rs rename to crates/ui-components/src/tag.rs diff --git a/crates/client/src/components/tooltip.rs b/crates/ui-components/src/tooltip.rs similarity index 100% rename from crates/client/src/components/tooltip.rs rename to crates/ui-components/src/tooltip.rs From fff617d74a7d70e0ad3c27a1853bb2b127c54f98 Mon Sep 17 00:00:00 2001 From: travolin Date: Sat, 15 Apr 2023 15:34:43 -0700 Subject: [PATCH 07/30] Move helper function (#435) Co-authored-by: Joel Bredeson --- crates/spyglass/src/api/handler/search.rs | 144 ++-------------------- crates/spyglass/src/search/mod.rs | 2 +- crates/spyglass/src/search/utils.rs | 140 ++++++++++++++++++++- 3 files changed, 150 insertions(+), 136 deletions(-) diff --git a/crates/spyglass/src/api/handler/search.rs b/crates/spyglass/src/api/handler/search.rs index 94bd2fcd8..88d4b63df 100644 --- a/crates/spyglass/src/api/handler/search.rs +++ b/crates/spyglass/src/api/handler/search.rs @@ -15,124 +15,6 @@ use std::collections::HashSet; use std::time::SystemTime; use tracing::instrument; -/// Max number of tokens we'll look at for matches before stopping. -const MAX_HIGHLIGHT_SCAN: usize = 10_000; -/// Max number of matches we need to generate a decent preview. -const MAX_HIGHLIGHT_MATCHES: usize = 5; - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] -struct WordRange { - start: usize, - end: usize, - matches: Vec, -} - -impl WordRange { - pub fn new(start: usize, end: usize, match_idx: usize) -> Self { - Self { - start, - end, - matches: vec![match_idx], - } - } - - pub fn overlaps(&self, other: &WordRange) -> bool { - self.start <= other.start && other.start <= self.end - || self.start <= other.end && other.end <= self.end - } - - pub fn merge(&mut self, other: &WordRange) { - self.start = self.start.min(other.start); - self.end = self.end.max(other.end); - self.matches.extend(other.matches.iter()); - } -} - -/// Creates a short preview from content based on the search query terms by -/// finding matches for words and creating a window around each match, joining -/// together overlaps & returning the final string. -fn generate_highlight_preview(index: &Searcher, query: &str, content: &str) -> String { - let fields = DocFields::as_fields(); - let tokenizer = index - .index - .tokenizer_for_field(fields.content) - .expect("Unable to get tokenizer for content field"); - - // tokenize search query - let mut terms = HashSet::new(); - let mut tokens = tokenizer.token_stream(query); - while let Some(t) = tokens.next() { - terms.insert(t.text.clone()); - } - - let tokens = content - .split_whitespace() - .map(|s| s.to_string()) - .collect::>(); - - let mut matched_indices = Vec::new(); - let mut num_tokens_scanned = 0; - for (idx, w) in content.split_whitespace().enumerate() { - num_tokens_scanned += 1; - - let normalized = tokenizer - .token_stream(w) - .next() - .map(|t| t.text.clone()) - .unwrap_or_else(|| w.to_string()); - if terms.contains(&normalized) { - matched_indices.push(idx); - } - - if matched_indices.len() > MAX_HIGHLIGHT_MATCHES { - break; - } - - if num_tokens_scanned > MAX_HIGHLIGHT_SCAN { - break; - } - } - - // Create word ranges from the indices - let mut ranges: Vec = Vec::new(); - for idx in matched_indices { - let start = (idx as i32 - 5).max(0) as usize; - let end = (idx + 5).min(tokens.len() - 1); - let new_range = WordRange::new(start, end, idx); - - if let Some(last) = ranges.last_mut() { - if last.overlaps(&new_range) { - last.merge(&new_range); - continue; - } - } - - ranges.push(new_range); - } - - // Create preview from word ranges - let mut desc: Vec = Vec::new(); - let mut num_windows = 0; - for range in ranges { - let mut slice = tokens[range.start..=range.end].to_vec(); - if !slice.is_empty() { - for idx in range.matches { - let slice_idx = idx - range.start; - slice[slice_idx] = format!("{}", &slice[slice_idx]); - } - desc.extend(slice); - desc.push("...".to_string()); - num_windows += 1; - - if num_windows > 3 { - break; - } - } - } - - format!("{}", desc.join(" ")) -} - /// Search the user's indexed documents #[instrument(skip(state))] pub async fn search_docs( @@ -195,8 +77,16 @@ pub async fn search_docs( .map(|tag| (tag.label.to_string(), tag.value.clone())) .collect::>(); - let description = - generate_highlight_preview(&state.index, &query, &doc.content); + let fields = DocFields::as_fields(); + let tokenizer = index + .index + .tokenizer_for_field(fields.content) + .expect("Unable to get tokenizer for content field"); + let description = libspyglass::search::utils::generate_highlight_preview( + &tokenizer, + &query, + &doc.content, + ); let result = SearchResult { doc_id: doc.doc_id.clone(), domain: doc.domain, @@ -304,17 +194,3 @@ pub async fn search_lenses( Ok(SearchLensesResp { results }) } - -#[cfg(test)] -mod test { - use crate::api::handler::search::generate_highlight_preview; - use libspyglass::search::{IndexPath, Searcher}; - - #[test] - fn test_find_highlights() { - let searcher = Searcher::with_index(&IndexPath::Memory).expect("Unable to open index"); - let blurb = r#"Rust rust is a multi-paradigm, high-level, general-purpose programming"#; - let desc = generate_highlight_preview(&searcher, "rust programming", &blurb); - assert_eq!(desc, "Rust rust is a multi-paradigm, high-level, general-purpose programming ..."); - } -} diff --git a/crates/spyglass/src/search/mod.rs b/crates/spyglass/src/search/mod.rs index 089490e0a..86da43afe 100644 --- a/crates/spyglass/src/search/mod.rs +++ b/crates/spyglass/src/search/mod.rs @@ -24,7 +24,7 @@ use entities::sea_orm::{prelude::*, DatabaseConnection}; pub mod grouping; pub mod lens; mod query; -mod utils; +pub mod utils; pub use query::QueryStats; diff --git a/crates/spyglass/src/search/utils.rs b/crates/spyglass/src/search/utils.rs index 3d0b7a008..299effa68 100644 --- a/crates/spyglass/src/search/utils.rs +++ b/crates/spyglass/src/search/utils.rs @@ -1,4 +1,41 @@ -use tantivy::{fastfield::MultiValuedFastFieldReader, termdict::TermDictionary, DocId}; +use std::collections::HashSet; + +use tantivy::{ + fastfield::MultiValuedFastFieldReader, termdict::TermDictionary, tokenizer::TextAnalyzer, DocId, +}; + +/// Max number of tokens we'll look at for matches before stopping. +const MAX_HIGHLIGHT_SCAN: usize = 10_000; +/// Max number of matches we need to generate a decent preview. +const MAX_HIGHLIGHT_MATCHES: usize = 5; + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct WordRange { + start: usize, + end: usize, + matches: Vec, +} + +impl WordRange { + pub fn new(start: usize, end: usize, match_idx: usize) -> Self { + Self { + start, + end, + matches: vec![match_idx], + } + } + + pub fn overlaps(&self, other: &WordRange) -> bool { + self.start <= other.start && other.start <= self.end + || self.start <= other.end && other.end <= self.end + } + + pub fn merge(&mut self, other: &WordRange) { + self.start = self.start.min(other.start); + self.end = self.end.max(other.end); + self.matches.extend(other.matches.iter()); + } +} #[allow(dead_code)] pub fn ff_to_string( @@ -20,3 +57,104 @@ pub fn ff_to_string( None } + +/// Creates a short preview from content based on the search query terms by +/// finding matches for words and creating a window around each match, joining +/// together overlaps & returning the final string. +pub fn generate_highlight_preview(tokenizer: &TextAnalyzer, query: &str, content: &str) -> String { + // tokenize search query + let mut terms = HashSet::new(); + let mut tokens = tokenizer.token_stream(query); + while let Some(t) = tokens.next() { + terms.insert(t.text.clone()); + } + + let tokens = content + .split_whitespace() + .map(|s| s.to_string()) + .collect::>(); + + let mut matched_indices = Vec::new(); + let mut num_tokens_scanned = 0; + for (idx, w) in content.split_whitespace().enumerate() { + num_tokens_scanned += 1; + + let normalized = tokenizer + .token_stream(w) + .next() + .map(|t| t.text.clone()) + .unwrap_or_else(|| w.to_string()); + if terms.contains(&normalized) { + matched_indices.push(idx); + } + + if matched_indices.len() > MAX_HIGHLIGHT_MATCHES { + break; + } + + if num_tokens_scanned > MAX_HIGHLIGHT_SCAN { + break; + } + } + + // Create word ranges from the indices + let mut ranges: Vec = Vec::new(); + for idx in matched_indices { + let start = (idx as i32 - 5).max(0) as usize; + let end = (idx + 5).min(tokens.len() - 1); + let new_range = WordRange::new(start, end, idx); + + if let Some(last) = ranges.last_mut() { + if last.overlaps(&new_range) { + last.merge(&new_range); + continue; + } + } + + ranges.push(new_range); + } + + // Create preview from word ranges + let mut desc: Vec = Vec::new(); + let mut num_windows = 0; + for range in ranges { + let mut slice = tokens[range.start..=range.end].to_vec(); + if !slice.is_empty() { + for idx in range.matches { + let slice_idx = idx - range.start; + slice[slice_idx] = format!("{}", &slice[slice_idx]); + } + desc.extend(slice); + desc.push("...".to_string()); + num_windows += 1; + + if num_windows > 3 { + break; + } + } + } + + format!("{}", desc.join(" ")) +} + +#[cfg(test)] +mod test { + use crate::search::utils::generate_highlight_preview; + use crate::search::{IndexPath, Searcher}; + use entities::schema::DocFields; + use entities::schema::SearchDocument; + + #[test] + fn test_find_highlights() { + let searcher = Searcher::with_index(&IndexPath::Memory).expect("Unable to open index"); + let blurb = r#"Rust rust is a multi-paradigm, high-level, general-purpose programming"#; + + let fields = DocFields::as_fields(); + let tokenizer = searcher + .index + .tokenizer_for_field(fields.content) + .expect("Unable to get tokenizer for content field"); + let desc = generate_highlight_preview(&tokenizer, "rust programming", &blurb); + assert_eq!(desc, "Rust rust is a multi-paradigm, high-level, general-purpose programming ..."); + } +} From cd39919c9a958047af7623ea096697c20f82039a Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Sat, 15 Apr 2023 16:23:23 -0700 Subject: [PATCH 08/30] feature: hook web backend (locally only) (#434) * moving results over to ui-components & adding search query * setting up jsonrpsee support in web client * use rpc endpoint for search request * only include the features w e need w/ spyglass-rpc * undoing spyglass-rpc feature cleanup --- Cargo.lock | 124 +- Cargo.toml | 8 +- apps/web/.cargo/config.toml | 3 + apps/web/Cargo.lock | 2116 +++++++++++++++++++++++++++ apps/web/Cargo.toml | 12 +- apps/web/public/main.css | 2 +- apps/web/src/constants.rs | 2 + apps/web/src/main.rs | 1 + apps/web/src/pages/search.rs | 122 +- crates/spyglass-rpc/Cargo.toml | 1 - crates/ui-components/Cargo.toml | 3 + crates/ui-components/src/lib.rs | 1 + crates/ui-components/src/results.rs | 280 ++++ 13 files changed, 2570 insertions(+), 105 deletions(-) create mode 100644 apps/web/.cargo/config.toml create mode 100644 apps/web/Cargo.lock create mode 100644 apps/web/src/constants.rs create mode 100644 crates/ui-components/src/results.rs diff --git a/Cargo.lock b/Cargo.lock index 94566d15c..bf897b26d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1036,16 +1036,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "console_log" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be8aed40e4edbf4d3b4431ab260b63fdc40f5780a4766824329ea0f1eefe3c0f" -dependencies = [ - "log", - "web-sys", -] - [[package]] name = "const_fn" version = "0.4.9" @@ -2066,10 +2056,6 @@ name = "futures-timer" version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" -dependencies = [ - "gloo-timers", - "send_wrapper", -] [[package]] name = "futures-util" @@ -2809,6 +2795,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +dependencies = [ + "hmac 0.12.1", +] + [[package]] name = "hmac" version = "0.11.0" @@ -2951,12 +2946,9 @@ checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" dependencies = [ "http", "hyper", - "log", "rustls", - "rustls-native-certs", "tokio", "tokio-rustls", - "webpki-roots", ] [[package]] @@ -3273,13 +3265,10 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d291e3a5818a2384645fd9756362e6d89cf0541b0b916fa7702ea4a9833608e" dependencies = [ - "jsonrpsee-client-transport", "jsonrpsee-core", - "jsonrpsee-http-client", "jsonrpsee-proc-macros", "jsonrpsee-server", "jsonrpsee-types", - "jsonrpsee-wasm-client", "jsonrpsee-ws-client", "tracing", ] @@ -3290,11 +3279,7 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "965de52763f2004bc91ac5bcec504192440f0b568a5d621c59d9dbd6f886c3fb" dependencies = [ - "anyhow", - "futures-channel", - "futures-timer", "futures-util", - "gloo-net", "http", "jsonrpsee-core", "jsonrpsee-types", @@ -3335,26 +3320,6 @@ dependencies = [ "thiserror", "tokio", "tracing", - "wasm-bindgen-futures", -] - -[[package]] -name = "jsonrpsee-http-client" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc345b0a43c6bc49b947ebeb936e886a419ee3d894421790c969cc56040542ad" -dependencies = [ - "async-trait", - "hyper", - "hyper-rustls", - "jsonrpsee-core", - "jsonrpsee-types", - "rustc-hash", - "serde", - "serde_json", - "thiserror", - "tokio", - "tracing", ] [[package]] @@ -3406,17 +3371,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "jsonrpsee-wasm-client" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77310456f43c6c89bcba1f6b2fc2a28300da7c341f320f5128f8c83cc63232d" -dependencies = [ - "jsonrpsee-client-transport", - "jsonrpsee-core", - "jsonrpsee-types", -] - [[package]] name = "jsonrpsee-ws-client" version = "0.16.2" @@ -3799,6 +3753,15 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "md-5" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +dependencies = [ + "digest 0.10.6", +] + [[package]] name = "measure_time" version = "0.8.2" @@ -5598,7 +5561,7 @@ dependencies = [ "http", "hyper", "log", - "md-5", + "md-5 0.9.1", "percent-encoding", "pin-project-lite", "rusoto_credential", @@ -6040,12 +6003,6 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -[[package]] -name = "send_wrapper" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" - [[package]] name = "sentry" version = "0.30.0" @@ -6808,11 +6765,13 @@ checksum = "fa8241483a83a3f33aa5fff7e7d9def398ff9990b2752b6c6112b83c6d246029" dependencies = [ "ahash", "atoi", + "base64 0.13.1", "bitflags", "byteorder", "bytes", "chrono", "crossbeam-queue", + "dirs", "dotenvy", "either", "event-listener", @@ -6824,19 +6783,25 @@ dependencies = [ "futures-util", "hashlink 0.8.1", "hex", + "hkdf", + "hmac 0.12.1", "indexmap", "itoa 1.0.6", "libc", "libsqlite3-sys", "log", + "md-5 0.10.5", "memchr", "once_cell", "paste", "percent-encoding", + "rand 0.8.5", "rustls", "rustls-pemfile 1.0.2", "serde", "serde_json", + "sha1 0.10.5", + "sha2 0.10.6", "smallvec", "sqlformat", "sqlx-rt", @@ -6845,6 +6810,7 @@ dependencies = [ "tokio-stream", "url", "webpki-roots", + "whoami", ] [[package]] @@ -8317,6 +8283,9 @@ checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" name = "ui-components" version = "0.1.0" dependencies = [ + "js-sys", + "shared", + "url", "wasm-bindgen-futures", "yew", ] @@ -8994,27 +8963,6 @@ dependencies = [ "wast", ] -[[package]] -name = "web" -version = "0.1.0" -dependencies = [ - "console_log", - "gloo", - "log", - "markdown", - "serde-wasm-bindgen 0.5.0", - "serde_json", - "shared", - "strum 0.24.1", - "strum_macros 0.24.3", - "ui-components", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "yew", - "yew-router", -] - [[package]] name = "web-sys" version = "0.3.61" @@ -9156,6 +9104,16 @@ dependencies = [ "bindgen", ] +[[package]] +name = "whoami" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c70234412ca409cc04e864e89523cb0fc37f5e1344ebed5a3ebf4192b6b9f68" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 5b97c6ae6..d4b810d76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,15 +8,17 @@ members = [ "crates/tauri", "crates/ui-components", - # Clients - "apps/web", - # Public published crates "crates/spyglass-plugin", "crates/spyglass-lens", "crates/spyglass-rpc", ] +exclude = [ + # Excluded due to strict wasm32-unknown-unknown requirement for jsonrpsee-wasm-client + "apps/web" +] + [profile.release] codegen-units = 1 lto = true diff --git a/apps/web/.cargo/config.toml b/apps/web/.cargo/config.toml new file mode 100644 index 000000000..aa59d2ee1 --- /dev/null +++ b/apps/web/.cargo/config.toml @@ -0,0 +1,3 @@ +[build] +target = "wasm32-unknown-unknown" + diff --git a/apps/web/Cargo.lock b/apps/web/Cargo.lock new file mode 100644 index 000000000..f89f4263b --- /dev/null +++ b/apps/web/Cargo.lock @@ -0,0 +1,2116 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" + +[[package]] +name = "anymap2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" + +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + +[[package]] +name = "async-lock" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-trait" +version = "0.1.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" +dependencies = [ + "serde", +] + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "boolinator" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfa8873f51c92e232f9bac4065cddef41b714152812bfc5f7672ba16d6ef8cd9" + +[[package]] +name = "bumpalo" +version = "3.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + +[[package]] +name = "console_log" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8aed40e4edbf4d3b4431ab260b63fdc40f5780a4766824329ea0f1eefe3c0f" +dependencies = [ + "log", + "web-sys", +] + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "diff-struct" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a699d33d273c6226fd3e8f310a9c90ca94f72be84f7aed95b22bb676f34fc90" +dependencies = [ + "diff_derive", + "num", + "serde", +] + +[[package]] +name = "diff_derive" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe165e7ead196bbbf44c7ce11a7a21157b5c002ce46d7098ff9c556784a4912d" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "digest" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "directories" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f51c5d4ddabd36886dd3e1438cb358cdcb0d7c499cb99cb4ac2e38e18b5cb210" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "encoding_rs" +version = "0.8.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +dependencies = [ + "gloo-timers", + "send_wrapper", +] + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "gloo" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a4bef6b277b3ab073253d4bca60761240cf8d6998f4bd142211957b69a61b20" +dependencies = [ + "gloo-console", + "gloo-dialogs", + "gloo-events", + "gloo-file", + "gloo-history", + "gloo-net", + "gloo-render", + "gloo-storage", + "gloo-timers", + "gloo-utils", + "gloo-worker", +] + +[[package]] +name = "gloo-console" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b7ce3c05debe147233596904981848862b068862e9ec3e34be446077190d3f" +dependencies = [ + "gloo-utils", + "js-sys", + "serde", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-dialogs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67062364ac72d27f08445a46cab428188e2e224ec9e37efdba48ae8c289002e6" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-events" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b107f8abed8105e4182de63845afcc7b69c098b7852a813ea7462a320992fc" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-file" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d5564e570a38b43d78bdc063374a0c3098c4f0d64005b12f9bbe87e869b6d7" +dependencies = [ + "futures-channel", + "gloo-events", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-history" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd451019e0b7a2b8a7a7b23e74916601abf1135c54664e57ff71dcc26dfcdeb7" +dependencies = [ + "gloo-events", + "gloo-utils", + "serde", + "serde-wasm-bindgen 0.4.5", + "serde_urlencoded", + "thiserror", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-net" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9902a044653b26b99f7e3693a42f171312d9be8b26b5697bd1e43ad1f8a35e10" +dependencies = [ + "futures-channel", + "futures-core", + "futures-sink", + "gloo-utils", + "js-sys", + "pin-project", + "serde", + "serde_json", + "thiserror", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "gloo-render" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd9306aef67cfd4449823aadcd14e3958e0800aa2183955a309112a84ec7764" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-storage" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6ab60bf5dbfd6f0ed1f7843da31b41010515c745735c970e821945ca91e480" +dependencies = [ + "gloo-utils", + "js-sys", + "serde", + "serde_json", + "thiserror", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-timers" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b995a66bb87bebce9a0f4a95aed01daca4872c050bfcb21653361c03bc35e5c" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "gloo-utils" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8e8fc851e9c7b9852508bc6e3f690f452f474417e8545ec9857b7f7377036b5" +dependencies = [ + "js-sys", + "serde", + "serde_json", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "gloo-worker" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13471584da78061a28306d1359dd0178d8d6fc1c7c80e5e35d27260346e0516a" +dependencies = [ + "anymap2", + "bincode", + "gloo-console", + "gloo-utils", + "js-sys", + "serde", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "h2" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66b91535aa35fea1523ad1b86cb6b53c28e0ae566ba4a460f4457e936cad7c6f" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "implicit-clone" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40fc102e70475c320b185cd18c1e48bba2d7210b63970a4d581ef903e4368ef7" +dependencies = [ + "indexmap", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "ipnet" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" + +[[package]] +name = "itoa" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" + +[[package]] +name = "js-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "jsonrpsee-client-transport" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965de52763f2004bc91ac5bcec504192440f0b568a5d621c59d9dbd6f886c3fb" +dependencies = [ + "anyhow", + "futures-channel", + "futures-timer", + "futures-util", + "gloo-net", + "jsonrpsee-core", + "thiserror", + "tracing", +] + +[[package]] +name = "jsonrpsee-core" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e70b4439a751a5de7dd5ed55eacff78ebf4ffe0fc009cb1ebb11417f5b536b" +dependencies = [ + "anyhow", + "async-lock", + "async-trait", + "beef", + "futures-channel", + "futures-timer", + "futures-util", + "jsonrpsee-types", + "rustc-hash", + "serde", + "serde_json", + "thiserror", + "tracing", + "wasm-bindgen-futures", +] + +[[package]] +name = "jsonrpsee-types" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bd522fe1ce3702fd94812965d7bb7a3364b1c9aba743944c5a00529aae80f8c" +dependencies = [ + "anyhow", + "beef", + "serde", + "serde_json", + "thiserror", + "tracing", +] + +[[package]] +name = "jsonrpsee-wasm-client" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77310456f43c6c89bcba1f6b2fc2a28300da7c341f320f5128f8c83cc63232d" +dependencies = [ + "jsonrpsee-client-transport", + "jsonrpsee-core", + "jsonrpsee-types", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.141" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" + +[[package]] +name = "linux-raw-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "markdown" +version = "1.0.0-alpha.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de49c677e95e00eaa74c42a0b07ea55e1e0b1ebca5b2cbc7657f288cd714eb" +dependencies = [ + "unicode-id", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.45.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +dependencies = [ + "hermit-abi 0.2.6", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" + +[[package]] +name = "openssl" +version = "0.10.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e30d8bc91859781f0a943411186324d580f2bbeb71b452fe91ae344806af3f1" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d3d193fb1488ad46ffe3aaabc912cc931d02ee8518fe2959aea8ef52718b0c0" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + +[[package]] +name = "pin-project" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pinned" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a829027bd95e54cfe13e3e258a1ae7b645960553fb82b75ff852c29688ee595b" +dependencies = [ + "futures", + "rustversion", + "thiserror", +] + +[[package]] +name = "pkg-config" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" + +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prokio" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03b55e106e5791fa5a13abd13c85d6127312e8e09098059ca2bc9b03ca4cf488" +dependencies = [ + "futures", + "gloo", + "num_cpus", + "once_cell", + "pin-project", + "pinned", + "tokio", + "tokio-stream", + "wasm-bindgen-futures", +] + +[[package]] +name = "quote" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall 0.2.16", + "thiserror", +] + +[[package]] +name = "regex" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "reqwest" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" +dependencies = [ + "base64 0.21.0", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "ron" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300a51053b1cb55c80b7a9fde4120726ddf25ca241a1cbb926626f62fb136bff" +dependencies = [ + "base64 0.13.1", + "bitflags", + "serde", +] + +[[package]] +name = "route-recognizer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.37.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "rustversion" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" + +[[package]] +name = "ryu" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" + +[[package]] +name = "schannel" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +dependencies = [ + "windows-sys 0.42.0", +] + +[[package]] +name = "security-framework" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "send_wrapper" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" + +[[package]] +name = "serde" +version = "1.0.160" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-wasm-bindgen" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b4c031cd0d9014307d82b8abf653c0290fbdaeb4c02d00c63cf52f728628bf" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + +[[package]] +name = "serde-wasm-bindgen" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3b143e2833c57ab9ad3ea280d21fd34e285a42837aeb0ee301f4f41890fa00e" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + +[[package]] +name = "serde_derive" +version = "1.0.160" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "serde_json" +version = "1.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shared" +version = "0.1.0" +dependencies = [ + "anyhow", + "bitflags", + "diff-struct", + "directories", + "log", + "num-format", + "regex", + "ron", + "serde", + "serde_json", + "spyglass-lens", + "strum", + "strum_macros", + "url", + "uuid", +] + +[[package]] +name = "slab" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] + +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "spyglass-lens" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e2ebbd0eff5a2ebf86cdc693c1b314722aee201702f9746b6309c43e4867f70" +dependencies = [ + "anyhow", + "blake2", + "hex", + "regex", + "ron", + "serde", +] + +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall 0.3.5", + "rustix", + "windows-sys 0.45.0", +] + +[[package]] +name = "thiserror" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.15", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +dependencies = [ + "autocfg", + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.45.0", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "tracing-core" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" + +[[package]] +name = "ui-components" +version = "0.1.0" +dependencies = [ + "js-sys", + "shared", + "url", + "wasm-bindgen-futures", + "yew", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-id" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d70b6494226b36008c8366c288d77190b3fad2eb4c10533139c1c1f461127f1a" + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "uuid" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb" +dependencies = [ + "getrandom", + "serde", + "wasm-bindgen", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.84" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" + +[[package]] +name = "web" +version = "0.1.0" +dependencies = [ + "console_log", + "gloo", + "jsonrpsee-core", + "jsonrpsee-wasm-client", + "log", + "markdown", + "reqwest", + "serde", + "serde-wasm-bindgen 0.5.0", + "serde_json", + "shared", + "strum", + "strum_macros", + "ui-components", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "yew", + "yew-router", +] + +[[package]] +name = "web-sys" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + +[[package]] +name = "yew" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dbecfe44343b70cc2932c3eb445425969ae21754a8ab3a0966981c1cf7af1cc" +dependencies = [ + "console_error_panic_hook", + "futures", + "gloo", + "implicit-clone", + "indexmap", + "js-sys", + "prokio", + "rustversion", + "serde", + "slab", + "thiserror", + "tokio", + "tracing", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "yew-macro", +] + +[[package]] +name = "yew-macro" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b64c253c1d401f1ea868ca9988db63958cfa15a69f739101f338d6f05eea8301" +dependencies = [ + "boolinator", + "once_cell", + "prettyplease", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "yew-router" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "426ee0486d2572a6c5e39fbdbc48b58d59bb555f3326f54631025266cf04146e" +dependencies = [ + "gloo", + "js-sys", + "route-recognizer", + "serde", + "serde_urlencoded", + "tracing", + "wasm-bindgen", + "web-sys", + "yew", + "yew-router-macro", +] + +[[package]] +name = "yew-router-macro" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b249cdb39e0cddaf0644dedc781854524374664793479fdc01e6a65d6e6ae3" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] diff --git a/apps/web/Cargo.toml b/apps/web/Cargo.toml index 25095d04e..377f7ed0c 100644 --- a/apps/web/Cargo.toml +++ b/apps/web/Cargo.toml @@ -3,21 +3,23 @@ name = "web" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] console_log = "1.0" -"shared" = { path = "../../crates/shared" } gloo = "0.8.0" log = "0.4" +markdown = "1.0.0-alpha.7" +reqwest = { version = "0.11", features = ["json"] } +serde = "1.0" serde_json = "1.0" serde-wasm-bindgen = "0.5" +"shared" = { path = "../../crates/shared" } strum = "0.24" strum_macros = "0.24" ui-components = { path = "../../crates/ui-components" } wasm-bindgen = "0.2.83" wasm-bindgen-futures = "0.4.33" web-sys = { version = "0.3.60", features = ["Navigator", "VisibilityState"] } +jsonrpsee-core = "0.16.2" +jsonrpsee-wasm-client = "0.16.2" yew = { version = "0.20.0", features = ["csr"] } -yew-router = "0.17" -markdown = "1.0.0-alpha.7" \ No newline at end of file +yew-router = "0.17" \ No newline at end of file diff --git a/apps/web/public/main.css b/apps/web/public/main.css index b5ed38ab9..0f9652607 100644 --- a/apps/web/public/main.css +++ b/apps/web/public/main.css @@ -1 +1 @@ -/*! tailwindcss v3.3.1 | MIT License | https://tailwindcss.com*/*,:after,:before{box-sizing:border-box;border:0 solid #e5e7eb}:after,:before{--tw-content:""}html{line-height:1.5;-webkit-text-size-adjust:100%;-moz-tab-size:4;-o-tab-size:4;tab-size:4;font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol,Noto Color Emoji;font-feature-settings:normal;font-variation-settings:normal}body{margin:0;line-height:inherit}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,pre,samp{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:initial}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}button,input,optgroup,select,textarea{font-family:inherit;font-size:100%;font-weight:inherit;line-height:inherit;color:inherit;margin:0;padding:0}button,select{text-transform:none}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button;background-color:initial;background-image:none}:-moz-focusring{outline:auto}:-moz-ui-invalid{box-shadow:none}progress{vertical-align:initial}::-webkit-inner-spin-button,::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}summary{display:list-item}blockquote,dd,dl,figure,h1,h2,h3,h4,h5,h6,hr,p,pre{margin:0}fieldset{margin:0}fieldset,legend{padding:0}menu,ol,ul{list-style:none;margin:0;padding:0}textarea{resize:vertical}input::-moz-placeholder,textarea::-moz-placeholder{opacity:1;color:#9ca3af}input::placeholder,textarea::placeholder{opacity:1;color:#9ca3af}[role=button],button{cursor:pointer}:disabled{cursor:default}audio,canvas,embed,iframe,img,object,svg,video{display:block;vertical-align:middle}img,video{max-width:100%;height:auto}[hidden]{display:none}*,::backdrop,:after,:before{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:#3b82f680;--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: }.absolute{position:absolute}.relative{position:relative}.sticky{position:sticky}.top-0{top:0}.left-0{left:0}.z-50{z-index:50}.z-40{z-index:40}.order-1{order:1}.mx-auto{margin-left:auto;margin-right:auto}.-ml-16{margin-left:-4rem}.-mt-2{margin-top:-.5rem}.mb-2{margin-bottom:.5rem}.mb-\[-4px\]{margin-bottom:-4px}.ml-auto{margin-left:auto}.mr-1{margin-right:.25rem}.mt-2{margin-top:.5rem}.mb-6{margin-bottom:1.5rem}.mr-2{margin-right:.5rem}.inline-block{display:inline-block}.inline{display:inline}.flex{display:flex}.contents{display:contents}.hidden{display:none}.h-4{height:1rem}.h-5{height:1.25rem}.h-6{height:1.5rem}.h-\[48px\]{height:48px}.h-screen{height:100vh}.h-8{height:2rem}.h-full{height:100%}.max-h-\[640px\]{max-height:640px}.min-h-\[128px\]{min-height:128px}.w-2{width:.5rem}.w-4{width:1rem}.w-5{width:1.25rem}.w-6{width:1.5rem}.w-\[48px\]{width:48px}.w-48{width:12rem}.w-full{width:100%}.flex-1{flex:1 1 0%}.flex-none{flex:none}.grow{flex-grow:1}@keyframes fade-in{0%{opacity:0;transform:translateY(10px)}to{opacity:1;transform:translateY(0)}}.animate-fade-in{animation:fade-in .5s ease-out}@keyframes pulse{50%{opacity:.5}}.animate-pulse{animation:pulse 2s cubic-bezier(.4,0,.6,1) infinite}@keyframes spin{to{transform:rotate(1turn)}}.animate-spin{animation:spin 1s linear infinite}.cursor-help{cursor:help}.cursor-pointer{cursor:pointer}.resize{resize:both}.flex-row{flex-direction:row}.flex-col{flex-direction:column}.flex-nowrap{flex-wrap:nowrap}.place-content-end{place-content:end}.items-center{align-items:center}.gap-1{gap:.25rem}.gap-2{gap:.5rem}.gap-4{gap:1rem}.gap-8{gap:2rem}.overflow-hidden{overflow:hidden}.overflow-y-auto{overflow-y:auto}.overflow-x-hidden{overflow-x:hidden}.overflow-y-scroll{overflow-y:scroll}.rounded{border-radius:.25rem}.rounded-full{border-radius:9999px}.rounded-md{border-radius:.375rem}.rounded-xl{border-radius:.75rem}.border{border-width:1px}.border-2{border-width:2px}.border-b-2{border-bottom-width:2px}.border-t{border-top-width:1px}.border-t-2{border-top-width:2px}.border-cyan-600{--tw-border-opacity:1;border-color:rgb(8 145 178/var(--tw-border-opacity))}.border-neutral-600{--tw-border-opacity:1;border-color:rgb(82 82 82/var(--tw-border-opacity))}.border-red-700{--tw-border-opacity:1;border-color:rgb(185 28 28/var(--tw-border-opacity))}.border-neutral-900{--tw-border-opacity:1;border-color:rgb(23 23 23/var(--tw-border-opacity))}.border-t-neutral-700{--tw-border-opacity:1;border-top-color:rgb(64 64 64/var(--tw-border-opacity))}.bg-cyan-600{--tw-bg-opacity:1;background-color:rgb(8 145 178/var(--tw-bg-opacity))}.bg-cyan-900{--tw-bg-opacity:1;background-color:rgb(22 78 99/var(--tw-bg-opacity))}.bg-green-700{--tw-bg-opacity:1;background-color:rgb(21 128 61/var(--tw-bg-opacity))}.bg-neutral-700{--tw-bg-opacity:1;background-color:rgb(64 64 64/var(--tw-bg-opacity))}.bg-neutral-800{--tw-bg-opacity:1;background-color:rgb(38 38 38/var(--tw-bg-opacity))}.bg-neutral-900{--tw-bg-opacity:1;background-color:rgb(23 23 23/var(--tw-bg-opacity))}.bg-transparent{background-color:initial}.bg-stone-900{--tw-bg-opacity:1;background-color:rgb(28 25 23/var(--tw-bg-opacity))}.bg-cyan-700{--tw-bg-opacity:1;background-color:rgb(14 116 144/var(--tw-bg-opacity))}.p-4{padding:1rem}.p-2{padding:.5rem}.p-0{padding:0}.px-2{padding-left:.5rem;padding-right:.5rem}.px-3{padding-left:.75rem;padding-right:.75rem}.px-4{padding-left:1rem;padding-right:1rem}.py-1{padding-top:.25rem;padding-bottom:.25rem}.py-2{padding-top:.5rem;padding-bottom:.5rem}.py-4{padding-top:1rem;padding-bottom:1rem}.px-8{padding-left:2rem;padding-right:2rem}.py-3{padding-top:.75rem;padding-bottom:.75rem}.px-1{padding-left:.25rem;padding-right:.25rem}.py-0{padding-top:0;padding-bottom:0}.py-0\.5{padding-top:.125rem;padding-bottom:.125rem}.pb-2{padding-bottom:.5rem}.pl-3{padding-left:.75rem}.text-left{text-align:left}.text-center{text-align:center}.text-right{text-align:right}.text-base{font-size:.875rem}.text-lg{font-size:1rem}.text-sm{font-size:.75rem}.text-xl{font-size:1.125rem}.text-xs{font-size:.625rem}.text-5xl{font-size:2rem}.text-2xl{font-size:1.25rem}.font-semibold{font-weight:600}.font-bold{font-weight:700}.uppercase{text-transform:uppercase}.leading-5{line-height:1.25rem}.text-cyan-500{--tw-text-opacity:1;color:rgb(6 182 212/var(--tw-text-opacity))}.text-neutral-400{--tw-text-opacity:1;color:rgb(163 163 163/var(--tw-text-opacity))}.text-neutral-500{--tw-text-opacity:1;color:rgb(115 115 115/var(--tw-text-opacity))}.text-red-500{--tw-text-opacity:1;color:rgb(239 68 68/var(--tw-text-opacity))}.text-stone-400{--tw-text-opacity:1;color:rgb(168 162 158/var(--tw-text-opacity))}.text-white{--tw-text-opacity:1;color:rgb(255 255 255/var(--tw-text-opacity))}.text-yellow-400{--tw-text-opacity:1;color:rgb(250 204 21/var(--tw-text-opacity))}.text-gray-500{--tw-text-opacity:1;color:rgb(107 114 128/var(--tw-text-opacity))}.text-cyan-600{--tw-text-opacity:1;color:rgb(8 145 178/var(--tw-text-opacity))}.text-yellow-500{--tw-text-opacity:1;color:rgb(234 179 8/var(--tw-text-opacity))}.underline{text-decoration-line:underline}.placeholder-neutral-300::-moz-placeholder{--tw-placeholder-opacity:1;color:rgb(212 212 212/var(--tw-placeholder-opacity))}.placeholder-neutral-300::placeholder{--tw-placeholder-opacity:1;color:rgb(212 212 212/var(--tw-placeholder-opacity))}.placeholder-neutral-600::-moz-placeholder{--tw-placeholder-opacity:1;color:rgb(82 82 82/var(--tw-placeholder-opacity))}.placeholder-neutral-600::placeholder{--tw-placeholder-opacity:1;color:rgb(82 82 82/var(--tw-placeholder-opacity))}.caret-white{caret-color:#fff}.outline-none{outline:2px solid #0000;outline-offset:2px}.filter{filter:var(--tw-blur) var(--tw-brightness) var(--tw-contrast) var(--tw-grayscale) var(--tw-hue-rotate) var(--tw-invert) var(--tw-saturate) var(--tw-sepia) var(--tw-drop-shadow)}input:checked~.dot{transform:translateX(100%);background-color:#48bb78}*{scrollbar-width:thin;scrollbar-color:#404040 #171717}::-webkit-scrollbar{width:10px}::-webkit-scrollbar-track{background:#171717}::-webkit-scrollbar-thumb{background:#404040;border-radius:100vh;border:2px solid #171717}::-webkit-scrollbar-thumb:hover{background:#164e63}mark{color:#fff;background-color:rgb(14 116 144/var(--tw-bg-opacity));padding-left:.125rem;padding-right:.125rem}.hover\:bg-cyan-600:hover{--tw-bg-opacity:1;background-color:rgb(8 145 178/var(--tw-bg-opacity))}.hover\:bg-green-900:hover{--tw-bg-opacity:1;background-color:rgb(20 83 45/var(--tw-bg-opacity))}.hover\:bg-neutral-600:hover{--tw-bg-opacity:1;background-color:rgb(82 82 82/var(--tw-bg-opacity))}.hover\:bg-red-700:hover{--tw-bg-opacity:1;background-color:rgb(185 28 28/var(--tw-bg-opacity))}.hover\:bg-neutral-700:hover{--tw-bg-opacity:1;background-color:rgb(64 64 64/var(--tw-bg-opacity))}.hover\:bg-cyan-900:hover{--tw-bg-opacity:1;background-color:rgb(22 78 99/var(--tw-bg-opacity))}.hover\:bg-cyan-800:hover{--tw-bg-opacity:1;background-color:rgb(21 94 117/var(--tw-bg-opacity))}.hover\:text-white:hover{--tw-text-opacity:1;color:rgb(255 255 255/var(--tw-text-opacity))}.focus\:outline-none:focus{outline:2px solid #0000;outline-offset:2px}.active\:bg-neutral-700:active{--tw-bg-opacity:1;background-color:rgb(64 64 64/var(--tw-bg-opacity))}.active\:outline-none:active{outline:2px solid #0000;outline-offset:2px}.group:hover .group-hover\:block{display:block}.group:hover .group-hover\:text-neutral-400{--tw-text-opacity:1;color:rgb(163 163 163/var(--tw-text-opacity))} \ No newline at end of file +/*! tailwindcss v3.3.1 | MIT License | https://tailwindcss.com*/*,:after,:before{box-sizing:border-box;border:0 solid #e5e7eb}:after,:before{--tw-content:""}html{line-height:1.5;-webkit-text-size-adjust:100%;-moz-tab-size:4;-o-tab-size:4;tab-size:4;font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol,Noto Color Emoji;font-feature-settings:normal;font-variation-settings:normal}body{margin:0;line-height:inherit}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,pre,samp{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:initial}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}button,input,optgroup,select,textarea{font-family:inherit;font-size:100%;font-weight:inherit;line-height:inherit;color:inherit;margin:0;padding:0}button,select{text-transform:none}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button;background-color:initial;background-image:none}:-moz-focusring{outline:auto}:-moz-ui-invalid{box-shadow:none}progress{vertical-align:initial}::-webkit-inner-spin-button,::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}summary{display:list-item}blockquote,dd,dl,figure,h1,h2,h3,h4,h5,h6,hr,p,pre{margin:0}fieldset{margin:0}fieldset,legend{padding:0}menu,ol,ul{list-style:none;margin:0;padding:0}textarea{resize:vertical}input::-moz-placeholder,textarea::-moz-placeholder{opacity:1;color:#9ca3af}input::placeholder,textarea::placeholder{opacity:1;color:#9ca3af}[role=button],button{cursor:pointer}:disabled{cursor:default}audio,canvas,embed,iframe,img,object,svg,video{display:block;vertical-align:middle}img,video{max-width:100%;height:auto}[hidden]{display:none}*,::backdrop,:after,:before{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:#3b82f680;--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: }.absolute{position:absolute}.relative{position:relative}.sticky{position:sticky}.bottom-0{bottom:0}.left-0{left:0}.right-0{right:0}.top-0{top:0}.z-40{z-index:40}.z-50{z-index:50}.m-auto{margin:auto}.mx-auto{margin-left:auto;margin-right:auto}.-ml-16{margin-left:-4rem}.-mt-2{margin-top:-.5rem}.mb-2{margin-bottom:.5rem}.mb-6{margin-bottom:1.5rem}.ml-auto{margin-left:auto}.mr-2{margin-right:.5rem}.mt-1{margin-top:.25rem}.mt-1\.5{margin-top:.375rem}.mt-2{margin-top:.5rem}.flex{display:flex}.hidden{display:none}.h-12{height:3rem}.h-4{height:1rem}.h-5{height:1.25rem}.h-8{height:2rem}.h-screen{height:100vh}.max-h-10{max-height:2.5rem}.w-12{width:3rem}.w-4{width:1rem}.w-48{width:12rem}.w-5{width:1.25rem}.w-8{width:2rem}.w-\[30rem\]{width:30rem}.w-full{width:100%}.flex-1{flex:1 1 0%}.flex-none{flex:none}.grow{flex-grow:1}@keyframes spin{to{transform:rotate(1turn)}}.animate-spin{animation:spin 1s linear infinite}@keyframes fade-in{0%{opacity:0;transform:translateY(10px)}to{opacity:1;transform:translateY(0)}}.animate-fade-in{animation:fade-in .5s ease-out}.cursor-pointer{cursor:pointer}.resize{resize:both}.scroll-mt-2{scroll-margin-top:.5rem}.flex-row{flex-direction:row}.flex-col{flex-direction:column}.flex-wrap{flex-wrap:wrap}.flex-nowrap{flex-wrap:nowrap}.place-items-center{place-items:center}.items-center{align-items:center}.gap-1{gap:.25rem}.gap-2{gap:.5rem}.gap-4{gap:1rem}.overflow-hidden{overflow:hidden}.overflow-y-auto{overflow-y:auto}.truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.rounded{border-radius:.25rem}.rounded-md{border-radius:.375rem}.border{border-width:1px}.border-t-2{border-top-width:2px}.border-b-2{border-bottom-width:2px}.border-neutral-600{--tw-border-opacity:1;border-color:rgb(82 82 82/var(--tw-border-opacity))}.border-neutral-900{--tw-border-opacity:1;border-color:rgb(23 23 23/var(--tw-border-opacity))}.border-red-700{--tw-border-opacity:1;border-color:rgb(185 28 28/var(--tw-border-opacity))}.bg-cyan-500{--tw-bg-opacity:1;background-color:rgb(6 182 212/var(--tw-bg-opacity))}.bg-cyan-600{--tw-bg-opacity:1;background-color:rgb(8 145 178/var(--tw-bg-opacity))}.bg-cyan-900{--tw-bg-opacity:1;background-color:rgb(22 78 99/var(--tw-bg-opacity))}.bg-green-700{--tw-bg-opacity:1;background-color:rgb(21 128 61/var(--tw-bg-opacity))}.bg-neutral-700{--tw-bg-opacity:1;background-color:rgb(64 64 64/var(--tw-bg-opacity))}.bg-neutral-800{--tw-bg-opacity:1;background-color:rgb(38 38 38/var(--tw-bg-opacity))}.bg-neutral-900{--tw-bg-opacity:1;background-color:rgb(23 23 23/var(--tw-bg-opacity))}.bg-stone-900{--tw-bg-opacity:1;background-color:rgb(28 25 23/var(--tw-bg-opacity))}.bg-transparent{background-color:initial}.p-0{padding:0}.p-0\.5{padding:.125rem}.p-2{padding:.5rem}.p-4{padding:1rem}.px-1{padding-left:.25rem;padding-right:.25rem}.px-2{padding-left:.5rem;padding-right:.5rem}.px-3{padding-left:.75rem;padding-right:.75rem}.px-4{padding-left:1rem;padding-right:1rem}.py-0{padding-top:0;padding-bottom:0}.py-0\.5{padding-top:.125rem;padding-bottom:.125rem}.py-1{padding-top:.25rem;padding-bottom:.25rem}.py-2{padding-top:.5rem;padding-bottom:.5rem}.py-3{padding-top:.75rem;padding-bottom:.75rem}.py-4{padding-top:1rem;padding-bottom:1rem}.pl-6{padding-left:1.5rem}.pr-2{padding-right:.5rem}.pb-2{padding-bottom:.5rem}.text-center{text-align:center}.text-right{text-align:right}.text-2xl{font-size:1.25rem}.text-base{font-size:.875rem}.text-lg{font-size:1rem}.text-sm{font-size:.75rem}.text-xl{font-size:1.125rem}.text-xs{font-size:.625rem}.font-bold{font-weight:700}.font-semibold{font-weight:600}.uppercase{text-transform:uppercase}.leading-5{line-height:1.25rem}.leading-relaxed{line-height:1.625}.text-cyan-500{--tw-text-opacity:1;color:rgb(6 182 212/var(--tw-text-opacity))}.text-cyan-600{--tw-text-opacity:1;color:rgb(8 145 178/var(--tw-text-opacity))}.text-gray-500{--tw-text-opacity:1;color:rgb(107 114 128/var(--tw-text-opacity))}.text-neutral-400{--tw-text-opacity:1;color:rgb(163 163 163/var(--tw-text-opacity))}.text-red-500{--tw-text-opacity:1;color:rgb(239 68 68/var(--tw-text-opacity))}.text-stone-400{--tw-text-opacity:1;color:rgb(168 162 158/var(--tw-text-opacity))}.text-white{--tw-text-opacity:1;color:rgb(255 255 255/var(--tw-text-opacity))}.text-yellow-500{--tw-text-opacity:1;color:rgb(234 179 8/var(--tw-text-opacity))}.placeholder-neutral-600::-moz-placeholder{--tw-placeholder-opacity:1;color:rgb(82 82 82/var(--tw-placeholder-opacity))}.placeholder-neutral-600::placeholder{--tw-placeholder-opacity:1;color:rgb(82 82 82/var(--tw-placeholder-opacity))}.caret-white{caret-color:#fff}.outline-none{outline:2px solid #0000;outline-offset:2px}input:checked~.dot{transform:translateX(100%);background-color:#48bb78}*{scrollbar-width:thin;scrollbar-color:#404040 #171717}::-webkit-scrollbar{width:10px}::-webkit-scrollbar-track{background:#171717}::-webkit-scrollbar-thumb{background:#404040;border-radius:100vh;border:2px solid #171717}::-webkit-scrollbar-thumb:hover{background:#164e63}mark{color:#fff;background-color:rgb(14 116 144/var(--tw-bg-opacity));padding-left:.125rem;padding-right:.125rem}.hover\:bg-cyan-800:hover{--tw-bg-opacity:1;background-color:rgb(21 94 117/var(--tw-bg-opacity))}.hover\:bg-green-900:hover{--tw-bg-opacity:1;background-color:rgb(20 83 45/var(--tw-bg-opacity))}.hover\:bg-neutral-600:hover{--tw-bg-opacity:1;background-color:rgb(82 82 82/var(--tw-bg-opacity))}.hover\:bg-neutral-700:hover{--tw-bg-opacity:1;background-color:rgb(64 64 64/var(--tw-bg-opacity))}.hover\:bg-red-700:hover{--tw-bg-opacity:1;background-color:rgb(185 28 28/var(--tw-bg-opacity))}.hover\:text-white:hover{--tw-text-opacity:1;color:rgb(255 255 255/var(--tw-text-opacity))}.focus\:outline-none:focus{outline:2px solid #0000;outline-offset:2px}.active\:bg-cyan-900:active{--tw-bg-opacity:1;background-color:rgb(22 78 99/var(--tw-bg-opacity))}.active\:bg-neutral-700:active{--tw-bg-opacity:1;background-color:rgb(64 64 64/var(--tw-bg-opacity))}.active\:outline-none:active{outline:2px solid #0000;outline-offset:2px}.group:hover .group-hover\:block{display:block}.group:hover .group-hover\:text-neutral-400{--tw-text-opacity:1;color:rgb(163 163 163/var(--tw-text-opacity))} \ No newline at end of file diff --git a/apps/web/src/constants.rs b/apps/web/src/constants.rs new file mode 100644 index 000000000..4605795a3 --- /dev/null +++ b/apps/web/src/constants.rs @@ -0,0 +1,2 @@ +// pub const SEARCH_ENDPOINT: &str = "https://search.spyglass.fyi/search"; +pub const RPC_ENDPOINT: &str = "ws://127.0.0.1:4664"; diff --git a/apps/web/src/main.rs b/apps/web/src/main.rs index 98a1330ad..1ef730d2d 100644 --- a/apps/web/src/main.rs +++ b/apps/web/src/main.rs @@ -2,6 +2,7 @@ use wasm_bindgen::{prelude::Closure, JsValue}; use yew::prelude::*; use yew_router::prelude::*; +mod constants; mod pages; use pages::AppPage; diff --git a/apps/web/src/pages/search.rs b/apps/web/src/pages/search.rs index b9edd796d..845245edf 100644 --- a/apps/web/src/pages/search.rs +++ b/apps/web/src/pages/search.rs @@ -1,37 +1,77 @@ -use std::str::FromStr; - -use shared::keyboard::KeyCode; -use ui_components::btn::{Btn, BtnType}; +use jsonrpsee_core::{client::ClientT, rpc_params}; +use jsonrpsee_wasm_client::{Client, WasmClientBuilder}; +use shared::request::SearchParam; +use shared::{ + keyboard::KeyCode, + response::{SearchResult, SearchResults}, +}; +use std::{ + str::FromStr, + sync::{Arc, Mutex}, +}; +use ui_components::{ + btn::{Btn, BtnType}, + icons::RefreshIcon, + results::SearchResultItem, +}; use web_sys::HtmlInputElement; -use yew::prelude::*; +use yew::{platform::spawn_local, prelude::*}; + +use crate::constants::RPC_ENDPOINT; +const RESULT_PREFIX: &str = "result"; + +pub type RpcMutex = Arc>; #[derive(Clone, Debug)] pub enum Msg { HandleKeyboardEvent(KeyboardEvent), HandleSearch, + SetClient(RpcMutex), + SetSearchResults(Vec), + OpenResult(SearchResult), } pub struct SearchPage { + rpc_client: Option, + results: Vec, search_wrapper_ref: NodeRef, search_input_ref: NodeRef, status_msg: Option, + in_progress: bool, } impl Component for SearchPage { type Message = Msg; type Properties = (); - fn create(_ctx: &yew::Context) -> Self { + fn create(ctx: &yew::Context) -> Self { + let link = ctx.link(); + link.send_future(async move { + let client = WasmClientBuilder::default() + .request_timeout(std::time::Duration::from_secs(10)) + .build(RPC_ENDPOINT) + .await + .expect("Unable to create WsClient"); + Msg::SetClient(Arc::new(Mutex::new(client))) + }); + Self { + rpc_client: None, + results: Vec::new(), search_input_ref: Default::default(), search_wrapper_ref: Default::default(), status_msg: None, + in_progress: false, } } fn update(&mut self, ctx: &yew::Context, msg: Self::Message) -> bool { let link = ctx.link(); match msg { + Msg::SetClient(client) => { + self.rpc_client = Some(client); + false + } Msg::HandleKeyboardEvent(event) => { let key = event.key(); if let Ok(code) = KeyCode::from_str(&key.to_uppercase()) { @@ -40,8 +80,12 @@ impl Component for SearchPage { link.send_message(Msg::HandleSearch); } } + false } Msg::HandleSearch => { + self.in_progress = true; + self.results = Vec::new(); + let query = self .search_input_ref .cast::() @@ -50,18 +94,69 @@ impl Component for SearchPage { log::info!("handling search! {:?}", query); if let Some(query) = query { self.status_msg = Some(format!("searching: {query}")); + let link = link.clone(); + if let Some(client) = &self.rpc_client { + let client = client.clone(); + spawn_local(async move { + if let Ok(client) = client.lock() { + let params = SearchParam { + lenses: Vec::new(), + query: query, + }; + match client + .request::( + "spyglass_search_docs", + rpc_params![params], + ) + .await + { + Ok(res) => { + link.send_message(Msg::SetSearchResults(res.results)); + } + Err(err) => { + log::error!("error rpc: {}", err); + } + } + } + }); + } } + true + } + Msg::SetSearchResults(results) => { + self.in_progress = false; + self.results = results; + true + } + Msg::OpenResult(result) => { + log::info!("opening result: {}", result.url); + false } } - false } fn view(&self, ctx: &yew::Context) -> yew::Html { let link = ctx.link(); + let html = self + .results + .iter() + .enumerate() + .map(|(idx, res)| { + let open_msg = Msg::OpenResult(res.to_owned()); + html! { + + } + }) + .collect::(); + let results = html! {
{html}
}; html! {
-
+
- {"Search"} + {if self.in_progress { + html! { } + } else { + html! { <>{"Search"} } + }}
-
- {self.status_msg.clone().unwrap_or_else(|| "how to guide?".into())} -
+
{results}
} } diff --git a/crates/spyglass-rpc/Cargo.toml b/crates/spyglass-rpc/Cargo.toml index 79cdc1787..7915f243a 100644 --- a/crates/spyglass-rpc/Cargo.toml +++ b/crates/spyglass-rpc/Cargo.toml @@ -6,7 +6,6 @@ description = "RPC definitions for spyglass server" edition = "2021" [dependencies] -# We only need the macros functionality for the shared library serde = "1.0" jsonrpsee = { version = "0.16.2", features = ["full"] } shared = { path = "../shared" } diff --git a/crates/ui-components/Cargo.toml b/crates/ui-components/Cargo.toml index c28cf8b87..e5274e995 100644 --- a/crates/ui-components/Cargo.toml +++ b/crates/ui-components/Cargo.toml @@ -6,5 +6,8 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +shared = { path = "../shared" } +js-sys = "0.3.61" +url = "2.3.1" wasm-bindgen-futures = "0.4.33" yew = { version = "0.20", features = ["csr"] } \ No newline at end of file diff --git a/crates/ui-components/src/lib.rs b/crates/ui-components/src/lib.rs index 3f6e6a7b6..b716273ba 100644 --- a/crates/ui-components/src/lib.rs +++ b/crates/ui-components/src/lib.rs @@ -1,4 +1,5 @@ pub mod btn; pub mod icons; +pub mod results; pub mod tag; pub mod tooltip; diff --git a/crates/ui-components/src/results.rs b/crates/ui-components/src/results.rs new file mode 100644 index 000000000..32dd408b5 --- /dev/null +++ b/crates/ui-components/src/results.rs @@ -0,0 +1,280 @@ +use js_sys::decode_uri_component; +use shared::response::{LensResult, SearchResult}; +use url::Url; +use yew::prelude::*; + +use super::icons; +use super::tag::{Tag, TagIcon}; + +#[derive(Properties, PartialEq)] +pub struct SearchResultProps { + pub id: String, + pub onclick: Callback, + pub result: SearchResult, + #[prop_or_default] + pub is_selected: bool, +} + +fn render_icon(result: &SearchResult) -> Html { + let url = Url::parse(&result.crawl_uri); + let icon_size = classes!("w-8", "h-8", "m-auto", "mt-2"); + + let is_directory = result.tags.iter().any(|(label, value)| { + label.to_lowercase() == "type" && value.to_lowercase() == "directory" + }); + + let is_file = result + .tags + .iter() + .any(|(label, value)| label.to_lowercase() == "type" && value.to_lowercase() == "file"); + + let ext = if let Some((_, ext)) = result.title.rsplit_once('.') { + ext.to_string() + } else { + "txt".to_string() + }; + + let icon = if let Ok(url) = &url { + let domain = url.domain().unwrap_or("example.com").to_owned(); + match url.scheme() { + "api" => { + let connection = url.host_str().unwrap_or_default(); + if is_directory { + html! { + <> + +
+ {icons::connection_icon(connection, "h-4", "w-4", classes!())} +
+ + } + } else if is_file { + html! { + <> + +
+ {icons::connection_icon(connection, "h-4", "w-4", classes!())} +
+ + } + } else { + icons::connection_icon(connection, "h-8", "w-8", classes!("m-auto", "mt-2")) + } + } + "file" => { + let is_directory = result.tags.iter().any(|(label, value)| { + label.to_lowercase() == "type" && value.to_lowercase() == "directory" + }); + + if is_directory { + html! { } + } else { + html! { } + } + } + _ => { + html! { + Website + } + } + } + } else { + html! {} + }; + + icon +} + +// TODO: Pull this special metadata from tags provided by the backend. +fn render_metadata(result: &SearchResult) -> Html { + let mut meta = Vec::new(); + // Generate the icons/labels required for tags + let mut priority_tags = Vec::new(); + let mut normal_tags = Vec::new(); + + let result_type = result + .tags + .iter() + .find(|(label, _)| label.to_lowercase() == "type") + .map(|(_, val)| val.as_str()) + .unwrap_or_default(); + + for (tag, value) in result.tags.iter() { + let tag = tag.to_lowercase(); + if tag == "source" || tag == "mimetype" { + continue; + } + + if result_type == "repository" && tag == "repository" { + continue; + } + + if tag == "favorited" { + priority_tags.push(html! { }); + } else { + normal_tags.push(html! { }); + } + } + + meta.extend(priority_tags); + meta.extend(normal_tags); + + html! { +
+ {meta} +
+ } +} + +/// Render search results +#[function_component(SearchResultItem)] +pub fn search_result_component(props: &SearchResultProps) -> Html { + let is_selected = props.is_selected; + let result = &props.result; + + let component_styles = classes!( + "flex", + "flex-row", + "gap-4", + "rounded", + "py-2", + "pr-2", + "mt-2", + "text-white", + "cursor-pointer", + "active:bg-cyan-900", + "scroll-mt-2", + if is_selected { + "bg-cyan-900" + } else { + "bg-neutral-800" + } + ); + + let icon = render_icon(result); + let metadata = render_metadata(result); + + let mut title = result.title.clone(); + if result.url.starts_with("file://") { + if let Ok(url) = Url::parse(&result.url) { + if let Some(path) = shorten_file_path(&url, 3, true) { + title = path; + } + } + } + + let url = Url::parse(&result.crawl_uri); + + let domain = if let Ok(url) = url { + if let Some(path) = shorten_file_path(&url, 3, false) { + html! { {path} } + } else { + html! { + {format!(" {}", result.domain.clone())} + } + } + } else { + html! {} + }; + + html! { + +
+
+ {icon} +
+
+
+
{domain}
+

+ {title} +

+
+ {Html::from_html_unchecked(result.description.clone().into())} +
+ {metadata} +
+ + } +} + +#[derive(Properties, PartialEq, Eq)] +pub struct LensResultProps { + pub id: String, + pub result: LensResult, + pub is_selected: bool, +} + +#[function_component(LensResultItem)] +pub fn lens_result_component(props: &LensResultProps) -> Html { + let is_selected = props.is_selected; + let result = &props.result; + + let component_styles = classes!( + "flex", + "flex-col", + "p-2", + "mt-2", + "text-white", + "rounded", + "scroll-mt-2", + if is_selected { + "bg-cyan-900" + } else { + "bg-neutral-800" + } + ); + + html! { +
+

+ {result.label.clone()} +

+
+ {result.description.clone()} +
+
+ } +} + +fn shorten_file_path(url: &Url, max_segments: usize, show_file_name: bool) -> Option { + if url.scheme() == "file" { + // Attempt to grab the folder this file resides + let path = if let Some(segments) = url.path_segments() { + let mut segs = segments + .into_iter() + .filter_map(|f| { + if f.is_empty() { + None + } else { + decode_uri_component(f) + .map(|s| s.as_string()) + .unwrap_or_else(|_| Some(f.to_string())) + } + }) + .collect::>(); + + if !show_file_name { + segs.pop(); + } + + let num_segs = segs.len(); + if num_segs > max_segments { + segs = segs[(num_segs - max_segments)..].to_vec(); + segs.insert(0, "...".to_string()); + } + + segs.join(" › ") + } else { + let path_str = url.path().to_string(); + decode_uri_component(&path_str) + .map(|s| s.as_string()) + .unwrap_or_else(|_| Some(path_str.to_string())) + .unwrap_or_else(|| path_str.to_string()) + }; + + return Some(path); + } + + None +} From 9554af774eb79d72c447c03e8366edf52794aa86 Mon Sep 17 00:00:00 2001 From: travolin Date: Tue, 25 Apr 2023 16:03:26 -0700 Subject: [PATCH 09/30] update tag copy to execute after document addition (#437) Co-authored-by: Joel Bredeson --- crates/entities/src/models/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/entities/src/models/mod.rs b/crates/entities/src/models/mod.rs index 204245f50..effef9d5d 100644 --- a/crates/entities/src/models/mod.rs +++ b/crates/entities/src/models/mod.rs @@ -59,7 +59,6 @@ pub async fn copy_all_tables( bootstrap_queue::copy_table(from, to).await?; connection::copy_table(from, to).await?; crawl_queue::copy_table(from, to).await?; - document_tag::copy_table(from, to).await?; fetch_history::copy_table(from, to).await?; indexed_document::copy_table(from, to).await?; lens::copy_table(from, to).await?; @@ -67,6 +66,7 @@ pub async fn copy_all_tables( processed_files::copy_table(from, to).await?; resource_rule::copy_table(from, to).await?; tag::copy_table(from, to).await?; + document_tag::copy_table(from, to).await?; Ok(()) } From aa0f484ea17dc3341df4d7477ce4594cd1635579 Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Wed, 26 Apr 2023 09:04:37 -0700 Subject: [PATCH 10/30] wip: web/larger client (#438) * feature: api-only / read-only flags, & basic dockerfiles (#436) * use docker build fixed whisper-rs * getting some basic Dockerfiles for server + web client (meant for local dev) * add a readonly flag, make writer optional. Locking the writer will now throw an error if we're in readonly mode * add flag to startup only API in readonly mode * no need to keep web css in repo, can be built at deploy time * separate api-only & read-only modes * basic working Dockerfile + compose for local web client * working Dockerfile for api-server & add service to docker-compose * add addr cli arg & use if set, add middleware for basic health check * updating tests * use debian:stable-slim instead of scratch, ran into some weird errors * cargo fmt * getting some basic Dockerfiles for server + web client (meant for local dev) * basic working Dockerfile + compose for local web client * messing combing vector + text search resutls * pull web client requests into its own module * update cargo.lock * move similarity search into its own module * renaming Vector -> Similarity result for clarity * load similarity search endpoint from envvars * open result in new tab * adding some timing logs to similarity search * add dev/prod endpoints for web client * similarity_search: separate log & json timings * adding different query boosts * cargo fmt * Add chat request objects * Add chat response * Add import * return documents that were added from `processed_records` * Update for chat test * Update for search index merge * cargo fmt and clippy * remove llama * update cargo.lock * removing jsonrpsee dependency from web client, we'll be using basic http event streams * handle events from search backend * Add additional error types * break when we're finished receiving updates * only print out some ChatUpdates & sleep a little so UI thread has time to repaint * handle different error types correctly * more UI cleanup - make search results column responsive - clear search box after search - don't clear in_progress until tokens are done/error - move query above columns & make more noticeable * use tailwind typography to give generated markdown a little pizazz * model answer as a history log to prep for chat piece * paginate search results * handle follow-up questions in the UI * no need to show progress icon for user chats/queries * cleaning up old user_icon code * add existing doc context to follow-up request * Add context generated event * - push og question & answer into history stack - disable followup until answer is finished * bump netrunner version * cleaning up unused code * route to different lenses based on url structure * add a "WebSearchResult" component to use more lax styling in web client * removing spyglass-clippy stub * point to og whisper-rs crate now that fix has been merged upstream * update whisper-rs code * clear followup box after submission * update transcriptions for tests * make fmt + make clippy --------- Co-authored-by: Joel Bredeson --- .dockerignore | 4 + Cargo.lock | 102 ++++- Cargo.toml | 9 +- apps/web/.gitignore | 1 + apps/web/Cargo.lock | 137 +----- apps/web/Cargo.toml | 8 +- apps/web/Makefile | 1 + apps/web/package-lock.json | 89 ++++ apps/web/package.json | 5 +- apps/web/public/main.css | 1 - apps/web/src/client.rs | 155 +++++++ apps/web/src/constants.rs | 7 +- apps/web/src/main.rs | 19 +- apps/web/src/pages/mod.rs | 20 +- apps/web/src/pages/search.rs | 407 ++++++++++++++---- apps/web/tailwind.config.js | 4 +- crates/shared/src/request.rs | 28 ++ crates/shared/src/response.rs | 56 +++ crates/spyglass-rpc/src/lib.rs | 5 +- crates/spyglass/Cargo.toml | 5 +- crates/spyglass/bin/debug/src/main.rs | 2 +- crates/spyglass/src/api/handler/mod.rs | 2 +- crates/spyglass/src/api/handler/search.rs | 15 +- crates/spyglass/src/api/mod.rs | 25 +- crates/spyglass/src/documents/mod.rs | 12 +- crates/spyglass/src/filesystem/audio.rs | 21 +- crates/spyglass/src/main.rs | 28 +- .../spyglass/src/pipeline/default_pipeline.rs | 2 +- crates/spyglass/src/search/lens.rs | 2 +- crates/spyglass/src/search/mod.rs | 120 ++++-- crates/spyglass/src/search/query.rs | 48 ++- crates/spyglass/src/search/similarity.rs | 145 +++++++ crates/spyglass/src/search/utils.rs | 3 +- crates/spyglass/src/state.rs | 13 +- crates/spyglass/src/task/worker.rs | 10 +- crates/ui-components/src/results.rs | 164 ++++++- docker-compose.yml | 15 + dockerfiles/api-server.Dockerfile | 25 ++ dockerfiles/web.Dockerfile | 13 + fixtures/audio/armstrong.txt | 2 +- 40 files changed, 1385 insertions(+), 345 deletions(-) create mode 100644 .dockerignore create mode 100644 apps/web/.gitignore delete mode 100644 apps/web/public/main.css create mode 100644 apps/web/src/client.rs create mode 100644 crates/spyglass/src/search/similarity.rs create mode 100644 docker-compose.yml create mode 100644 dockerfiles/api-server.Dockerfile create mode 100644 dockerfiles/web.Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..6a878e042 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +assets +docs +fixtures +target diff --git a/Cargo.lock b/Cargo.lock index bf897b26d..e155849db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1036,6 +1036,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "console_log" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8aed40e4edbf4d3b4431ab260b63fdc40f5780a4766824329ea0f1eefe3c0f" +dependencies = [ + "log", + "web-sys", +] + [[package]] name = "const_fn" version = "0.4.9" @@ -2056,6 +2066,10 @@ name = "futures-timer" version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" +dependencies = [ + "gloo-timers", + "send_wrapper", +] [[package]] name = "futures-util" @@ -2946,9 +2960,12 @@ checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" dependencies = [ "http", "hyper", + "log", "rustls", + "rustls-native-certs", "tokio", "tokio-rustls", + "webpki-roots", ] [[package]] @@ -3265,10 +3282,13 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d291e3a5818a2384645fd9756362e6d89cf0541b0b916fa7702ea4a9833608e" dependencies = [ + "jsonrpsee-client-transport", "jsonrpsee-core", + "jsonrpsee-http-client", "jsonrpsee-proc-macros", "jsonrpsee-server", "jsonrpsee-types", + "jsonrpsee-wasm-client", "jsonrpsee-ws-client", "tracing", ] @@ -3279,7 +3299,11 @@ version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "965de52763f2004bc91ac5bcec504192440f0b568a5d621c59d9dbd6f886c3fb" dependencies = [ + "anyhow", + "futures-channel", + "futures-timer", "futures-util", + "gloo-net", "http", "jsonrpsee-core", "jsonrpsee-types", @@ -3320,6 +3344,26 @@ dependencies = [ "thiserror", "tokio", "tracing", + "wasm-bindgen-futures", +] + +[[package]] +name = "jsonrpsee-http-client" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc345b0a43c6bc49b947ebeb936e886a419ee3d894421790c969cc56040542ad" +dependencies = [ + "async-trait", + "hyper", + "hyper-rustls", + "jsonrpsee-core", + "jsonrpsee-types", + "rustc-hash", + "serde", + "serde_json", + "thiserror", + "tokio", + "tracing", ] [[package]] @@ -3371,6 +3415,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "jsonrpsee-wasm-client" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77310456f43c6c89bcba1f6b2fc2a28300da7c341f320f5128f8c83cc63232d" +dependencies = [ + "jsonrpsee-client-transport", + "jsonrpsee-core", + "jsonrpsee-types", +] + [[package]] name = "jsonrpsee-ws-client" version = "0.16.2" @@ -6003,6 +6058,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "send_wrapper" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" + [[package]] name = "sentry" version = "0.30.0" @@ -6568,6 +6629,7 @@ dependencies = [ "thiserror", "tokio", "tokio-retry", + "tower", "tracing", "tracing-appender", "tracing-log", @@ -6673,9 +6735,9 @@ dependencies = [ [[package]] name = "spyglass-netrunner" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3136ddfe60460cb35aa266d3462f397ad75c6d3aeb13e0f19e8ef4393d60519" +checksum = "c3f491f6fe67a4bac08e6c793d744d8446e2460269c5976288276aebcb2689d9" dependencies = [ "anyhow", "async-recursion", @@ -6700,6 +6762,7 @@ dependencies = [ "rusoto_core", "rusoto_s3", "serde", + "serde_json", "sitemap", "spyglass-lens 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "tendril", @@ -8083,6 +8146,7 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "hdrhistogram", "indexmap", "pin-project", "pin-project-lite", @@ -8963,6 +9027,31 @@ dependencies = [ "wast", ] +[[package]] +name = "web" +version = "0.1.0" +dependencies = [ + "console_log", + "futures", + "gloo", + "log", + "markdown", + "reqwest", + "serde", + "serde-wasm-bindgen 0.5.0", + "serde_json", + "shared", + "strum 0.24.1", + "strum_macros 0.24.3", + "thiserror", + "ui-components", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "yew", + "yew-router", +] + [[package]] name = "web-sys" version = "0.3.61" @@ -9090,16 +9179,17 @@ dependencies = [ [[package]] name = "whisper-rs" -version = "0.4.0" -source = "git+https://github.com/tazz4843/whisper-rs?rev=cf278027eeb5aa849360f664de2c34287418c625#cf278027eeb5aa849360f664de2c34287418c625" +version = "0.6.0" +source = "git+https://github.com/tazz4843/whisper-rs?rev=efd18b6cc1ffc2b9561d038e9306251d5248c25d#efd18b6cc1ffc2b9561d038e9306251d5248c25d" dependencies = [ + "dashmap", "whisper-rs-sys", ] [[package]] name = "whisper-rs-sys" -version = "0.3.1" -source = "git+https://github.com/tazz4843/whisper-rs?rev=cf278027eeb5aa849360f664de2c34287418c625#cf278027eeb5aa849360f664de2c34287418c625" +version = "0.4.0" +source = "git+https://github.com/tazz4843/whisper-rs?rev=efd18b6cc1ffc2b9561d038e9306251d5248c25d#efd18b6cc1ffc2b9561d038e9306251d5248c25d" dependencies = [ "bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index d4b810d76..d6d7ca3bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,15 +8,12 @@ members = [ "crates/tauri", "crates/ui-components", + "apps/web", + # Public published crates "crates/spyglass-plugin", "crates/spyglass-lens", - "crates/spyglass-rpc", -] - -exclude = [ - # Excluded due to strict wasm32-unknown-unknown requirement for jsonrpsee-wasm-client - "apps/web" + "crates/spyglass-rpc" ] [profile.release] diff --git a/apps/web/.gitignore b/apps/web/.gitignore new file mode 100644 index 000000000..d17452757 --- /dev/null +++ b/apps/web/.gitignore @@ -0,0 +1 @@ +public/main.css diff --git a/apps/web/Cargo.lock b/apps/web/Cargo.lock index f89f4263b..490729e3e 100644 --- a/apps/web/Cargo.lock +++ b/apps/web/Cargo.lock @@ -29,26 +29,6 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" -[[package]] -name = "async-lock" -version = "2.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" -dependencies = [ - "event-listener", -] - -[[package]] -name = "async-trait" -version = "0.1.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -67,15 +47,6 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" -[[package]] -name = "beef" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" -dependencies = [ - "serde", -] - [[package]] name = "bincode" version = "1.3.3" @@ -268,12 +239,6 @@ dependencies = [ "libc", ] -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - [[package]] name = "fastrand" version = "1.9.0" @@ -321,6 +286,7 @@ checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -343,6 +309,17 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +[[package]] +name = "futures-executor" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.28" @@ -372,16 +349,6 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" -[[package]] -name = "futures-timer" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" -dependencies = [ - "gloo-timers", - "send_wrapper", -] - [[package]] name = "futures-util" version = "0.3.28" @@ -784,69 +751,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "jsonrpsee-client-transport" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965de52763f2004bc91ac5bcec504192440f0b568a5d621c59d9dbd6f886c3fb" -dependencies = [ - "anyhow", - "futures-channel", - "futures-timer", - "futures-util", - "gloo-net", - "jsonrpsee-core", - "thiserror", - "tracing", -] - -[[package]] -name = "jsonrpsee-core" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e70b4439a751a5de7dd5ed55eacff78ebf4ffe0fc009cb1ebb11417f5b536b" -dependencies = [ - "anyhow", - "async-lock", - "async-trait", - "beef", - "futures-channel", - "futures-timer", - "futures-util", - "jsonrpsee-types", - "rustc-hash", - "serde", - "serde_json", - "thiserror", - "tracing", - "wasm-bindgen-futures", -] - -[[package]] -name = "jsonrpsee-types" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bd522fe1ce3702fd94812965d7bb7a3364b1c9aba743944c5a00529aae80f8c" -dependencies = [ - "anyhow", - "beef", - "serde", - "serde_json", - "thiserror", - "tracing", -] - -[[package]] -name = "jsonrpsee-wasm-client" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77310456f43c6c89bcba1f6b2fc2a28300da7c341f320f5128f8c83cc63232d" -dependencies = [ - "jsonrpsee-client-transport", - "jsonrpsee-core", - "jsonrpsee-types", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -1295,12 +1199,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustix" version = "0.37.11" @@ -1359,12 +1257,6 @@ dependencies = [ "libc", ] -[[package]] -name = "send_wrapper" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" - [[package]] name = "serde" version = "1.0.160" @@ -1838,9 +1730,9 @@ name = "web" version = "0.1.0" dependencies = [ "console_log", + "futures", "gloo", - "jsonrpsee-core", - "jsonrpsee-wasm-client", + "gloo-net", "log", "markdown", "reqwest", @@ -1850,6 +1742,7 @@ dependencies = [ "shared", "strum", "strum_macros", + "thiserror", "ui-components", "wasm-bindgen", "wasm-bindgen-futures", diff --git a/apps/web/Cargo.toml b/apps/web/Cargo.toml index 377f7ed0c..c3ae7f48a 100644 --- a/apps/web/Cargo.toml +++ b/apps/web/Cargo.toml @@ -4,22 +4,22 @@ version = "0.1.0" edition = "2021" [dependencies] +"shared" = { path = "../../crates/shared" } console_log = "1.0" +futures = "0.3" gloo = "0.8.0" log = "0.4" markdown = "1.0.0-alpha.7" -reqwest = { version = "0.11", features = ["json"] } +reqwest = { version = "0.11", features = ["json", "stream"] } serde = "1.0" serde_json = "1.0" serde-wasm-bindgen = "0.5" -"shared" = { path = "../../crates/shared" } strum = "0.24" strum_macros = "0.24" +thiserror = "1.0" ui-components = { path = "../../crates/ui-components" } wasm-bindgen = "0.2.83" wasm-bindgen-futures = "0.4.33" web-sys = { version = "0.3.60", features = ["Navigator", "VisibilityState"] } -jsonrpsee-core = "0.16.2" -jsonrpsee-wasm-client = "0.16.2" yew = { version = "0.20.0", features = ["csr"] } yew-router = "0.17" \ No newline at end of file diff --git a/apps/web/Makefile b/apps/web/Makefile index f0a4519e4..8b2b9f59d 100644 --- a/apps/web/Makefile +++ b/apps/web/Makefile @@ -1,5 +1,6 @@ .PHONY: deploy deploy: + npm run build trunk build --release aws s3 cp --recursive dist s3://app.spyglass.fyi diff --git a/apps/web/package-lock.json b/apps/web/package-lock.json index 58a5c88c5..a80566a03 100644 --- a/apps/web/package-lock.json +++ b/apps/web/package-lock.json @@ -6,6 +6,7 @@ "": { "devDependencies": { "@tailwindcss/forms": "^0.5.2", + "@tailwindcss/typography": "^0.5.9", "tailwindcss": "^3.0.24" } }, @@ -110,6 +111,34 @@ "tailwindcss": ">=3.0.0 || >= 3.0.0-alpha.1" } }, + "node_modules/@tailwindcss/typography": { + "version": "0.5.9", + "resolved": "https://registry.npmjs.org/@tailwindcss/typography/-/typography-0.5.9.tgz", + "integrity": "sha512-t8Sg3DyynFysV9f4JDOVISGsjazNb48AeIYQwcL+Bsq5uf4RYL75C1giZ43KISjeDGBaTN3Kxh7Xj/vRSMJUUg==", + "dev": true, + "dependencies": { + "lodash.castarray": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.merge": "^4.6.2", + "postcss-selector-parser": "6.0.10" + }, + "peerDependencies": { + "tailwindcss": ">=3.0.0 || insiders" + } + }, + "node_modules/@tailwindcss/typography/node_modules/postcss-selector-parser": { + "version": "6.0.10", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.10.tgz", + "integrity": "sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w==", + "dev": true, + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/any-promise": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", @@ -478,6 +507,24 @@ "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", "dev": true }, + "node_modules/lodash.castarray": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.castarray/-/lodash.castarray-4.4.0.tgz", + "integrity": "sha512-aVx8ztPv7/2ULbArGJ2Y42bG1mEQ5mGjpdvrbJcJFU3TbYybe+QlLS4pst9zV52ymy2in1KpFPiZnAOATxD4+Q==", + "dev": true + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "dev": true + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -1096,6 +1143,30 @@ "mini-svg-data-uri": "^1.2.3" } }, + "@tailwindcss/typography": { + "version": "0.5.9", + "resolved": "https://registry.npmjs.org/@tailwindcss/typography/-/typography-0.5.9.tgz", + "integrity": "sha512-t8Sg3DyynFysV9f4JDOVISGsjazNb48AeIYQwcL+Bsq5uf4RYL75C1giZ43KISjeDGBaTN3Kxh7Xj/vRSMJUUg==", + "dev": true, + "requires": { + "lodash.castarray": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.merge": "^4.6.2", + "postcss-selector-parser": "6.0.10" + }, + "dependencies": { + "postcss-selector-parser": { + "version": "6.0.10", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.10.tgz", + "integrity": "sha512-IQ7TZdoaqbT+LCpShg46jnZVlhWD2w6iQYAcYXfHARZ7X1t/UGhhceQDs5X0cGqKvYlHNOuv7Oa1xmb0oQuA3w==", + "dev": true, + "requires": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + } + } + } + }, "any-promise": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", @@ -1384,6 +1455,24 @@ "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", "dev": true }, + "lodash.castarray": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.castarray/-/lodash.castarray-4.4.0.tgz", + "integrity": "sha512-aVx8ztPv7/2ULbArGJ2Y42bG1mEQ5mGjpdvrbJcJFU3TbYybe+QlLS4pst9zV52ymy2in1KpFPiZnAOATxD4+Q==", + "dev": true + }, + "lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "dev": true + }, + "lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true + }, "merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", diff --git a/apps/web/package.json b/apps/web/package.json index 5112da096..02cf44a4e 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -4,7 +4,8 @@ "watch": "tailwindcss -i src-css/main.css -o ./public/main.css --minify --watch" }, "devDependencies": { - "@tailwindcss/forms": "^0.5.2", - "tailwindcss": "^3.0.24" + "@tailwindcss/forms": "^0.5.2", + "@tailwindcss/typography": "^0.5.9", + "tailwindcss": "^3.0.24" } } diff --git a/apps/web/public/main.css b/apps/web/public/main.css deleted file mode 100644 index 0f9652607..000000000 --- a/apps/web/public/main.css +++ /dev/null @@ -1 +0,0 @@ -/*! tailwindcss v3.3.1 | MIT License | https://tailwindcss.com*/*,:after,:before{box-sizing:border-box;border:0 solid #e5e7eb}:after,:before{--tw-content:""}html{line-height:1.5;-webkit-text-size-adjust:100%;-moz-tab-size:4;-o-tab-size:4;tab-size:4;font-family:ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Helvetica Neue,Arial,Noto Sans,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol,Noto Color Emoji;font-feature-settings:normal;font-variation-settings:normal}body{margin:0;line-height:inherit}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,pre,samp{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:initial}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}button,input,optgroup,select,textarea{font-family:inherit;font-size:100%;font-weight:inherit;line-height:inherit;color:inherit;margin:0;padding:0}button,select{text-transform:none}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button;background-color:initial;background-image:none}:-moz-focusring{outline:auto}:-moz-ui-invalid{box-shadow:none}progress{vertical-align:initial}::-webkit-inner-spin-button,::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}summary{display:list-item}blockquote,dd,dl,figure,h1,h2,h3,h4,h5,h6,hr,p,pre{margin:0}fieldset{margin:0}fieldset,legend{padding:0}menu,ol,ul{list-style:none;margin:0;padding:0}textarea{resize:vertical}input::-moz-placeholder,textarea::-moz-placeholder{opacity:1;color:#9ca3af}input::placeholder,textarea::placeholder{opacity:1;color:#9ca3af}[role=button],button{cursor:pointer}:disabled{cursor:default}audio,canvas,embed,iframe,img,object,svg,video{display:block;vertical-align:middle}img,video{max-width:100%;height:auto}[hidden]{display:none}*,::backdrop,:after,:before{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:#3b82f680;--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: }.absolute{position:absolute}.relative{position:relative}.sticky{position:sticky}.bottom-0{bottom:0}.left-0{left:0}.right-0{right:0}.top-0{top:0}.z-40{z-index:40}.z-50{z-index:50}.m-auto{margin:auto}.mx-auto{margin-left:auto;margin-right:auto}.-ml-16{margin-left:-4rem}.-mt-2{margin-top:-.5rem}.mb-2{margin-bottom:.5rem}.mb-6{margin-bottom:1.5rem}.ml-auto{margin-left:auto}.mr-2{margin-right:.5rem}.mt-1{margin-top:.25rem}.mt-1\.5{margin-top:.375rem}.mt-2{margin-top:.5rem}.flex{display:flex}.hidden{display:none}.h-12{height:3rem}.h-4{height:1rem}.h-5{height:1.25rem}.h-8{height:2rem}.h-screen{height:100vh}.max-h-10{max-height:2.5rem}.w-12{width:3rem}.w-4{width:1rem}.w-48{width:12rem}.w-5{width:1.25rem}.w-8{width:2rem}.w-\[30rem\]{width:30rem}.w-full{width:100%}.flex-1{flex:1 1 0%}.flex-none{flex:none}.grow{flex-grow:1}@keyframes spin{to{transform:rotate(1turn)}}.animate-spin{animation:spin 1s linear infinite}@keyframes fade-in{0%{opacity:0;transform:translateY(10px)}to{opacity:1;transform:translateY(0)}}.animate-fade-in{animation:fade-in .5s ease-out}.cursor-pointer{cursor:pointer}.resize{resize:both}.scroll-mt-2{scroll-margin-top:.5rem}.flex-row{flex-direction:row}.flex-col{flex-direction:column}.flex-wrap{flex-wrap:wrap}.flex-nowrap{flex-wrap:nowrap}.place-items-center{place-items:center}.items-center{align-items:center}.gap-1{gap:.25rem}.gap-2{gap:.5rem}.gap-4{gap:1rem}.overflow-hidden{overflow:hidden}.overflow-y-auto{overflow-y:auto}.truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.rounded{border-radius:.25rem}.rounded-md{border-radius:.375rem}.border{border-width:1px}.border-t-2{border-top-width:2px}.border-b-2{border-bottom-width:2px}.border-neutral-600{--tw-border-opacity:1;border-color:rgb(82 82 82/var(--tw-border-opacity))}.border-neutral-900{--tw-border-opacity:1;border-color:rgb(23 23 23/var(--tw-border-opacity))}.border-red-700{--tw-border-opacity:1;border-color:rgb(185 28 28/var(--tw-border-opacity))}.bg-cyan-500{--tw-bg-opacity:1;background-color:rgb(6 182 212/var(--tw-bg-opacity))}.bg-cyan-600{--tw-bg-opacity:1;background-color:rgb(8 145 178/var(--tw-bg-opacity))}.bg-cyan-900{--tw-bg-opacity:1;background-color:rgb(22 78 99/var(--tw-bg-opacity))}.bg-green-700{--tw-bg-opacity:1;background-color:rgb(21 128 61/var(--tw-bg-opacity))}.bg-neutral-700{--tw-bg-opacity:1;background-color:rgb(64 64 64/var(--tw-bg-opacity))}.bg-neutral-800{--tw-bg-opacity:1;background-color:rgb(38 38 38/var(--tw-bg-opacity))}.bg-neutral-900{--tw-bg-opacity:1;background-color:rgb(23 23 23/var(--tw-bg-opacity))}.bg-stone-900{--tw-bg-opacity:1;background-color:rgb(28 25 23/var(--tw-bg-opacity))}.bg-transparent{background-color:initial}.p-0{padding:0}.p-0\.5{padding:.125rem}.p-2{padding:.5rem}.p-4{padding:1rem}.px-1{padding-left:.25rem;padding-right:.25rem}.px-2{padding-left:.5rem;padding-right:.5rem}.px-3{padding-left:.75rem;padding-right:.75rem}.px-4{padding-left:1rem;padding-right:1rem}.py-0{padding-top:0;padding-bottom:0}.py-0\.5{padding-top:.125rem;padding-bottom:.125rem}.py-1{padding-top:.25rem;padding-bottom:.25rem}.py-2{padding-top:.5rem;padding-bottom:.5rem}.py-3{padding-top:.75rem;padding-bottom:.75rem}.py-4{padding-top:1rem;padding-bottom:1rem}.pl-6{padding-left:1.5rem}.pr-2{padding-right:.5rem}.pb-2{padding-bottom:.5rem}.text-center{text-align:center}.text-right{text-align:right}.text-2xl{font-size:1.25rem}.text-base{font-size:.875rem}.text-lg{font-size:1rem}.text-sm{font-size:.75rem}.text-xl{font-size:1.125rem}.text-xs{font-size:.625rem}.font-bold{font-weight:700}.font-semibold{font-weight:600}.uppercase{text-transform:uppercase}.leading-5{line-height:1.25rem}.leading-relaxed{line-height:1.625}.text-cyan-500{--tw-text-opacity:1;color:rgb(6 182 212/var(--tw-text-opacity))}.text-cyan-600{--tw-text-opacity:1;color:rgb(8 145 178/var(--tw-text-opacity))}.text-gray-500{--tw-text-opacity:1;color:rgb(107 114 128/var(--tw-text-opacity))}.text-neutral-400{--tw-text-opacity:1;color:rgb(163 163 163/var(--tw-text-opacity))}.text-red-500{--tw-text-opacity:1;color:rgb(239 68 68/var(--tw-text-opacity))}.text-stone-400{--tw-text-opacity:1;color:rgb(168 162 158/var(--tw-text-opacity))}.text-white{--tw-text-opacity:1;color:rgb(255 255 255/var(--tw-text-opacity))}.text-yellow-500{--tw-text-opacity:1;color:rgb(234 179 8/var(--tw-text-opacity))}.placeholder-neutral-600::-moz-placeholder{--tw-placeholder-opacity:1;color:rgb(82 82 82/var(--tw-placeholder-opacity))}.placeholder-neutral-600::placeholder{--tw-placeholder-opacity:1;color:rgb(82 82 82/var(--tw-placeholder-opacity))}.caret-white{caret-color:#fff}.outline-none{outline:2px solid #0000;outline-offset:2px}input:checked~.dot{transform:translateX(100%);background-color:#48bb78}*{scrollbar-width:thin;scrollbar-color:#404040 #171717}::-webkit-scrollbar{width:10px}::-webkit-scrollbar-track{background:#171717}::-webkit-scrollbar-thumb{background:#404040;border-radius:100vh;border:2px solid #171717}::-webkit-scrollbar-thumb:hover{background:#164e63}mark{color:#fff;background-color:rgb(14 116 144/var(--tw-bg-opacity));padding-left:.125rem;padding-right:.125rem}.hover\:bg-cyan-800:hover{--tw-bg-opacity:1;background-color:rgb(21 94 117/var(--tw-bg-opacity))}.hover\:bg-green-900:hover{--tw-bg-opacity:1;background-color:rgb(20 83 45/var(--tw-bg-opacity))}.hover\:bg-neutral-600:hover{--tw-bg-opacity:1;background-color:rgb(82 82 82/var(--tw-bg-opacity))}.hover\:bg-neutral-700:hover{--tw-bg-opacity:1;background-color:rgb(64 64 64/var(--tw-bg-opacity))}.hover\:bg-red-700:hover{--tw-bg-opacity:1;background-color:rgb(185 28 28/var(--tw-bg-opacity))}.hover\:text-white:hover{--tw-text-opacity:1;color:rgb(255 255 255/var(--tw-text-opacity))}.focus\:outline-none:focus{outline:2px solid #0000;outline-offset:2px}.active\:bg-cyan-900:active{--tw-bg-opacity:1;background-color:rgb(22 78 99/var(--tw-bg-opacity))}.active\:bg-neutral-700:active{--tw-bg-opacity:1;background-color:rgb(64 64 64/var(--tw-bg-opacity))}.active\:outline-none:active{outline:2px solid #0000;outline-offset:2px}.group:hover .group-hover\:block{display:block}.group:hover .group-hover\:text-neutral-400{--tw-text-opacity:1;color:rgb(163 163 163/var(--tw-text-opacity))} \ No newline at end of file diff --git a/apps/web/src/client.rs b/apps/web/src/client.rs new file mode 100644 index 000000000..ef36fa3bb --- /dev/null +++ b/apps/web/src/client.rs @@ -0,0 +1,155 @@ +use std::str::Utf8Error; +use std::time::Duration; + +use futures::io::BufReader; +use futures::{AsyncBufReadExt, TryStreamExt}; +use gloo::timers::future::sleep; +use reqwest::Client; +use shared::request::{AskClippyRequest, ClippyContext}; +use shared::response::{ChatErrorType, ChatUpdate, SearchResult}; +use thiserror::Error; +use yew::html::Scope; + +use crate::pages::search::{HistoryItem, HistorySource}; +use crate::{ + constants, + pages::search::{Msg, SearchPage}, +}; + +#[allow(clippy::enum_variant_names)] +#[derive(Error, Debug)] +pub enum ClientError { + #[error("HTTP request error: {0}")] + HttpError(#[from] reqwest::Error), + #[error("RequestError: {0}")] + RequestError(#[from] serde_json::Error), + #[error("Received malformed data: {0}")] + StreamError(#[from] Utf8Error), +} + +pub struct SpyglassClient { + client: Client, + lens: String, +} + +impl SpyglassClient { + pub fn new(lens: String) -> Self { + let client = Client::new(); + Self { client, lens } + } + + pub async fn followup( + &mut self, + followup: &str, + history: &[HistoryItem], + doc_context: &[SearchResult], + link: Scope, + ) -> Result<(), ClientError> { + let mut context = history + .iter() + .filter(|x| x.source != HistorySource::System) + .map(|x| ClippyContext::History(x.source.to_string(), x.value.clone())) + .collect::>(); + + // Add urls to context + for result in doc_context.iter() { + context.push(ClippyContext::DocId(result.doc_id.clone())); + } + + let body = AskClippyRequest { + query: followup.to_string(), + lens: Some(vec![self.lens.clone()]), + context, + }; + + self.handle_request(&body, link.clone()).await + } + + pub async fn search( + &mut self, + query: &str, + link: Scope, + ) -> Result<(), ClientError> { + let body = AskClippyRequest { + query: query.to_string(), + lens: Some(vec![self.lens.clone()]), + context: Vec::new(), + }; + + self.handle_request(&body, link.clone()).await + } + + async fn handle_request( + &mut self, + body: &AskClippyRequest, + link: Scope, + ) -> Result<(), ClientError> { + let url = format!("{}/chat", constants::HTTP_ENDPOINT); + + let res = self + .client + .post(url) + .body(serde_json::to_string(&body)?) + .send() + .await?; + + let res = res + .bytes_stream() + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e)) + .into_async_read(); + + let mut reader = BufReader::new(res); + let mut buf = String::new(); + while reader.read_line(&mut buf).await.is_ok() { + let line = buf.trim_end_matches(|c| c == '\r' || c == '\n'); + let line = line.strip_prefix("data:").unwrap_or(line); + if line.is_empty() { + buf.clear(); + continue; + } + + let update = serde_json::from_str::(line)?; + match update { + ChatUpdate::SearchingDocuments => { + log::info!("ChatUpdate::SearchingDocuments"); + link.send_message(Msg::SetStatus("Searching...".into())) + } + ChatUpdate::DocumentContextAdded(docs) => { + log::info!("ChatUpdate::DocumentContextAdded"); + link.send_message(Msg::SetSearchResults(docs)) + } + ChatUpdate::GeneratingContext => { + log::info!("ChatUpdate::SearchingDocuments"); + link.send_message(Msg::SetStatus("Analyzing documents...".into())) + } + ChatUpdate::ContextGenerated(context) => { + log::info!("ChatUpdate::ContextGenerated {}", context); + link.send_message(Msg::ContextAdded(context)); + } + ChatUpdate::LoadingModel | ChatUpdate::LoadingPrompt => { + link.send_message(Msg::SetStatus("Generating answer...".into())) + } + ChatUpdate::Token(token) => link.send_message(Msg::TokenReceived(token)), + ChatUpdate::EndOfText => { + link.send_message(Msg::SetFinished); + break; + } + ChatUpdate::Error(err) => { + log::error!("ChatUpdate::Error: {err:?}"); + let msg = match err { + ChatErrorType::ContextLengthExceeded(msg) => msg, + ChatErrorType::APIKeyMissing => "No API key".into(), + ChatErrorType::UnknownError(msg) => msg, + }; + link.send_message(Msg::SetError(msg)); + break; + } + } + buf.clear(); + // give ui thread a chance to do something + sleep(Duration::from_millis(50)).await; + } + + Ok(()) + } +} diff --git a/apps/web/src/constants.rs b/apps/web/src/constants.rs index 4605795a3..a34bb1ee9 100644 --- a/apps/web/src/constants.rs +++ b/apps/web/src/constants.rs @@ -1,2 +1,5 @@ -// pub const SEARCH_ENDPOINT: &str = "https://search.spyglass.fyi/search"; -pub const RPC_ENDPOINT: &str = "ws://127.0.0.1:4664"; +// todo: pull these from environment variables? config? +#[cfg(not(debug_assertions))] +pub const HTTP_ENDPOINT: &str = "https://search.spyglass.fyi"; +#[cfg(debug_assertions)] +pub const HTTP_ENDPOINT: &str = "http://127.0.0.1:8757"; diff --git a/apps/web/src/main.rs b/apps/web/src/main.rs index 1ef730d2d..a8903f9e4 100644 --- a/apps/web/src/main.rs +++ b/apps/web/src/main.rs @@ -2,6 +2,7 @@ use wasm_bindgen::{prelude::Closure, JsValue}; use yew::prelude::*; use yew_router::prelude::*; +mod client; mod constants; mod pages; use pages::AppPage; @@ -10,22 +11,20 @@ use pages::AppPage; pub enum Route { #[at("/")] Start, - #[at("/result")] - Result, - #[at("/library")] - MyLibrary, + #[at("/lens/:lens")] + Search { lens: String }, + // #[at("/library")] + // MyLibrary, #[not_found] #[at("/404")] NotFound, } fn switch(routes: Route) -> Html { - if routes == Route::NotFound { - html! {
{"Not Found!"}
} - } else { - html! { - - } + match &routes { + Route::Start => html! { to={Route::Search { lens: "yc".into() }} /> }, + Route::Search { lens } => html! { }, + Route::NotFound => html! {
{"Not Found!"}
}, } } diff --git a/apps/web/src/pages/mod.rs b/apps/web/src/pages/mod.rs index c4b3ac3ce..97bdf883c 100644 --- a/apps/web/src/pages/mod.rs +++ b/apps/web/src/pages/mod.rs @@ -34,7 +34,7 @@ pub fn nav_link(props: &NavLinkProps) -> Html { #[derive(Properties, PartialEq)] pub struct AppPageProps { - pub tab: Route, + pub lens: String, } #[function_component] @@ -47,30 +47,26 @@ pub fn AppPage(props: &AppPageProps) -> Html { {"Spyglass"}
    -
  • - - - {"My Library"} - +
  • + + {props.lens.clone()}
-
+
- +
} diff --git a/apps/web/src/pages/search.rs b/apps/web/src/pages/search.rs index 845245edf..d288f21c5 100644 --- a/apps/web/src/pages/search.rs +++ b/apps/web/src/pages/search.rs @@ -1,76 +1,152 @@ -use jsonrpsee_core::{client::ClientT, rpc_params}; -use jsonrpsee_wasm_client::{Client, WasmClientBuilder}; -use shared::request::SearchParam; -use shared::{ - keyboard::KeyCode, - response::{SearchResult, SearchResults}, -}; -use std::{ - str::FromStr, - sync::{Arc, Mutex}, -}; +use crate::client::SpyglassClient; +use futures::lock::Mutex; +use shared::keyboard::KeyCode; +use shared::response::SearchResult; +use std::str::FromStr; +use std::sync::Arc; +use strum_macros::Display; use ui_components::{ btn::{Btn, BtnType}, - icons::RefreshIcon, - results::SearchResultItem, + icons::{RefreshIcon, SearchIcon}, + results::{ResultPaginator, WebSearchResultItem}, }; +use wasm_bindgen_futures::spawn_local; use web_sys::HtmlInputElement; -use yew::{platform::spawn_local, prelude::*}; +use yew::prelude::*; -use crate::constants::RPC_ENDPOINT; -const RESULT_PREFIX: &str = "result"; +// make sure we only have one connection per client +type Client = Arc>; -pub type RpcMutex = Arc>; +#[derive(Clone, PartialEq, Eq, Display)] +pub enum HistorySource { + #[strum(serialize = "assistant")] + Clippy, + #[strum(serialize = "user")] + User, + #[strum(serialize = "system")] + System, +} + +#[derive(Clone, PartialEq, Eq)] +pub struct HistoryItem { + /// who "wrote" this response + pub source: HistorySource, + pub value: String, +} #[derive(Clone, Debug)] pub enum Msg { HandleKeyboardEvent(KeyboardEvent), + HandleFollowup(String), HandleSearch, - SetClient(RpcMutex), SetSearchResults(Vec), - OpenResult(SearchResult), + ContextAdded(String), + SetError(String), + SetStatus(String), + TokenReceived(String), + SetFinished, +} + +#[derive(Properties, PartialEq)] +pub struct SearchPageProps { + // todo: allow multiple + pub lens: String, } pub struct SearchPage { - rpc_client: Option, + client: Client, + current_query: Option, + history: Vec, + in_progress: bool, results: Vec, - search_wrapper_ref: NodeRef, search_input_ref: NodeRef, + search_wrapper_ref: NodeRef, status_msg: Option, - in_progress: bool, + tokens: Option, + context: Option, } impl Component for SearchPage { type Message = Msg; - type Properties = (); + type Properties = SearchPageProps; fn create(ctx: &yew::Context) -> Self { - let link = ctx.link(); - link.send_future(async move { - let client = WasmClientBuilder::default() - .request_timeout(std::time::Duration::from_secs(10)) - .build(RPC_ENDPOINT) - .await - .expect("Unable to create WsClient"); - Msg::SetClient(Arc::new(Mutex::new(client))) - }); - + let props = ctx.props(); Self { - rpc_client: None, + client: Arc::new(Mutex::new(SpyglassClient::new(props.lens.clone()))), + current_query: None, + history: Vec::new(), + in_progress: false, results: Vec::new(), search_input_ref: Default::default(), search_wrapper_ref: Default::default(), status_msg: None, - in_progress: false, + tokens: None, + context: None, } } fn update(&mut self, ctx: &yew::Context, msg: Self::Message) -> bool { let link = ctx.link(); match msg { - Msg::SetClient(client) => { - self.rpc_client = Some(client); - false + Msg::HandleFollowup(question) => { + log::info!("handling followup: {}", question); + // Push existing question & answer into history + if let Some(value) = &self.current_query { + self.history.push(HistoryItem { + source: HistorySource::User, + value: value.to_owned(), + }); + } + + // Push existing answer into history + if let Some(value) = &self.tokens { + self.history.push(HistoryItem { + source: HistorySource::Clippy, + value: value.to_owned(), + }); + } + + // Push user's question into history + self.history.push(HistoryItem { + source: HistorySource::User, + value: question.clone(), + }); + + self.tokens = None; + self.status_msg = None; + self.context = None; + self.in_progress = true; + + let link = link.clone(); + let client = self.client.clone(); + + let mut cur_history = self.history.clone(); + // Add context to the beginning + if let Some(context) = &self.context { + cur_history.insert( + 0, + HistoryItem { + source: HistorySource::User, + value: context.to_owned(), + }, + ); + } + + let cur_doc_context = self.results.clone(); + + spawn_local(async move { + let mut client = client.lock().await; + if let Err(err) = client + .followup(&question, &cur_history, &cur_doc_context, link.clone()) + .await + { + log::error!("{}", err.to_string()); + link.send_message(Msg::SetError(err.to_string())); + } + }); + + true } Msg::HandleKeyboardEvent(event) => { let key = event.key(); @@ -84,76 +160,80 @@ impl Component for SearchPage { } Msg::HandleSearch => { self.in_progress = true; + self.tokens = None; + self.status_msg = None; self.results = Vec::new(); - let query = self - .search_input_ref - .cast::() - .map(|x| x.value()); + if let Some(search_input) = self.search_input_ref.cast::() { + let query = search_input.value(); + log::info!("handling search! {:?}", query); - log::info!("handling search! {:?}", query); - if let Some(query) = query { + self.current_query = Some(query.clone()); + search_input.set_value(""); self.status_msg = Some(format!("searching: {query}")); + let link = link.clone(); - if let Some(client) = &self.rpc_client { - let client = client.clone(); - spawn_local(async move { - if let Ok(client) = client.lock() { - let params = SearchParam { - lenses: Vec::new(), - query: query, - }; - match client - .request::( - "spyglass_search_docs", - rpc_params![params], - ) - .await - { - Ok(res) => { - link.send_message(Msg::SetSearchResults(res.results)); - } - Err(err) => { - log::error!("error rpc: {}", err); - } - } - } - }); - } + let client = self.client.clone(); + spawn_local(async move { + let mut client = client.lock().await; + if let Err(err) = client.search(&query, link.clone()).await { + log::error!("{}", err.to_string()); + link.send_message(Msg::SetError(err.to_string())); + } + }); } true } Msg::SetSearchResults(results) => { - self.in_progress = false; self.results = results; true } - Msg::OpenResult(result) => { - log::info!("opening result: {}", result.url); + Msg::ContextAdded(context) => { + self.context = Some(context); false } + Msg::SetError(err) => { + self.in_progress = false; + self.status_msg = Some(err); + true + } + Msg::SetFinished => { + self.in_progress = false; + self.status_msg = None; + true + } + Msg::SetStatus(msg) => { + self.status_msg = Some(msg); + true + } + Msg::TokenReceived(token) => { + if let Some(tokens) = self.tokens.as_mut() { + tokens.push_str(&token); + } else { + self.tokens = Some(token.to_owned()); + } + true + } } } fn view(&self, ctx: &yew::Context) -> yew::Html { let link = ctx.link(); - let html = self + + let placeholder = format!("Ask anything related to {}", ctx.props().lens); + + let results = self .results .iter() - .enumerate() - .map(|(idx, res)| { - let open_msg = Msg::OpenResult(res.to_owned()); + .map(|result| { html! { - } }) - .collect::(); - let results = html! {
{html}
}; - + .collect::>(); html! {
@@ -162,7 +242,7 @@ impl Component for SearchPage { id="searchbox" type="text" class="bg-neutral-800 text-white text-2xl py-3 overflow-hidden flex-1 outline-none active:outline-none focus:outline-none caret-white placeholder-neutral-600" - placeholder="how do i resize a window in tauri?" + placeholder={self.current_query.clone().unwrap_or(placeholder)} spellcheck="false" tabindex="-1" onkeyup={link.callback(Msg::HandleKeyboardEvent)} @@ -179,8 +259,165 @@ impl Component for SearchPage { }}
-
{results}
+ {if let Some(query) = &self.current_query { + html! {
{query}
} + } else { html! {}}} +
+ { if !self.history.is_empty() || self.tokens.is_some() || self.status_msg.is_some() { + html! { + + } + } else { + html! {} + }} + +
+ {if !self.results.is_empty() { + html! { + <> +
{"Sources"}
+ {results} + + } + } else { + html! {} + }} +
+
} } } + +#[derive(Properties, PartialEq)] +struct AnswerSectionProps { + pub history: Vec, + pub tokens: Option, + pub status: Option, + #[prop_or_default] + pub in_progress: bool, + #[prop_or_default] + pub on_followup: Callback, +} + +#[function_component(AnswerSection)] +fn answer_section(props: &AnswerSectionProps) -> Html { + let ask_followup = use_node_ref(); + let ask_followup_handle = ask_followup.clone(); + let on_followup_cb = props.on_followup.clone(); + let on_ask_followup = Callback::from(move |event: SubmitEvent| { + event.prevent_default(); + if let Some(node) = ask_followup_handle.cast::() { + on_followup_cb.emit(node.value()); + node.set_value(""); + } + }); + + html! { +
+
{"Answer"}
+
+
+ + { if let Some(tokens) = &props.tokens { + html!{ } + } else if let Some(msg) = &props.status { + html!{ } + } else { + html! {} + }} +
+
+ + +
+
+
+ } +} + +#[derive(Properties, PartialEq)] +struct HistoryLogProps { + pub history: Vec, +} + +#[function_component(HistoryLog)] +fn history_log(props: &HistoryLogProps) -> Html { + let html = props + .history + .iter() + // Skip the initial question, we already show this at the top. + .skip(1) + .map(|item| { + html! { + + } + }) + .collect::(); + html! { <>{html} } +} + +#[derive(Properties, PartialEq)] +struct HistoryLogItemProps { + pub source: HistorySource, + pub tokens: String, + // Is this a item currently generating tokens? + #[prop_or_default] + pub in_progress: bool, +} + +#[function_component(HistoryLogItem)] +fn history_log_item(props: &HistoryLogItemProps) -> Html { + let user_icon = match props.source { + HistorySource::Clippy | HistorySource::System => html! {<>{"🔭"}}, + HistorySource::User => html! {<>{"🧙‍♂️"}}, + }; + + let html = markdown::to_html(&props.tokens.clone()); + let html = html.trim_start_matches("

").to_string(); + let html = html.trim_end_matches("

").to_string(); + let html = format!("{}", html); + + let item_classes = if props.source == HistorySource::User { + classes!("text-white", "font-bold", "text-lg") + } else { + classes!("prose", "prose-invert", "inline") + }; + + html! { +
+

+ {Html::from_html_unchecked(AttrValue::from(html))} + { if props.in_progress && props.source != HistorySource::User { + html! {

} + } else { html! {} }} +

+ { if !props.in_progress && props.source != HistorySource::User { + html! {
{user_icon}
} + } else { + html! {} + }} +
+ } +} diff --git a/apps/web/tailwind.config.js b/apps/web/tailwind.config.js index 7cac5cbe1..b8973fafe 100644 --- a/apps/web/tailwind.config.js +++ b/apps/web/tailwind.config.js @@ -28,6 +28,7 @@ module.exports = { }, animation: { "fade-in": "fade-in 0.5s ease-out", + "pulse-fast": "pulse 1s cubic-bezier(0.4, 0, 0.6, 1) infinite", "wiggle-short": "wiggle 1s ease-in-out 10", "wiggle": "wiggle 1s ease-in-out infinite", } @@ -49,6 +50,7 @@ module.exports = { } }, plugins: [ - require("@tailwindcss/forms")({ strategy: "class" }) + require("@tailwindcss/forms")({ strategy: "class" }), + require("@tailwindcss/typography") ], } diff --git a/crates/shared/src/request.rs b/crates/shared/src/request.rs index c9575a891..72c38aad2 100644 --- a/crates/shared/src/request.rs +++ b/crates/shared/src/request.rs @@ -1,3 +1,4 @@ +use crate::response::DocMetadata; use serde::{Deserialize, Serialize}; use strum_macros::{Display, EnumString}; @@ -56,3 +57,30 @@ pub struct BatchDocumentRequest { pub source: RawDocSource, pub tags: Vec<(String, String)>, } + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub enum ClippyContext { + /// Document the user is asking about + DocId(String), + /// Previous log of questions/answers + History(String, String), +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct AskClippyRequest { + pub query: String, + pub context: Vec, + pub lens: Option>, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum LLMResponsePayload { + Error(String), + Finished, + SearchingDocuments, + DocumentContextAdded(Vec), + GeneratingContext, + LoadingModel, + LoadingPrompt, + Token(String), +} diff --git a/crates/shared/src/response.rs b/crates/shared/src/response.rs index e03040e64..1f848274e 100644 --- a/crates/shared/src/response.rs +++ b/crates/shared/src/response.rs @@ -291,3 +291,59 @@ pub struct DefaultIndices { pub file_paths: Vec, pub extensions: Vec, } + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct SimilarityResultPayload { + pub title: String, + pub url: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct SimilaritySearchResult { + pub id: usize, + pub version: usize, + pub score: f32, + pub payload: SimilarityResultPayload, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct DocMetadata { + pub doc_id: String, + pub title: String, + pub open_url: String, +} + +/// From backend -> client for display +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct SendToAskClippyPayload { + pub question: Option, + pub docs: Vec, +} + +// Rougly in order of occurrence +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +pub enum ChatUpdate { + /// Searching our document index for relevant documents + SearchingDocuments, + /// Documents returned from search + DocumentContextAdded(Vec), + /// Generating context from documents + GeneratingContext, + /// Context that has been generated + ContextGenerated(String), + /// Loading model / sending to API endpoint + LoadingModel, + LoadingPrompt, + /// Tokens being received from model + Token(String), + /// Done! + EndOfText, + Error(ChatErrorType), +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +pub enum ChatErrorType { + ContextLengthExceeded(String), + APIKeyMissing, + UnknownError(String), +} diff --git a/crates/spyglass-rpc/src/lib.rs b/crates/spyglass-rpc/src/lib.rs index 3f0e82825..da6413874 100644 --- a/crates/spyglass-rpc/src/lib.rs +++ b/crates/spyglass-rpc/src/lib.rs @@ -1,4 +1,4 @@ -use jsonrpsee::core::Error; +use jsonrpsee::core::{Error, JsonValue}; use jsonrpsee::proc_macros::rpc; use shared::config::UserSettings; use shared::request::{BatchDocumentRequest, RawDocumentRequest, SearchLensesParam, SearchParam}; @@ -18,6 +18,9 @@ pub trait Rpc { #[method(name = "protocol_version")] fn protocol_version(&self) -> Result; + #[method(name = "system_health")] + fn system_health(&self) -> Result; + /// Adds an unparsed document to the spyglass index. #[method(name = "index.add_raw_document")] async fn add_raw_document(&self, doc: RawDocumentRequest) -> Result<(), Error>; diff --git a/crates/spyglass/Cargo.toml b/crates/spyglass/Cargo.toml index a0b4054fd..cf33a8030 100644 --- a/crates/spyglass/Cargo.toml +++ b/crates/spyglass/Cargo.toml @@ -55,6 +55,7 @@ tendril = "0.4.2" thiserror = "1.0.37" tokio = { version = "1", features = ["full"] } tokio-retry = "0.3" +tower = { version = "0.4", features = ["full"] } tracing = "0.1" tracing-appender = "0.2" tracing-log = "0.1.3" @@ -65,7 +66,7 @@ warc = "0.3" warp = "0.3" wasmer = "2.3.0" wasmer-wasi = "2.3.0" -whisper-rs = { git = "https://github.com/tazz4843/whisper-rs", rev = "cf278027eeb5aa849360f664de2c34287418c625" } +whisper-rs = { git = "https://github.com/tazz4843/whisper-rs", rev = "efd18b6cc1ffc2b9561d038e9306251d5248c25d" } # Spyglass libs auth_core = { git = "https://github.com/spyglass-search/third-party-apis", rev = "2a2f4532364e931b4ce5cfdeb55383a43d092391" } @@ -75,7 +76,7 @@ reddit = { git = "https://github.com/spyglass-search/third-party-apis", rev = "2 migration = { path = "../migrations" } shared = { path = "../shared", features = ["metrics"] } -spyglass-netrunner = "0.2.10" +spyglass-netrunner = "0.2.11" spyglass-plugin = { path = "../spyglass-plugin" } spyglass-rpc = { path = "../spyglass-rpc" } diff --git a/crates/spyglass/bin/debug/src/main.rs b/crates/spyglass/bin/debug/src/main.rs index 21ad4298d..f927d75b6 100644 --- a/crates/spyglass/bin/debug/src/main.rs +++ b/crates/spyglass/bin/debug/src/main.rs @@ -211,7 +211,7 @@ async fn main() -> anyhow::Result { } let config = Config::new(); - let state = AppState::new(&config).await; + let state = AppState::new(&config, false).await; let lens = shared::config::LensConfig { author: "spyglass-search".into(), diff --git a/crates/spyglass/src/api/handler/mod.rs b/crates/spyglass/src/api/handler/mod.rs index 9e05d3811..a516b40d4 100644 --- a/crates/spyglass/src/api/handler/mod.rs +++ b/crates/spyglass/src/api/handler/mod.rs @@ -670,7 +670,7 @@ mod test { ..Default::default() }; - if let Ok(mut writer) = state.index.writer.lock() { + if let Ok(mut writer) = state.index.lock_writer() { Searcher::upsert_document( &mut writer, DocumentUpdate { diff --git a/crates/spyglass/src/api/handler/search.rs b/crates/spyglass/src/api/handler/search.rs index 88d4b63df..706b6c905 100644 --- a/crates/spyglass/src/api/handler/search.rs +++ b/crates/spyglass/src/api/handler/search.rs @@ -31,6 +31,7 @@ pub async fn search_docs( let start = SystemTime::now(); let index = &state.index; let searcher = index.reader.searcher(); + let query = search_req.query.clone(); let tags = tag::Entity::find() .filter(tag::Column::Label.eq(tag::TagType::Lens.to_string())) @@ -44,19 +45,13 @@ pub async fn search_docs( .collect::>(); let mut stats = QueryStats::new(); - let docs = Searcher::search_with_lens( - state.db.clone(), - &tag_ids, - index, - &search_req.query, - &mut stats, - ) - .await; + + let docs = + Searcher::search_with_lens(&state.db, &tag_ids, index, &query, &[], &mut stats).await; let mut results: Vec = Vec::new(); let mut missing: Vec<(String, String)> = Vec::new(); - let query = search_req.query.clone(); for (score, doc_addr) in docs { if let Ok(Ok(doc)) = searcher.doc(doc_addr).map(|doc| document_to_struct(&doc)) { log::debug!("Got id with url {} {}", doc.doc_id, doc.url); @@ -82,11 +77,13 @@ pub async fn search_docs( .index .tokenizer_for_field(fields.content) .expect("Unable to get tokenizer for content field"); + let description = libspyglass::search::utils::generate_highlight_preview( &tokenizer, &query, &doc.content, ); + let result = SearchResult { doc_id: doc.doc_id.clone(), domain: doc.domain, diff --git a/crates/spyglass/src/api/mod.rs b/crates/spyglass/src/api/mod.rs index a3ad91fda..c8e395c35 100644 --- a/crates/spyglass/src/api/mod.rs +++ b/crates/spyglass/src/api/mod.rs @@ -1,7 +1,8 @@ use entities::get_library_stats; use entities::models::indexed_document; use entities::sea_orm::{ColumnTrait, Condition, EntityTrait, QueryFilter}; -use jsonrpsee::core::{async_trait, Error}; +use jsonrpsee::core::{async_trait, Error, JsonValue}; +use jsonrpsee::server::middleware::proxy_get_request::ProxyGetRequestLayer; use jsonrpsee::server::{ServerBuilder, ServerHandle}; use jsonrpsee::types::{SubscriptionEmptyError, SubscriptionResult}; use jsonrpsee::SubscriptionSink; @@ -29,6 +30,10 @@ impl RpcServer for SpyglassRpc { Ok("0.1.2".into()) } + fn system_health(&self) -> Result { + Ok(serde_json::json!({ "health": true })) + } + async fn add_raw_document(&self, req: RawDocumentRequest) -> Result<(), Error> { handler::add_raw_document(&self.state, &req).await } @@ -247,18 +252,28 @@ impl RpcServer for SpyglassRpc { } pub async fn start_api_server( + addr: Option, state: AppState, config: Config, ) -> anyhow::Result<(SocketAddr, ServerHandle)> { - let server_addr = SocketAddr::new( - IpAddr::V4(Ipv4Addr::LOCALHOST), - state.user_settings.load_full().port, + let middleware = tower::ServiceBuilder::new().layer( + ProxyGetRequestLayer::new("/health", "spyglass_system_health") + .expect("Unable to create middleware"), ); - let server = ServerBuilder::default().build(server_addr).await?; + + let ip = addr.unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST)); + let server_addr = SocketAddr::new(ip, state.user_settings.load_full().port); + + let server = ServerBuilder::default() + .set_middleware(middleware) + .build(server_addr) + .await?; + let rpc_module = SpyglassRpc { state: state.clone(), config: config.clone(), }; + let addr = server.local_addr()?; let server_handle = server.start(rpc_module.into_rpc())?; diff --git a/crates/spyglass/src/documents/mod.rs b/crates/spyglass/src/documents/mod.rs index 02461e198..503bc7e9d 100644 --- a/crates/spyglass/src/documents/mod.rs +++ b/crates/spyglass/src/documents/mod.rs @@ -147,7 +147,7 @@ pub async fn process_crawl_results( let url = Url::parse(&crawl_result.url)?; let url_host = url.host_str().unwrap_or(""); // Add document to index - if let Ok(mut index_writer) = state.index.writer.lock() { + if let Ok(mut index_writer) = state.index.lock_writer() { let doc_id = Searcher::upsert_document( &mut index_writer, DocumentUpdate { @@ -228,7 +228,7 @@ pub async fn process_records( state: &AppState, lens: &LensConfig, results: &mut Vec, -) -> anyhow::Result<()> { +) -> anyhow::Result> { // get a list of all urls let parsed_urls = results .iter() @@ -288,7 +288,7 @@ pub async fn process_records( let url_host = url.host_str().unwrap_or(""); // Add document to index let doc_id: Option = { - if let Ok(mut index_writer) = state.index.writer.lock() { + if let Ok(mut index_writer) = state.index.lock_writer() { match Searcher::upsert_document( &mut index_writer, DocumentUpdate { @@ -338,7 +338,7 @@ pub async fn process_records( // Save the data indexed_document::insert_many(&transaction, &updates).await?; transaction.commit().await?; - if let Ok(mut writer) = state.index.writer.lock() { + if let Ok(mut writer) = state.index.lock_writer() { let _ = writer.commit(); } @@ -356,7 +356,7 @@ pub async fn process_records( } } - Ok(()) + Ok(added_entries) } /// Processes an update tags request for the specified documents @@ -459,7 +459,7 @@ pub async fn update_tags( let _ = Searcher::save(state).await; log::debug!("Tag map generated {}", tag_map.len()); - if let Ok(mut index_writer) = state.index.writer.lock() { + if let Ok(mut index_writer) = state.index.lock_writer() { for (_, (doc, ids)) in tag_map.iter() { let _doc_id = Searcher::upsert_document( &mut index_writer, diff --git a/crates/spyglass/src/filesystem/audio.rs b/crates/spyglass/src/filesystem/audio.rs index 69fb0ee99..3e452b40c 100644 --- a/crates/spyglass/src/filesystem/audio.rs +++ b/crates/spyglass/src/filesystem/audio.rs @@ -170,7 +170,9 @@ fn parse_audio_file(path: &PathBuf) -> anyhow::Result { log::debug!("Detected {} audio channels", channels.count()); if channels.count() > 1 { // convert stereo audio to mono for whisper. - samples = convert_stereo_to_mono_audio(&samples); + if let Ok(converted) = convert_stereo_to_mono_audio(&samples) { + samples = converted; + } } log::debug!( @@ -235,7 +237,7 @@ pub fn transcibe_audio( match parse_audio_file(&path) { Ok(audio_file) => { - let mut ctx = match WhisperContext::new(&model_path.to_string_lossy()) { + let ctx = match WhisperContext::new(&model_path.to_string_lossy()) { Ok(ctx) => ctx, Err(err) => { log::warn!("unable to load model: {:?}", err); @@ -243,6 +245,8 @@ pub fn transcibe_audio( } }; + let state = ctx.create_state()?; + res.metadata = Some(audio_file.metadata); let mut params = FullParams::new(SamplingStrategy::default()); @@ -250,14 +254,15 @@ pub fn transcibe_audio( params.set_print_progress(false); params.set_token_timestamps(true); - ctx.full(params, &audio_file.samples) - .expect("failed to convert samples"); - let num_segments = ctx.full_n_segments(); + ctx.full(&state, params, &audio_file.samples)?; + let num_segments = ctx.full_n_segments(&state)?; log::debug!("Extracted {} segments", num_segments); for i in 0..num_segments { - let segment = ctx.full_get_segment_text(i).expect("failed to get segment"); - let start_timestamp = ctx.full_get_segment_t0(i); - let end_timestamp = ctx.full_get_segment_t1(i); + let segment = ctx + .full_get_segment_text(&state, i) + .expect("failed to get segment"); + let start_timestamp = ctx.full_get_segment_t0(&state, i)?; + let end_timestamp = ctx.full_get_segment_t1(&state, i)?; res.segments .push(Segment::new(start_timestamp, end_timestamp, &segment)); } diff --git a/crates/spyglass/src/main.rs b/crates/spyglass/src/main.rs index 2e7792645..fb27b145b 100644 --- a/crates/spyglass/src/main.rs +++ b/crates/spyglass/src/main.rs @@ -8,6 +8,7 @@ use libspyglass::task::{self, AppPause, AppShutdown, ManagerCommand}; use sentry::ClientInitGuard; use shared::config::{self, Config}; use std::io; +use std::net::IpAddr; use tokio::signal; use tokio::sync::{broadcast, mpsc}; use tracing_appender::non_blocking::WorkerGuard; @@ -33,6 +34,15 @@ struct CliArgs { /// Run migrations & basic checks. #[arg(short, long)] check: bool, + /// IP address to host on, defaults to "127.0.0.1". + #[arg(short, long)] + addr: Option, + /// Only enable API server (no indexing, lens install, etc.) + #[arg(long)] + api_only: bool, + /// Only enable readonly functionality + #[arg(long)] + read_only: bool, } #[cfg(feature = "tokio-console")] @@ -165,11 +175,23 @@ async fn main() -> Result<(), ()> { } // Initialize/Load user preferences - let state = AppState::new(&config).await; - if !args.check { + let state = AppState::new(&config, args.read_only).await; + // Only startup API server if we're in readonly mode. + if args.check { + // config check mode, nothing to do. + return Ok(()); + } else if args.api_only { + match api::start_api_server(args.addr, state, config).await { + Ok((_, handle)) => handle.stopped().await, + Err(err) => { + log::error!("Unable to start API server: {err}"); + return Err(()); + } + } + } else { let indexer_handle = start_backend(state.clone(), config.clone()); // API server - let api_handle = api::start_api_server(state, config); + let api_handle = api::start_api_server(args.addr, state, config); let _ = tokio::join!(indexer_handle, api_handle); } diff --git a/crates/spyglass/src/pipeline/default_pipeline.rs b/crates/spyglass/src/pipeline/default_pipeline.rs index 5f227aaea..f73ce76d1 100644 --- a/crates/spyglass/src/pipeline/default_pipeline.rs +++ b/crates/spyglass/src/pipeline/default_pipeline.rs @@ -132,7 +132,7 @@ async fn start_crawl( // Add document to index let doc_id: Option = { - if let Ok(mut index_writer) = state.index.writer.lock() { + if let Ok(mut index_writer) = state.index.lock_writer() { match Searcher::upsert_document( &mut index_writer, DocumentUpdate { diff --git a/crates/spyglass/src/search/lens.rs b/crates/spyglass/src/search/lens.rs index 222389692..3620de5d8 100644 --- a/crates/spyglass/src/search/lens.rs +++ b/crates/spyglass/src/search/lens.rs @@ -274,7 +274,7 @@ mod test { .with_db(db) .with_lenses(&vec![test_lens]) .with_user_settings(&UserSettings::default()) - .with_index(&IndexPath::Memory) + .with_index(&IndexPath::Memory, false) .build(); let filters = lens_to_filters(state, "test").await; diff --git a/crates/spyglass/src/search/mod.rs b/crates/spyglass/src/search/mod.rs index 86da43afe..bd19f6d91 100644 --- a/crates/spyglass/src/search/mod.rs +++ b/crates/spyglass/src/search/mod.rs @@ -3,7 +3,7 @@ use spyglass_plugin::DocumentQuery; use std::collections::HashSet; use std::fmt::{Debug, Error, Formatter}; use std::path::PathBuf; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, MutexGuard}; use std::time::Instant; use anyhow::anyhow; @@ -15,7 +15,7 @@ use tantivy::{schema::*, DocAddress}; use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy}; use uuid::Uuid; -use crate::search::query::{build_document_query, build_query}; +use crate::search::query::{build_document_query, build_query, QueryBoosts}; use crate::state::AppState; use entities::models::{document_tag, indexed_document, tag}; use entities::schema::{self, DocFields, SearchDocument}; @@ -24,6 +24,7 @@ use entities::sea_orm::{prelude::*, DatabaseConnection}; pub mod grouping; pub mod lens; mod query; +pub mod similarity; pub mod utils; pub use query::QueryStats; @@ -42,7 +43,7 @@ pub enum IndexPath { pub struct Searcher { pub index: Index, pub reader: IndexReader, - pub writer: Arc>, + pub writer: Option>>, } #[derive(Clone)] @@ -62,6 +63,12 @@ pub struct DocumentUpdate<'a> { pub tags: &'a [i64], } +#[derive(Clone)] +pub enum QueryBoost { + Url(String), + DocId(String), +} + impl Debug for Searcher { fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { f.debug_struct("Searcher") @@ -79,8 +86,19 @@ impl Debug for ReadonlySearcher { } impl Searcher { + pub fn lock_writer(&self) -> anyhow::Result> { + if let Some(index) = &self.writer { + match index.lock() { + Ok(lock) => Ok(lock), + Err(_) => Err(anyhow!("writer already locked!")), + } + } else { + Err(anyhow!("readonly only mode enabled")) + } + } + pub async fn save(state: &AppState) -> anyhow::Result<()> { - if let Ok(mut writer) = state.index.writer.lock() { + if let Ok(mut writer) = state.index.lock_writer() { match writer.commit() { Ok(_) => Ok(()), Err(err) => Err(anyhow::anyhow!(err.to_string())), @@ -105,7 +123,7 @@ impl Searcher { remove_documents: bool, ) -> anyhow::Result<()> { // Remove from search index, immediately. - if let Ok(mut writer) = state.index.writer.lock() { + if let Ok(mut writer) = state.index.lock_writer() { Searcher::remove_many_from_index(&mut writer, doc_ids)?; }; @@ -198,7 +216,7 @@ impl Searcher { } /// Constructs a new Searcher object w/ the index @ `index_path` - pub fn with_index(index_path: &IndexPath) -> anyhow::Result { + pub fn with_index(index_path: &IndexPath, readonly: bool) -> anyhow::Result { let index = match index_path { IndexPath::LocalPath(path) => schema::initialize_index(path)?, IndexPath::Memory => schema::initialize_in_memory_index(), @@ -206,9 +224,15 @@ impl Searcher { // Should only be one writer at a time. This single IndexWriter is already // multithreaded. - let writer = index - .writer(50_000_000) - .expect("Unable to create index_writer"); + let writer = if readonly { + None + } else { + Some(Arc::new(Mutex::new( + index + .writer(50_000_000) + .expect("Unable to create index_writer"), + ))) + }; // For a search server you will typically create on reader for the entire // lifetime of your program. @@ -221,7 +245,7 @@ impl Searcher { Ok(Searcher { index, reader, - writer: Arc::new(Mutex::new(writer)), + writer, }) } @@ -300,10 +324,11 @@ impl Searcher { } pub async fn search_with_lens( - db: DatabaseConnection, + db: &DatabaseConnection, applied_lenses: &Vec, searcher: &Searcher, query_string: &str, + boosts: &[QueryBoost], stats: &mut QueryStats, ) -> Vec { let start_timer = Instant::now(); @@ -311,7 +336,7 @@ impl Searcher { let mut tag_boosts = HashSet::new(); let favorite_boost = if let Ok(Some(favorited)) = tag::Entity::find() .filter(tag::Column::Label.eq(tag::TagType::Favorited.to_string())) - .one(&db) + .one(db) .await { Some(favorited.id) @@ -319,7 +344,7 @@ impl Searcher { None }; - let tag_checks = get_tag_checks(&db, query_string).await.unwrap_or_default(); + let tag_checks = get_tag_checks(db, query_string).await.unwrap_or_default(); tag_boosts.extend(tag_checks); let index = &searcher.index; @@ -327,18 +352,34 @@ impl Searcher { let fields = DocFields::as_fields(); let searcher = reader.searcher(); let tokenizers = index.tokenizers().clone(); + + let mut docid_boosts = Vec::new(); + let mut url_boosts = Vec::new(); + for boost in boosts { + match boost { + QueryBoost::DocId(doc_id) => docid_boosts.push(doc_id.clone()), + QueryBoost::Url(url) => url_boosts.push(url.clone()), + } + } + + let boosts = QueryBoosts { + tags: tag_boosts.into_iter().collect(), + favorite: favorite_boost, + urls: url_boosts, + doc_ids: docid_boosts, + }; + let query = build_query( index.schema(), tokenizers, fields, query_string, applied_lenses, - tag_boosts.into_iter(), - favorite_boost, stats, + &boosts, ); - let collector = TopDocs::with_limit(5); + let collector = TopDocs::with_limit(10); let top_docs = searcher .search(&query, &collector) @@ -484,15 +525,20 @@ impl ReadonlySearcher { let fields = DocFields::as_fields(); let tantivy_searcher = reader.searcher(); let tokenizers = index.tokenizers().clone(); + let boosts = QueryBoosts { + tags: tag_boosts.into_iter().collect(), + favorite: favorite_boost, + ..Default::default() + }; + let query = build_query( index.schema(), tokenizers.clone(), fields.clone(), query_string, applied_lenses, - tag_boosts.into_iter(), - favorite_boost, stats, + &boosts, ); let mut combined: Vec<(Occur, Box)> = vec![(Occur::Should, Box::new(query))]; @@ -544,6 +590,7 @@ impl ReadonlySearcher { applied_lenses: &Vec, searcher: &ReadonlySearcher, query_string: &str, + boosts: &[QueryBoost], stats: &mut QueryStats, ) -> Vec { let start_timer = Instant::now(); @@ -567,15 +614,31 @@ impl ReadonlySearcher { let fields = DocFields::as_fields(); let searcher = reader.searcher(); let tokenizers = index.tokenizers().clone(); + + let mut docid_boosts = Vec::new(); + let mut url_boosts = Vec::new(); + for boost in boosts { + match boost { + QueryBoost::DocId(doc_id) => docid_boosts.push(doc_id.clone()), + QueryBoost::Url(url) => url_boosts.push(url.clone()), + } + } + + let boosts = QueryBoosts { + tags: tag_boosts.into_iter().collect(), + favorite: favorite_boost, + urls: url_boosts, + doc_ids: docid_boosts, + }; + let query = build_query( index.schema(), tokenizers, fields, query_string, applied_lenses, - tag_boosts.into_iter(), - favorite_boost, stats, + &boosts, ); let collector = TopDocs::with_limit(5); @@ -674,7 +737,7 @@ mod test { use shared::config::{Config, LensConfig}; fn _build_test_index(searcher: &mut Searcher) { - let writer = &mut searcher.writer.lock().unwrap(); + let writer = &mut searcher.lock_writer().unwrap(); Searcher::upsert_document( writer, DocumentUpdate { @@ -775,13 +838,14 @@ mod test { ..Default::default() }; - let mut searcher = Searcher::with_index(&IndexPath::Memory).expect("Unable to open index"); + let mut searcher = + Searcher::with_index(&IndexPath::Memory, false).expect("Unable to open index"); _build_test_index(&mut searcher); let mut stats = QueryStats::new(); let query = "salinas"; let results = - Searcher::search_with_lens(db, &vec![2_u64], &searcher, query, &mut stats).await; + Searcher::search_with_lens(&db, &vec![2_u64], &searcher, query, &[], &mut stats).await; assert_eq!(results.len(), 1); } @@ -797,13 +861,14 @@ mod test { ..Default::default() }; - let mut searcher = Searcher::with_index(&IndexPath::Memory).expect("Unable to open index"); + let mut searcher = + Searcher::with_index(&IndexPath::Memory, false).expect("Unable to open index"); let mut stats = QueryStats::new(); _build_test_index(&mut searcher); let query = "salinas"; let results = - Searcher::search_with_lens(db, &vec![2_u64], &searcher, query, &mut stats).await; + Searcher::search_with_lens(&db, &vec![2_u64], &searcher, query, &[], &mut stats).await; assert_eq!(results.len(), 1); } @@ -819,13 +884,14 @@ mod test { ..Default::default() }; - let mut searcher = Searcher::with_index(&IndexPath::Memory).expect("Unable to open index"); + let mut searcher = + Searcher::with_index(&IndexPath::Memory, false).expect("Unable to open index"); _build_test_index(&mut searcher); let mut stats = QueryStats::new(); let query = "salinasd"; let results = - Searcher::search_with_lens(db, &vec![2_u64], &searcher, query, &mut stats).await; + Searcher::search_with_lens(&db, &vec![2_u64], &searcher, query, &[], &mut stats).await; assert_eq!(results.len(), 0); } } diff --git a/crates/spyglass/src/search/query.rs b/crates/spyglass/src/search/query.rs index 0a46f81e9..8a391b9b7 100644 --- a/crates/spyglass/src/search/query.rs +++ b/crates/spyglass/src/search/query.rs @@ -46,23 +46,29 @@ fn _boosted_phrase(terms: Vec<(usize, Term)>, boost: Score) -> Box { )) } +#[derive(Clone, Default)] +pub struct QueryBoosts { + /// Boosts based on implicit/explicit tag detection + pub tags: Vec, + /// Id of favorited boost + pub favorite: Option, + /// Urls to boost + pub urls: Vec, + /// Specific doc ids to boost + pub doc_ids: Vec, +} + #[allow(clippy::too_many_arguments)] -pub fn build_query( +pub fn build_query( schema: Schema, tokenizers: TokenizerManager, fields: DocFields, query_string: &str, // Applied filters applied_lenses: &Vec, - // Boosts based on implicit/explicit tag detection - tag_boosts: I, - // Id of favorited boost - favorite_boost: Option, stats: &mut QueryStats, -) -> BooleanQuery -where - I: Iterator, -{ + boosts: &QueryBoosts, +) -> BooleanQuery { let content_terms = terms_for_field(&schema, &tokenizers, query_string, fields.content); let title_terms = terms_for_field(&schema, &tokenizers, query_string, fields.title); @@ -96,15 +102,31 @@ where } // Tags that might be represented by search terms (e.g. "repository" or "file") - for tag_id in tag_boosts { + for tag_id in &boosts.tags { term_query.push(( Occur::Should, - _boosted_term(Term::from_field_u64(fields.tags, tag_id as u64), 1.5), + _boosted_term(Term::from_field_u64(fields.tags, *tag_id as u64), 1.5), )) } - let mut combined: QueryVec = vec![(Occur::Must, Box::new(BooleanQuery::new(term_query)))]; + // Greatly boost selected urls + // todo: handle regex/prefixes? + for url in &boosts.urls { + term_query.push(( + Occur::Should, + _boosted_term(Term::from_field_text(fields.url, url), 3.0), + )); + } + // Greatly boost selected docs + for doc_id in &boosts.doc_ids { + term_query.push(( + Occur::Should, + _boosted_term(Term::from_field_text(fields.id, doc_id), 3.0), + )); + } + + let mut combined: QueryVec = vec![(Occur::Must, Box::new(BooleanQuery::new(term_query)))]; for id in applied_lenses { combined.push(( Occur::Must, @@ -113,7 +135,7 @@ where } // Greatly boost content that have our terms + a favorite. - if let Some(favorite_boost) = favorite_boost { + if let Some(favorite_boost) = boosts.favorite { combined.push(( Occur::Should, _boosted_term( diff --git a/crates/spyglass/src/search/similarity.rs b/crates/spyglass/src/search/similarity.rs new file mode 100644 index 000000000..eed7ac8d6 --- /dev/null +++ b/crates/spyglass/src/search/similarity.rs @@ -0,0 +1,145 @@ +use reqwest::Client; +use serde::{Deserialize, Serialize}; +use shared::response::SimilaritySearchResult; +use std::env; +use std::time::SystemTime; + +use crate::search::{document_to_struct, Searcher}; + +const EMBEDDING_ENDPOINT: &str = "SIMILARITY_SEARCH_ENDPOINT"; +const EMBEDDING_PORT: &str = "SIMILARITY_SEARCH_PORT"; +const DEFAULT_HOST: &str = "localhost"; +const DEFAULT_PORT: &str = "8000"; + +#[derive(Deserialize, Serialize)] +struct SimilarityContext { + content: String, + document: String, +} + +#[derive(Deserialize, Serialize, Debug)] +struct SimilarityContextResponse { + context: Vec, +} + +pub async fn similarity_search(query: &str) -> Vec { + let bench_start = SystemTime::now(); + let endpoint = env::var(EMBEDDING_ENDPOINT).unwrap_or(DEFAULT_HOST.into()); + let port = env::var(EMBEDDING_PORT).unwrap_or(DEFAULT_PORT.into()); + log::info!("using {}:{}", endpoint, port); + log::info!( + "env_check: {}ms", + bench_start.elapsed().unwrap().as_millis() + ); + + // search vector db, todo: if enabled/available + let bench_start = SystemTime::now(); + let client = reqwest::Client::builder().build().unwrap(); + log::info!( + "client_build: {}ms", + bench_start.elapsed().unwrap().as_millis() + ); + + // todo: pull endpoint from environment / configuration + let bench_start = SystemTime::now(); + let resp = client + .post(format!("http://{}:{}/search", endpoint, port)) + .json(&serde_json::json!({ "query": query })) + .send() + .await + .unwrap(); + log::info!( + "similarity_search_call: {}ms", + bench_start.elapsed().unwrap().as_millis() + ); + + let bench_start = SystemTime::now(); + let results = resp.json().await.unwrap_or_default(); + log::info!( + "similarity_json: {}ms", + bench_start.elapsed().unwrap().as_millis() + ); + + results +} + +pub async fn generate_similarity_context( + searcher: &Searcher, + query: &str, + doc_ids: &Vec, +) -> String { + let client = Client::new(); + let mut context = Vec::new(); + let doc_sim_start = SystemTime::now(); + log::info!("Generate Similarity for {} docs", doc_ids.len()); + + for doc_id in doc_ids { + if let Some(doc) = Searcher::get_by_id(&searcher.reader, doc_id) { + if let Ok(doc) = document_to_struct(&doc) { + if let Some(doc_context) = + generate_similarity_context_for_doc(&client, query, &doc.content, &doc.url) + .await + { + context.push(doc_context); + } + } + } + } + log::info!( + "generate_similarity_context: {}ms", + doc_sim_start.elapsed().unwrap().as_millis() + ); + context.join("\n") +} + +pub async fn generate_similarity_context_for_doc( + client: &Client, + query: &str, + content: &str, + url: &str, +) -> Option { + let doc_sim_start = SystemTime::now(); + let endpoint = env::var(EMBEDDING_ENDPOINT).unwrap_or(DEFAULT_HOST.into()); + let port = env::var(EMBEDDING_PORT).unwrap_or(DEFAULT_PORT.into()); + log::info!("Generate Similarity Using {}:{}", endpoint, port); + log::info!( + "context env_check: {}ms", + doc_sim_start.elapsed().unwrap().as_millis() + ); + let body = SimilarityContext { + content: String::from(query), + document: String::from(content), + }; + let request = client + .post(format!("http://{}:{}/compare", endpoint, port)) + .json(&body); + + log::info!( + "generate_similarity_context_for_doc: {}ms", + doc_sim_start.elapsed().unwrap().as_millis() + ); + + let doc_sim_start = SystemTime::now(); + + match request.send().await { + Ok(response) => match response.json::().await { + Ok(json) => { + let context = json.context.join("\n"); + let doc_context = format!("URL: {}\n{}", url, context); + log::info!( + "generate_similarity_context_for_doc: {}ms", + doc_sim_start.elapsed().unwrap().as_millis() + ); + Some(doc_context) + } + Err(err) => { + log::error!("Error processing response {:?}", err); + None + } + }, + Err(err) => { + log::error!("Error sending request {:?}", err); + None + } + } +} diff --git a/crates/spyglass/src/search/utils.rs b/crates/spyglass/src/search/utils.rs index 299effa68..eab736b20 100644 --- a/crates/spyglass/src/search/utils.rs +++ b/crates/spyglass/src/search/utils.rs @@ -146,7 +146,8 @@ mod test { #[test] fn test_find_highlights() { - let searcher = Searcher::with_index(&IndexPath::Memory).expect("Unable to open index"); + let searcher = + Searcher::with_index(&IndexPath::Memory, false).expect("Unable to open index"); let blurb = r#"Rust rust is a multi-paradigm, high-level, general-purpose programming"#; let fields = DocFields::as_fields(); diff --git a/crates/spyglass/src/state.rs b/crates/spyglass/src/state.rs index 50844aeac..75f7d6b2f 100644 --- a/crates/spyglass/src/state.rs +++ b/crates/spyglass/src/state.rs @@ -77,17 +77,18 @@ pub struct AppState { pub file_watcher: Arc>>, // Keep track of in-flight tasks pub fetch_limits: Arc>, + pub readonly_mode: bool, } impl AppState { - pub async fn new(config: &Config) -> Self { + pub async fn new(config: &Config, readonly_mode: bool) -> Self { let db = create_connection(config, false) .await .expect("Unable to connect to database"); AppStateBuilder::new() .with_db(db) - .with_index(&IndexPath::LocalPath(config.index_dir())) + .with_index(&IndexPath::LocalPath(config.index_dir()), readonly_mode) .with_lenses(&config.lenses.values().cloned().collect()) .with_pipelines( &config @@ -143,6 +144,7 @@ pub struct AppStateBuilder { lenses: Option>, pipelines: Option>, user_settings: Option, + readonly_mode: Option, } impl AppStateBuilder { @@ -164,7 +166,7 @@ impl AppStateBuilder { let index = if let Some(index) = &self.index { index.to_owned() } else { - Searcher::with_index(&IndexPath::Memory).expect("Unable to open search index") + Searcher::with_index(&IndexPath::Memory, false).expect("Unable to open search index") }; let user_settings = if let Some(settings) = &self.user_settings { @@ -199,6 +201,7 @@ impl AppStateBuilder { file_watcher: Arc::new(Mutex::new(None)), user_settings: Arc::new(ArcSwap::from_pointee(user_settings)), fetch_limits: Arc::new(DashMap::new()), + readonly_mode: self.readonly_mode.unwrap_or_default(), } } @@ -226,14 +229,14 @@ impl AppStateBuilder { self } - pub fn with_index(&mut self, index: &IndexPath) -> &mut Self { + pub fn with_index(&mut self, index: &IndexPath, readonly: bool) -> &mut Self { if let IndexPath::LocalPath(path) = &index { if !path.exists() { let _ = std::fs::create_dir_all(path); } } - self.index = Some(Searcher::with_index(index).expect("Unable to open index")); + self.index = Some(Searcher::with_index(index, readonly).expect("Unable to open index")); self } } diff --git a/crates/spyglass/src/task/worker.rs b/crates/spyglass/src/task/worker.rs index 17852a49a..cf80c311e 100644 --- a/crates/spyglass/src/task/worker.rs +++ b/crates/spyglass/src/task/worker.rs @@ -332,7 +332,7 @@ mod test { let state = AppState::builder() .with_db(db) .with_user_settings(&UserSettings::default()) - .with_index(&IndexPath::Memory) + .with_index(&IndexPath::Memory, false) .build(); // Should skip this lens since it's been bootstrapped already. @@ -350,7 +350,7 @@ mod test { let state = AppState::builder() .with_db(db.clone()) .with_user_settings(&UserSettings::default()) - .with_index(&IndexPath::Memory) + .with_index(&IndexPath::Memory, false) .build(); let model = crawl_queue::ActiveModel { @@ -399,7 +399,7 @@ mod test { let state = AppState::builder() .with_db(db.clone()) .with_user_settings(&UserSettings::default()) - .with_index(&IndexPath::Memory) + .with_index(&IndexPath::Memory, false) .build(); let task = crawl_queue::ActiveModel { @@ -450,7 +450,7 @@ mod test { let state = AppState::builder() .with_db(db.clone()) .with_user_settings(&UserSettings::default()) - .with_index(&IndexPath::Memory) + .with_index(&IndexPath::Memory, false) .build(); let model = crawl_queue::ActiveModel { @@ -515,7 +515,7 @@ mod test { let state = AppState::builder() .with_db(db.clone()) .with_user_settings(&UserSettings::default()) - .with_index(&IndexPath::Memory) + .with_index(&IndexPath::Memory, false) .build(); let task = crawl_queue::ActiveModel { diff --git a/crates/ui-components/src/results.rs b/crates/ui-components/src/results.rs index 32dd408b5..533d99665 100644 --- a/crates/ui-components/src/results.rs +++ b/crates/ui-components/src/results.rs @@ -9,6 +9,7 @@ use super::tag::{Tag, TagIcon}; #[derive(Properties, PartialEq)] pub struct SearchResultProps { pub id: String, + #[prop_or_default] pub onclick: Callback, pub result: SearchResult, #[prop_or_default] @@ -177,22 +178,24 @@ pub fn search_result_component(props: &SearchResultProps) -> Html { html! {} }; + let icon_classes = classes!("mt-1", "flex", "flex-none", "pr-2", "pl-6"); + let title_classes = classes!("text-base", "truncate", "font-semibold", "w-[30rem]"); + html! { -
+
{icon}
{domain}
-

- {title} -

+

{title}

{Html::from_html_unchecked(result.description.clone().into())}
{metadata} +
{result.score}
 } @@ -278,3 +281,156 @@ fn shorten_file_path(url: &Url, max_segments: usize, show_file_name: bool) -> Op None } + +#[function_component(WebSearchResultItem)] +pub fn web_search_result_component(props: &SearchResultProps) -> Html { + let is_selected = props.is_selected; + let result = &props.result; + + let component_styles = classes!( + "flex", + "flex-row", + "gap-4", + "rounded", + "py-2", + "pr-2", + "mt-2", + "text-white", + "cursor-pointer", + "active:bg-cyan-900", + "scroll-mt-2", + if is_selected { + "bg-cyan-900" + } else { + "bg-neutral-800" + } + ); + + let metadata = render_metadata(result); + + let score = { + #[cfg(debug_assertions)] + html! {
{result.score}
} + + #[cfg(not(debug_assertions))] + html! {} + }; + + html! { + +
+
+ website icon +
+
+
+
+ {format!("{}", result.domain.clone())} +
+

{result.title.clone()}

+
+ {Html::from_html_unchecked(result.description.clone().into())} +
+ {metadata} + {score} +
+ + } +} + +#[derive(Properties, PartialEq)] +pub struct ResultPaginatorProps { + pub children: Children, + pub page_size: usize, +} + +#[function_component(ResultPaginator)] +pub fn result_paginator(props: &ResultPaginatorProps) -> Html { + let page: UseStateHandle = use_state(|| 0); + + let num_pages = props.children.len() / props.page_size; + + let result_html = props + .children + .iter() + .skip(*page * props.page_size) + .take(props.page_size) + .collect::>(); + + let mut pages_html = Vec::new(); + let component_classes = classes!( + "cursor-pointer", + "relative", + "block", + "rounded", + "px-3", + "py-1.5", + "text-sm", + "text-neutral-600", + "transition-all", + "duration-300", + "hover:bg-neutral-100", + "dark:text-white", + "dark:hover:bg-neutral-700", + "dark:hover:text-white" + ); + + for page_num in 0..num_pages { + // Highlight the current page + let mut classes = component_classes.clone(); + if *page == page_num { + classes.push("bg-cyan-500"); + } + + let page_handle = page.clone(); + pages_html.push(html! { +
  • + + {page_num + 1} + +
    • + }); + } + + let page_handle = page.clone(); + let handle_previous = move |_| { + if *page_handle > 0 { + page_handle.set(*page_handle - 1); + } + }; + + let page_handle = page.clone(); + let handle_next = move |_| { + if *page_handle < (num_pages - 1) { + page_handle.set(*page_handle + 1); + } + }; + + html! { +
    +
    {result_html}
    + +
    + } +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..5db4f43a1 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +services: + web: + build: + context: . + dockerfile: ./dockerfiles/web.Dockerfile + ports: + - 8080:8080 + depends_on: + - api-server + api-server: + build: + context: . + dockerfile: ./dockerfiles/api-server.Dockerfile + ports: + - 4664:4664 \ No newline at end of file diff --git a/dockerfiles/api-server.Dockerfile b/dockerfiles/api-server.Dockerfile new file mode 100644 index 000000000..7b0264354 --- /dev/null +++ b/dockerfiles/api-server.Dockerfile @@ -0,0 +1,25 @@ +FROM rust:1.68 AS builder + +WORKDIR /usr/src +# Need for a successful whisper-rs build for some reason... +RUN rustup component add rustfmt +# cmake/clang required for llama-rs/whisper-rs builds +RUN apt update -y && apt upgrade -y +RUN apt install build-essential -y \ + cmake \ + clang + +COPY . . +RUN cargo build -p spyglass --bin spyglass --release + +FROM debian:stable-slim +WORKDIR /app +RUN apt update \ + && apt install -y openssl ca-certificates \ + && apt clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY --from=builder /usr/src/target/release/spyglass ./ + +EXPOSE 4664 +CMD ["./spyglass", "--api-only", "--read-only", "--addr", "0.0.0.0"] \ No newline at end of file diff --git a/dockerfiles/web.Dockerfile b/dockerfiles/web.Dockerfile new file mode 100644 index 000000000..23caa640a --- /dev/null +++ b/dockerfiles/web.Dockerfile @@ -0,0 +1,13 @@ +FROM rust:1.68 + +WORKDIR /usr/src/web +RUN rustup target add wasm32-unknown-unknown +RUN cargo install --locked trunk + +COPY ./apps/web /usr/src/web +RUN mkdir -p /usr/crates + +COPY ./crates /usr/crates + +EXPOSE 8080 +CMD ["trunk", "serve", "--address", "0.0.0.0"] \ No newline at end of file diff --git a/fixtures/audio/armstrong.txt b/fixtures/audio/armstrong.txt index 5f14655cc..ccf6cf141 100644 --- a/fixtures/audio/armstrong.txt +++ b/fixtures/audio/armstrong.txt @@ -1 +1 @@ -I'm going to step off the land now. That's one small step for man. One giant leap for mankind. +I'm going to step off the land now. That's one small step for man, one giant leap for mankind. *thump* From 7ce12e019adf31f347aa3588527b37657c53c446 Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Wed, 26 Apr 2023 14:23:56 -0700 Subject: [PATCH 11/30] tweak: breaking apart searcher into a separate crate (#439) * merging ReadonlySearcher w/ Searcher * moving spyglass search functionality/utilities into a separate crate * update tests * removing unused dependencies * removing reliance on DatabaseConnection in spyglass-searcher crate * removing db dependency from spyglass-searcher crate * moving search index schema to spyglass-searcher crate * make number of search results returned configurable * clean up delete_many_by_doc_id * use thiserror to clean up searcher errors --- Cargo.lock | 57 +- Cargo.toml | 3 +- apps/web/Cargo.toml | 1 - crates/entities/src/lib.rs | 1 - .../entities/src/models/indexed_document.rs | 19 + crates/entities/src/models/tag.rs | 32 +- crates/migrations/Cargo.toml | 1 + .../m20221115_000001_local_file_pathfix.rs | 2 +- .../m20230315_000001_migrate_search_schema.rs | 5 +- crates/spyglass-plugin/Cargo.toml | 1 - crates/spyglass-searcher/Cargo.toml | 28 + .../mod.rs => spyglass-searcher/src/lib.rs} | 568 ++++-------------- .../search => spyglass-searcher/src}/query.rs | 15 +- .../src/schema.rs | 0 .../src}/similarity.rs | 6 +- .../search => spyglass-searcher/src}/utils.rs | 23 +- crates/spyglass/Cargo.toml | 4 +- crates/spyglass/bin/debug/src/main.rs | 55 +- crates/spyglass/src/api/handler/mod.rs | 51 +- crates/spyglass/src/api/handler/search.rs | 26 +- crates/spyglass/src/api/mod.rs | 8 +- crates/spyglass/src/connection/github.rs | 9 +- crates/spyglass/src/connection/reddit.rs | 10 +- crates/spyglass/src/documents/mod.rs | 135 ++--- crates/spyglass/src/lib.rs | 1 - .../spyglass/src/pipeline/default_pipeline.rs | 41 +- crates/spyglass/src/pipeline/mod.rs | 5 +- crates/spyglass/src/plugin/exports.rs | 71 ++- crates/spyglass/src/plugin/mod.rs | 100 +++ crates/spyglass/src/search/grouping.rs | 15 - crates/spyglass/src/state.rs | 2 +- crates/spyglass/src/task.rs | 6 +- crates/spyglass/src/{search => task}/lens.rs | 147 +---- crates/spyglass/src/task/worker.rs | 22 +- 34 files changed, 616 insertions(+), 854 deletions(-) create mode 100644 crates/spyglass-searcher/Cargo.toml rename crates/{spyglass/src/search/mod.rs => spyglass-searcher/src/lib.rs} (50%) rename crates/{spyglass/src/search => spyglass-searcher/src}/query.rs (95%) rename crates/{entities => spyglass-searcher}/src/schema.rs (100%) rename crates/{spyglass/src/search => spyglass-searcher/src}/similarity.rs (95%) rename crates/{spyglass/src/search => spyglass-searcher/src}/utils.rs (88%) delete mode 100644 crates/spyglass/src/search/grouping.rs rename crates/spyglass/src/{search => task}/lens.rs (63%) diff --git a/Cargo.lock b/Cargo.lock index e155849db..77c559ae0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1800,12 +1800,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - [[package]] name = "fastdivide" version = "0.4.0" @@ -2714,15 +2708,6 @@ dependencies = [ "ahash", ] -[[package]] -name = "hashlink" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" -dependencies = [ - "hashbrown 0.11.2", -] - [[package]] name = "hashlink" version = "0.8.1" @@ -3870,6 +3855,7 @@ dependencies = [ "rayon", "sea-orm-migration", "shared", + "spyglass-searcher", "tantivy 0.18.1", "tantivy 0.19.2", "tar", @@ -5626,21 +5612,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "rusqlite" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a" -dependencies = [ - "bitflags", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink 0.7.0", - "libsqlite3-sys", - "memchr", - "smallvec", -] - [[package]] name = "rust-stemmers" version = "1.2.0" @@ -6611,7 +6582,6 @@ dependencies = [ "reqwest", "ron", "rubato", - "rusqlite", "sentry", "sentry-tracing", "serde", @@ -6621,6 +6591,7 @@ dependencies = [ "spyglass-netrunner", "spyglass-plugin", "spyglass-rpc", + "spyglass-searcher", "strum 0.24.1", "strum_macros 0.24.3", "symphonia", @@ -6787,7 +6758,6 @@ dependencies = [ "serde", "serde_json", "url", - "uuid 1.3.0", ] [[package]] @@ -6799,6 +6769,26 @@ dependencies = [ "shared", ] +[[package]] +name = "spyglass-searcher" +version = "0.1.0" +dependencies = [ + "anyhow", + "log", + "reqwest", + "ron", + "serde", + "serde_json", + "shared", + "tantivy 0.19.2", + "thiserror", + "tokio", + "tracing", + "tracing-log", + "tracing-subscriber", + "uuid 1.3.0", +] + [[package]] name = "sqlformat" version = "0.2.1" @@ -6844,7 +6834,7 @@ dependencies = [ "futures-executor", "futures-intrusive", "futures-util", - "hashlink 0.8.1", + "hashlink", "hex", "hkdf", "hmac 0.12.1", @@ -9038,7 +9028,6 @@ dependencies = [ "markdown", "reqwest", "serde", - "serde-wasm-bindgen 0.5.0", "serde_json", "shared", "strum 0.24.1", diff --git a/Cargo.toml b/Cargo.toml index d6d7ca3bf..ccb890336 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,8 @@ members = [ # Public published crates "crates/spyglass-plugin", "crates/spyglass-lens", - "crates/spyglass-rpc" + "crates/spyglass-rpc", + "crates/spyglass-searcher" ] [profile.release] diff --git a/apps/web/Cargo.toml b/apps/web/Cargo.toml index c3ae7f48a..0ebedd851 100644 --- a/apps/web/Cargo.toml +++ b/apps/web/Cargo.toml @@ -13,7 +13,6 @@ markdown = "1.0.0-alpha.7" reqwest = { version = "0.11", features = ["json", "stream"] } serde = "1.0" serde_json = "1.0" -serde-wasm-bindgen = "0.5" strum = "0.24" strum_macros = "0.24" thiserror = "1.0" diff --git a/crates/entities/src/lib.rs b/crates/entities/src/lib.rs index fe589643b..2ad62a36c 100644 --- a/crates/entities/src/lib.rs +++ b/crates/entities/src/lib.rs @@ -1,6 +1,5 @@ use std::collections::HashMap; pub mod models; -pub mod schema; pub mod test; pub use sea_orm; diff --git a/crates/entities/src/models/indexed_document.rs b/crates/entities/src/models/indexed_document.rs index 605c88ea6..db1e6de2e 100644 --- a/crates/entities/src/models/indexed_document.rs +++ b/crates/entities/src/models/indexed_document.rs @@ -314,6 +314,25 @@ pub async fn delete_by_rule(db: &DatabaseConnection, rule: &str) -> anyhow::Resu .collect::>()) } +/// Remove by `doc_id` +pub async fn delete_many_by_doc_id( + db: &DatabaseConnection, + doc_ids: &[String], +) -> Result { + let mut num_deleted = 0; + for chunk in doc_ids.chunks(BATCH_SIZE) { + let docs = Entity::find() + .filter(Column::DocId.is_in(chunk.to_vec())) + .all(db) + .await?; + + let dbids: Vec = docs.iter().map(|x| x.id).collect(); + num_deleted += delete_many_by_id(db, &dbids).await?; + } + + Ok(num_deleted) +} + /// Helper method used to delete multiple documents by id. This method will first /// delete all related tag references before deleting the documents pub async fn delete_many_by_id( diff --git a/crates/entities/src/models/tag.rs b/crates/entities/src/models/tag.rs index 04204e735..f70c54674 100644 --- a/crates/entities/src/models/tag.rs +++ b/crates/entities/src/models/tag.rs @@ -1,4 +1,8 @@ -use sea_orm::{entity::prelude::*, Condition, ConnectionTrait, Set}; +use sea_orm::{ + entity::prelude::*, + sea_query::{Expr, Func}, + Condition, ConnectionTrait, Set, +}; use serde::{Deserialize, Serialize}; use strum_macros::{AsRefStr, Display, EnumString}; @@ -344,6 +348,32 @@ pub async fn copy_table( Ok(()) } +pub async fn get_favorite_tag(db: &DatabaseConnection) -> Option { + if let Ok(Some(favorited)) = Entity::find() + .filter(Column::Label.eq(TagType::Favorited.to_string())) + .one(db) + .await + { + Some(favorited.id as u64) + } else { + None + } +} + +// Helper method used to get the list of tag ids that should be included in the search +pub async fn check_query_for_tags(db: &DatabaseConnection, search: &str) -> Vec { + let lower: String = search.to_lowercase(); + let tokens = lower.split(' ').collect::>(); + let expr = Expr::expr(Func::lower(Expr::col(Column::Value))).is_in(tokens); + let tag_rslt = Entity::find().filter(expr).all(db).await; + + if let Ok(tags) = tag_rslt { + return tags.iter().map(|tag| tag.id as u64).collect(); + } else { + Vec::new() + } +} + #[cfg(test)] mod test { use crate::models::tag; diff --git a/crates/migrations/Cargo.toml b/crates/migrations/Cargo.toml index 528c3dc42..84d57dc31 100644 --- a/crates/migrations/Cargo.toml +++ b/crates/migrations/Cargo.toml @@ -15,6 +15,7 @@ log = "0.4" rayon = "1.5" sea-orm-migration = { version = "0.11" } shared = { path = "../shared" } +spyglass-searcher = { path = "../spyglass-searcher" } tantivy_18 = { package="tantivy", version="0.18" } tantivy = "0.19" tar = "0.4" diff --git a/crates/migrations/src/m20221115_000001_local_file_pathfix.rs b/crates/migrations/src/m20221115_000001_local_file_pathfix.rs index 2445a86e0..8b5d743e5 100644 --- a/crates/migrations/src/m20221115_000001_local_file_pathfix.rs +++ b/crates/migrations/src/m20221115_000001_local_file_pathfix.rs @@ -1,7 +1,7 @@ use crate::sea_orm::Statement; -use entities::schema::{DocFields, SearchDocument}; use sea_orm_migration::prelude::*; use sea_orm_migration::sea_orm::ConnectionTrait; +use spyglass_searcher::schema::{DocFields, SearchDocument}; use shared::config::Config; use tantivy::collector::TopDocs; diff --git a/crates/migrations/src/m20230315_000001_migrate_search_schema.rs b/crates/migrations/src/m20230315_000001_migrate_search_schema.rs index 92d139d57..66555c574 100644 --- a/crates/migrations/src/m20230315_000001_migrate_search_schema.rs +++ b/crates/migrations/src/m20230315_000001_migrate_search_schema.rs @@ -4,14 +4,15 @@ use std::time::SystemTime; use std::time::UNIX_EPOCH; use entities::models::schema::v3::SchemaReader; -use entities::schema::DocFields; use sea_orm_migration::prelude::*; use tantivy::DateTime; use tantivy::{schema::*, IndexWriter}; -use entities::schema::{self, mapping_to_schema, SchemaMapping, SearchDocument}; use entities::sea_orm::{ConnectionTrait, Statement}; use shared::config::Config; +use spyglass_searcher::schema::{ + self, mapping_to_schema, DocFields, SchemaMapping, SearchDocument, +}; use crate::utils::migration_utils; pub struct Migration; diff --git a/crates/spyglass-plugin/Cargo.toml b/crates/spyglass-plugin/Cargo.toml index a1a32eeeb..beb4610d4 100644 --- a/crates/spyglass-plugin/Cargo.toml +++ b/crates/spyglass-plugin/Cargo.toml @@ -15,7 +15,6 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" ron = "0.8" url = "2.2" -uuid = { version = "1.0.0", features = ["serde", "v4"], default-features = false } [lib] name = "spyglass_plugin" diff --git a/crates/spyglass-searcher/Cargo.toml b/crates/spyglass-searcher/Cargo.toml new file mode 100644 index 000000000..16e09d525 --- /dev/null +++ b/crates/spyglass-searcher/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "spyglass-searcher" +version = "0.1.0" +authors = ["Spyglass "] +description = "Search related functionality / utilities for Spyglass" +edition = "2021" + +[dependencies] +anyhow = "1.0" +log = "0.4" +serde = "1.0" +serde_json = "1.0" +reqwest = { version = "0.11", features = ["stream", "json"] } +ron = "0.8" +tantivy = "0.19" +thiserror = "1.0" +tracing = "0.1" +tracing-log = "0.1.3" +tracing-subscriber = { version = "0.3", features = ["env-filter", "std"]} +tokio = { version = "1", features = ["full"] } + +# Internal spyglass libs +shared = { path = "../shared" } +uuid = { version = "1.0.0", features = ["serde", "v4"], default-features = false } + +[lib] +path = "src/lib.rs" +crate-type = ["lib"] \ No newline at end of file diff --git a/crates/spyglass/src/search/mod.rs b/crates/spyglass-searcher/src/lib.rs similarity index 50% rename from crates/spyglass/src/search/mod.rs rename to crates/spyglass-searcher/src/lib.rs index bd19f6d91..d8dfc49c9 100644 --- a/crates/spyglass/src/search/mod.rs +++ b/crates/spyglass-searcher/src/lib.rs @@ -1,33 +1,26 @@ use serde::Serialize; -use spyglass_plugin::DocumentQuery; use std::collections::HashSet; use std::fmt::{Debug, Error, Formatter}; use std::path::PathBuf; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::Instant; -use anyhow::anyhow; -use entities::BATCH_SIZE; -use migration::{Expr, Func}; use tantivy::collector::TopDocs; use tantivy::query::{BooleanQuery, Occur, Query, TermQuery}; use tantivy::{schema::*, DocAddress}; use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy}; +use thiserror::Error; use uuid::Uuid; -use crate::search::query::{build_document_query, build_query, QueryBoosts}; -use crate::state::AppState; -use entities::models::{document_tag, indexed_document, tag}; -use entities::schema::{self, DocFields, SearchDocument}; -use entities::sea_orm::{prelude::*, DatabaseConnection}; +pub mod schema; +use schema::{DocFields, SearchDocument}; -pub mod grouping; -pub mod lens; mod query; pub mod similarity; pub mod utils; pub use query::QueryStats; +use query::{build_document_query, build_query, QueryBoosts}; type Score = f32; type SearchResult = (Score, DocAddress); @@ -39,19 +32,6 @@ pub enum IndexPath { Memory, } -#[derive(Clone)] -pub struct Searcher { - pub index: Index, - pub reader: IndexReader, - pub writer: Option>>, -} - -#[derive(Clone)] -pub struct ReadonlySearcher { - pub index: Index, - pub reader: IndexReader, -} - #[derive(Clone)] pub struct DocumentUpdate<'a> { pub doc_id: Option, @@ -67,6 +47,28 @@ pub struct DocumentUpdate<'a> { pub enum QueryBoost { Url(String), DocId(String), + Tag(u64), +} + +#[allow(clippy::enum_variant_names)] +#[derive(Error, Debug)] +pub enum SearchError { + #[error("Unable to perform action on index: {0}")] + IndexError(#[from] tantivy::TantivyError), + #[error("Index is in read only mode")] + ReadOnly, + #[error("Index writer is deadlocked")] + WriterLocked, + #[error("{0}")] + Other(#[from] anyhow::Error), +} + +type SearcherResult = Result; +#[derive(Clone)] +pub struct Searcher { + pub index: Index, + pub reader: IndexReader, + pub writer: Option>>, } impl Debug for Searcher { @@ -77,121 +79,54 @@ impl Debug for Searcher { } } -impl Debug for ReadonlySearcher { - fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { - f.debug_struct("ReadonlySearcher") - .field("index", &self.index) - .finish() +impl Searcher { + pub fn is_readonly(&self) -> bool { + self.writer.is_none() } -} -impl Searcher { - pub fn lock_writer(&self) -> anyhow::Result> { + pub fn lock_writer(&self) -> SearcherResult> { if let Some(index) = &self.writer { match index.lock() { Ok(lock) => Ok(lock), - Err(_) => Err(anyhow!("writer already locked!")), + Err(_) => Err(SearchError::WriterLocked), } } else { - Err(anyhow!("readonly only mode enabled")) + Err(SearchError::ReadOnly) } } - pub async fn save(state: &AppState) -> anyhow::Result<()> { - if let Ok(mut writer) = state.index.lock_writer() { - match writer.commit() { - Ok(_) => Ok(()), - Err(err) => Err(anyhow::anyhow!(err.to_string())), - } - } else { - Ok(()) - } + pub async fn save(&self) -> SearcherResult<()> { + let mut writer = self.lock_writer()?; + writer.commit()?; + Ok(()) } /// Deletes a single entry from the database & index - pub async fn delete_by_id(state: &AppState, doc_id: &str) -> anyhow::Result<()> { - Searcher::delete_many_by_id(state, &[doc_id.into()], true).await?; + pub async fn delete_by_id(&self, doc_id: &str) -> SearcherResult<()> { + self.delete_many_by_id(&[doc_id.into()]).await?; Ok(()) } /// Deletes multiple ids from the searcher at one time. The caller can decide if the /// documents should also be removed from the database by setting the remove_documents /// flag. - pub async fn delete_many_by_id( - state: &AppState, - doc_ids: &[String], - remove_documents: bool, - ) -> anyhow::Result<()> { - // Remove from search index, immediately. - if let Ok(mut writer) = state.index.lock_writer() { - Searcher::remove_many_from_index(&mut writer, doc_ids)?; - }; - - if remove_documents { - // Remove from indexed_doc table - let doc_refs: Vec<&str> = doc_ids.iter().map(AsRef::as_ref).collect(); - // Chunk deletions - for doc_refs in doc_refs.chunks(BATCH_SIZE) { - let doc_refs = doc_refs.to_vec(); - let docs = indexed_document::Entity::find() - .filter(indexed_document::Column::DocId.is_in(doc_refs.clone())) - .all(&state.db) - .await?; - - let dbids: Vec = docs.iter().map(|x| x.id).collect(); - // Remove tags - document_tag::Entity::delete_many() - .filter(document_tag::Column::IndexedDocumentId.is_in(dbids)) - .exec(&state.db) - .await?; - - indexed_document::Entity::delete_many() - .filter(indexed_document::Column::DocId.is_in(doc_refs)) - .exec(&state.db) - .await?; - } - } - Ok(()) - } - - /// Deletes a single entry from the database/index. - pub async fn delete_by_url(state: &AppState, url: &str) -> anyhow::Result<()> { - if let Some(model) = indexed_document::Entity::find() - .filter(indexed_document::Column::Url.eq(url)) - .one(&state.db) - .await? + pub async fn delete_many_by_id(&self, doc_ids: &[String]) -> SearcherResult<()> { { - Self::delete_by_id(state, &model.doc_id).await?; - } - - Ok(()) - } - - /// Remove document w/ `doc_id` from the search index but will still have a - /// reference in the database. - pub fn remove_from_index(writer: &mut IndexWriter, doc_id: &str) -> anyhow::Result<()> { - let fields = DocFields::as_fields(); - writer.delete_term(Term::from_field_text(fields.id, doc_id)); - Ok(()) - } - - /// Removes multiple documents from the index - pub fn remove_many_from_index( - writer: &mut IndexWriter, - doc_ids: &[String], - ) -> anyhow::Result<()> { - let fields = DocFields::as_fields(); - for doc_id in doc_ids { - writer.delete_term(Term::from_field_text(fields.id, doc_id)); + let writer = self.lock_writer()?; + let fields = DocFields::as_fields(); + for doc_id in doc_ids { + writer.delete_term(Term::from_field_text(fields.id, doc_id)); + } } + self.save().await?; Ok(()) } /// Get document with `doc_id` from index. - pub fn get_by_id(reader: &IndexReader, doc_id: &str) -> Option { + pub fn get_by_id(&self, doc_id: &str) -> Option { let fields = DocFields::as_fields(); - let searcher = reader.searcher(); + let searcher = self.reader.searcher(); let query = TermQuery::new( Term::from_field_text(fields.id, doc_id), @@ -216,7 +151,7 @@ impl Searcher { } /// Constructs a new Searcher object w/ the index @ `index_path` - pub fn with_index(index_path: &IndexPath, readonly: bool) -> anyhow::Result { + pub fn with_index(index_path: &IndexPath, readonly: bool) -> SearcherResult { let index = match index_path { IndexPath::LocalPath(path) => schema::initialize_index(path)?, IndexPath::Memory => schema::initialize_in_memory_index(), @@ -249,10 +184,7 @@ impl Searcher { }) } - pub fn upsert_document( - writer: &mut IndexWriter, - doc_update: DocumentUpdate, - ) -> tantivy::Result { + pub fn upsert_document(&self, doc_update: DocumentUpdate) -> SearcherResult { let fields = DocFields::as_fields(); let doc_id = doc_update @@ -269,6 +201,8 @@ impl Searcher { for t in doc_update.tags { doc.add_u64(fields.tags, *t as u64); } + + let writer = self.lock_writer()?; writer.add_document(doc)?; Ok(doc_id) @@ -276,44 +210,21 @@ impl Searcher { /// Helper method to execute a search based on the provided document query pub async fn search_by_query( - db: &DatabaseConnection, - searcher: &Searcher, - query: &DocumentQuery, + &self, + urls: Option>, + ids: Option>, + has_tags: &[u64], + exclude_tags: &[u64], ) -> Vec { - let tag_ids = match &query.has_tags { - Some(include_tags) => { - let tags = tag::get_tags_by_value(db, include_tags) - .await - .unwrap_or_default(); - tags.iter() - .map(|model| model.id as u64) - .collect::>() - } - None => Vec::new(), - }; - - let exclude_tag_ids = match &query.exclude_tags { - Some(excludes) => { - let exclude_tags = tag::get_tags_by_value(db, excludes) - .await - .unwrap_or_default(); - exclude_tags - .iter() - .map(|model| model.id as u64) - .collect::>() - } - None => Vec::new(), - }; - - let urls = query.urls.clone().unwrap_or_default(); - let ids = query.ids.clone().unwrap_or_default(); + let urls = urls.unwrap_or_default(); + let ids = ids.unwrap_or_default(); let fields = DocFields::as_fields(); - let query = build_document_query(fields, &urls, &ids, &tag_ids, &exclude_tag_ids); + let query = build_document_query(fields, &urls, &ids, has_tags, exclude_tags); let collector = tantivy::collector::DocSetCollector; - let reader = &searcher.reader; + let reader = &self.reader; let index_search = reader.searcher(); let docs = index_search @@ -324,47 +235,37 @@ impl Searcher { } pub async fn search_with_lens( - db: &DatabaseConnection, + &self, applied_lenses: &Vec, - searcher: &Searcher, query_string: &str, + favorite_id: Option, boosts: &[QueryBoost], stats: &mut QueryStats, + num_results: usize, ) -> Vec { let start_timer = Instant::now(); - - let mut tag_boosts = HashSet::new(); - let favorite_boost = if let Ok(Some(favorited)) = tag::Entity::find() - .filter(tag::Column::Label.eq(tag::TagType::Favorited.to_string())) - .one(db) - .await - { - Some(favorited.id) - } else { - None - }; - - let tag_checks = get_tag_checks(db, query_string).await.unwrap_or_default(); - tag_boosts.extend(tag_checks); - - let index = &searcher.index; - let reader = &searcher.reader; + let index = &self.index; + let reader = &self.reader; let fields = DocFields::as_fields(); let searcher = reader.searcher(); let tokenizers = index.tokenizers().clone(); + let mut tag_boosts = HashSet::new(); let mut docid_boosts = Vec::new(); let mut url_boosts = Vec::new(); for boost in boosts { match boost { QueryBoost::DocId(doc_id) => docid_boosts.push(doc_id.clone()), QueryBoost::Url(url) => url_boosts.push(url.clone()), + QueryBoost::Tag(tag_id) => { + tag_boosts.insert(*tag_id); + } } } let boosts = QueryBoosts { tags: tag_boosts.into_iter().collect(), - favorite: favorite_boost, + favorite: favorite_id, urls: url_boosts, doc_ids: docid_boosts, }; @@ -379,7 +280,7 @@ impl Searcher { &boosts, ); - let collector = TopDocs::with_limit(10); + let collector = TopDocs::with_limit(num_results); let top_docs = searcher .search(&query, &collector) @@ -399,136 +300,40 @@ impl Searcher { .filter(|(score, _)| *score > 0.0) .collect() } -} - -// Readonly Searcher implementation used for utilities that can run while -// the spyglass system is running -impl ReadonlySearcher { - /// Get document with `doc_id` from index. - pub fn get_by_id(reader: &IndexReader, doc_id: &str) -> Option { - let fields = DocFields::as_fields(); - let searcher = reader.searcher(); - - let query = TermQuery::new( - Term::from_field_text(fields.id, doc_id), - IndexRecordOption::Basic, - ); - - let res = searcher - .search(&query, &TopDocs::with_limit(1)) - .map_or(Vec::new(), |x| x); - - if res.is_empty() { - return None; - } - - if let Some((_, doc_address)) = res.first() { - if let Ok(doc) = searcher.doc(*doc_address) { - return Some(doc); - } - } - - None - } - - /// Constructs a new Searcher object w/ the index @ `index_path` - pub fn with_index(index_path: &IndexPath) -> anyhow::Result { - let index = match index_path { - IndexPath::LocalPath(path) => schema::initialize_index(path)?, - IndexPath::Memory => schema::initialize_in_memory_index(), - }; - - // For a search server you will typically create on reader for the entire - // lifetime of your program. - let reader = index - .reader_builder() - .reload_policy(ReloadPolicy::OnCommit) - .try_into() - .expect("Unable to create reader"); - - Ok(ReadonlySearcher { index, reader }) - } - - /// Helper method to execute a search based on the provided document query - pub async fn search_by_query( - db: &DatabaseConnection, - searcher: &ReadonlySearcher, - query: &DocumentQuery, - ) -> Vec { - let tag_ids = match &query.has_tags { - Some(include_tags) => { - let tags = tag::get_tags_by_value(db, include_tags) - .await - .unwrap_or_default(); - tags.iter() - .map(|model| model.id as u64) - .collect::>() - } - None => Vec::new(), - }; - - let exclude_tag_ids = match &query.exclude_tags { - Some(excludes) => { - let exclude_tags = tag::get_tags_by_value(db, excludes) - .await - .unwrap_or_default(); - exclude_tags - .iter() - .map(|model| model.id as u64) - .collect::>() - } - None => Vec::new(), - }; - - let urls = query.urls.clone().unwrap_or_default(); - let ids = query.ids.clone().unwrap_or_default(); - - let fields = DocFields::as_fields(); - let query = build_document_query(fields, &urls, &ids, &tag_ids, &exclude_tag_ids); - - let collector = tantivy::collector::DocSetCollector; - - let reader = &searcher.reader; - let index_search = reader.searcher(); - - let docs = index_search - .search(&query, &collector) - .expect("Unable to execute query"); - - docs.into_iter().map(|addr| (1.0, addr)).collect() - } pub async fn explain_search_with_lens( - db: &DatabaseConnection, + &self, doc_address: DocAddress, applied_lenses: &Vec, - searcher: &ReadonlySearcher, query_string: &str, + favorite_id: Option, + boosts: &[QueryBoost], stats: &mut QueryStats, ) -> Option { let mut tag_boosts = HashSet::new(); - let favorite_boost = if let Ok(Some(favorited)) = tag::Entity::find() - .filter(tag::Column::Label.eq(tag::TagType::Favorited.to_string())) - .one(db) - .await - { - Some(favorited.id) - } else { - None - }; - - let tag_checks = get_tag_checks(db, query_string).await.unwrap_or_default(); - tag_boosts.extend(tag_checks); + let mut docid_boosts = Vec::new(); + let mut url_boosts = Vec::new(); + for boost in boosts { + match boost { + QueryBoost::DocId(doc_id) => docid_boosts.push(doc_id.clone()), + QueryBoost::Url(url) => url_boosts.push(url.clone()), + QueryBoost::Tag(tag_id) => { + tag_boosts.insert(*tag_id); + } + } + } - let index = &searcher.index; - let reader = &searcher.reader; + let index = &self.index; + let reader = &self.reader; let fields = DocFields::as_fields(); + let tantivy_searcher = reader.searcher(); let tokenizers = index.tokenizers().clone(); let boosts = QueryBoosts { tags: tag_boosts.into_iter().collect(), - favorite: favorite_boost, - ..Default::default() + favorite: favorite_id, + urls: url_boosts, + doc_ids: docid_boosts, }; let query = build_query( @@ -542,7 +347,7 @@ impl ReadonlySearcher { ); let mut combined: Vec<(Occur, Box)> = vec![(Occur::Should, Box::new(query))]; - if let Ok(Ok(doc)) = searcher + if let Ok(Some(doc)) = self .reader .searcher() .doc(doc_address) @@ -584,100 +389,6 @@ impl ReadonlySearcher { } None } - - pub async fn search_with_lens( - db: &DatabaseConnection, - applied_lenses: &Vec, - searcher: &ReadonlySearcher, - query_string: &str, - boosts: &[QueryBoost], - stats: &mut QueryStats, - ) -> Vec { - let start_timer = Instant::now(); - - let mut tag_boosts = HashSet::new(); - let favorite_boost = if let Ok(Some(favorited)) = tag::Entity::find() - .filter(tag::Column::Label.eq(tag::TagType::Favorited.to_string())) - .one(db) - .await - { - Some(favorited.id) - } else { - None - }; - - let tag_checks = get_tag_checks(db, query_string).await.unwrap_or_default(); - tag_boosts.extend(tag_checks); - - let index = &searcher.index; - let reader = &searcher.reader; - let fields = DocFields::as_fields(); - let searcher = reader.searcher(); - let tokenizers = index.tokenizers().clone(); - - let mut docid_boosts = Vec::new(); - let mut url_boosts = Vec::new(); - for boost in boosts { - match boost { - QueryBoost::DocId(doc_id) => docid_boosts.push(doc_id.clone()), - QueryBoost::Url(url) => url_boosts.push(url.clone()), - } - } - - let boosts = QueryBoosts { - tags: tag_boosts.into_iter().collect(), - favorite: favorite_boost, - urls: url_boosts, - doc_ids: docid_boosts, - }; - - let query = build_query( - index.schema(), - tokenizers, - fields, - query_string, - applied_lenses, - stats, - &boosts, - ); - - let collector = TopDocs::with_limit(5); - - let top_docs = searcher - .search(&query, &collector) - .expect("Unable to execute query"); - - log::debug!( - "query `{}` returned {} results from {} docs in {} ms", - query_string, - top_docs.len(), - searcher.num_docs(), - Instant::now().duration_since(start_timer).as_millis() - ); - - top_docs - .into_iter() - // Filter out negative scores - .filter(|(score, _)| *score > 0.0) - .collect() - } -} - -// Helper method used to get the list of tag ids that should be included in the search -async fn get_tag_checks(db: &DatabaseConnection, search: &str) -> Option> { - let lower = search.to_lowercase(); - let tokens = lower.split(' ').collect::>(); - let expr = - Expr::expr(Func::lower(Expr::col(entities::models::tag::Column::Value))).is_in(tokens); - let tag_rslt = entities::models::tag::Entity::find() - .filter(expr) - .all(db) - .await; - - if let Ok(tags) = tag_rslt { - return Some(tags.iter().map(|tag| tag.id).collect::>()); - } - None } #[derive(Serialize)] @@ -705,11 +416,11 @@ fn field_to_u64vec(doc: &Document, field: Field) -> Vec { } /// Helper method used to convert the provided document to a struct -pub fn document_to_struct(doc: &Document) -> anyhow::Result { +pub fn document_to_struct(doc: &Document) -> Option { let fields = DocFields::as_fields(); let doc_id = field_to_string(doc, fields.id); if doc_id.is_empty() { - return Err(anyhow!("Invalid doc_id")); + return None; } let domain = field_to_string(doc, fields.domain); @@ -719,7 +430,7 @@ pub fn document_to_struct(doc: &Document) -> anyhow::Result { let content = field_to_string(doc, fields.content); let tags = field_to_u64vec(doc, fields.tags); - Ok(RetrievedDocument { + Some(RetrievedDocument { doc_id, domain, title, @@ -732,15 +443,11 @@ pub fn document_to_struct(doc: &Document) -> anyhow::Result { #[cfg(test)] mod test { - use crate::search::{DocumentUpdate, IndexPath, QueryStats, Searcher}; - use entities::models::create_connection; - use shared::config::{Config, LensConfig}; + use crate::{DocumentUpdate, IndexPath, QueryStats, Searcher}; - fn _build_test_index(searcher: &mut Searcher) { - let writer = &mut searcher.lock_writer().unwrap(); - Searcher::upsert_document( - writer, - DocumentUpdate { + async fn _build_test_index(searcher: &mut Searcher) { + searcher + .upsert_document(DocumentUpdate { doc_id: None, title: "Of Mice and Men", description: "Of Mice and Men passage", @@ -756,13 +463,11 @@ mod test { debris of the winter’s flooding; and sycamores with mottled, white, recumbent limbs and branches that arch over the pool", tags: &vec![1_i64], - }, - ) - .expect("Unable to add doc"); + }) + .expect("Unable to add doc"); - Searcher::upsert_document( - writer, - DocumentUpdate { + searcher + .upsert_document(DocumentUpdate { doc_id: None, title: "Of Mice and Men", description: "Of Mice and Men passage", @@ -778,13 +483,11 @@ mod test { debris of the winter’s flooding; and sycamores with mottled, white, recumbent limbs and branches that arch over the pool", tags: &vec![2_i64], - }, - ) - .expect("Unable to add doc"); + }) + .expect("Unable to add doc"); - Searcher::upsert_document( - writer, - DocumentUpdate { + searcher + .upsert_document(DocumentUpdate { doc_id: None, title: "Of Cheese and Crackers", description: "Of Cheese and Crackers Passage", @@ -798,12 +501,10 @@ mod test { ac volutpat massa. Vivamus sed imperdiet est, id pretium ex. Praesent suscipit mattis ipsum, a lacinia nunc semper vitae.", tags: &vec![2_i64], - }, - ) - .expect("Unable to add doc"); + }) + .expect("Unable to add doc"); - Searcher::upsert_document( - writer, + searcher.upsert_document( DocumentUpdate { doc_id: None, title:"Frankenstein: The Modern Prometheus", @@ -819,7 +520,7 @@ mod test { ) .expect("Unable to add doc"); - let res = writer.commit(); + let res = searcher.save().await; if let Err(err) = res { println!("{err:?}"); } @@ -830,68 +531,45 @@ mod test { #[tokio::test] pub async fn test_basic_lense_search() { - let db = create_connection(&Config::default(), true).await.unwrap(); - let _lens = LensConfig { - name: "wiki".to_string(), - domains: vec!["en.wikipedia.org".to_string()], - urls: Vec::new(), - ..Default::default() - }; - let mut searcher = Searcher::with_index(&IndexPath::Memory, false).expect("Unable to open index"); - _build_test_index(&mut searcher); + _build_test_index(&mut searcher).await; let mut stats = QueryStats::new(); let query = "salinas"; - let results = - Searcher::search_with_lens(&db, &vec![2_u64], &searcher, query, &[], &mut stats).await; + let results: Vec<(f32, tantivy::DocAddress)> = searcher + .search_with_lens(&vec![2_u64], query, None, &[], &mut stats, 5) + .await; assert_eq!(results.len(), 1); } #[tokio::test] pub async fn test_url_lens_search() { - let db = create_connection(&Config::default(), true).await.unwrap(); - - let _lens = LensConfig { - name: "wiki".to_string(), - domains: Vec::new(), - urls: vec!["https://en.wikipedia.org/mice".to_string()], - ..Default::default() - }; - let mut searcher = Searcher::with_index(&IndexPath::Memory, false).expect("Unable to open index"); let mut stats = QueryStats::new(); - _build_test_index(&mut searcher); + _build_test_index(&mut searcher).await; let query = "salinas"; - let results = - Searcher::search_with_lens(&db, &vec![2_u64], &searcher, query, &[], &mut stats).await; + let results = searcher + .search_with_lens(&vec![2_u64], query, None, &[], &mut stats, 5) + .await; assert_eq!(results.len(), 1); } #[tokio::test] pub async fn test_singular_url_lens_search() { - let db = create_connection(&Config::default(), true).await.unwrap(); - - let _lens = LensConfig { - name: "wiki".to_string(), - domains: Vec::new(), - urls: vec!["https://en.wikipedia.org/mice$".to_string()], - ..Default::default() - }; - let mut searcher = Searcher::with_index(&IndexPath::Memory, false).expect("Unable to open index"); - _build_test_index(&mut searcher); + _build_test_index(&mut searcher).await; let mut stats = QueryStats::new(); let query = "salinasd"; - let results = - Searcher::search_with_lens(&db, &vec![2_u64], &searcher, query, &[], &mut stats).await; + let results = searcher + .search_with_lens(&vec![2_u64], query, None, &[], &mut stats, 5) + .await; assert_eq!(results.len(), 0); } } diff --git a/crates/spyglass/src/search/query.rs b/crates/spyglass-searcher/src/query.rs similarity index 95% rename from crates/spyglass/src/search/query.rs rename to crates/spyglass-searcher/src/query.rs index 8a391b9b7..2d98e1bc5 100644 --- a/crates/spyglass/src/search/query.rs +++ b/crates/spyglass-searcher/src/query.rs @@ -49,9 +49,9 @@ fn _boosted_phrase(terms: Vec<(usize, Term)>, boost: Score) -> Box { #[derive(Clone, Default)] pub struct QueryBoosts { /// Boosts based on implicit/explicit tag detection - pub tags: Vec, + pub tags: Vec, /// Id of favorited boost - pub favorite: Option, + pub favorite: Option, /// Urls to boost pub urls: Vec, /// Specific doc ids to boost @@ -105,7 +105,7 @@ pub fn build_query( for tag_id in &boosts.tags { term_query.push(( Occur::Should, - _boosted_term(Term::from_field_u64(fields.tags, *tag_id as u64), 1.5), + _boosted_term(Term::from_field_u64(fields.tags, *tag_id), 1.5), )) } @@ -138,10 +138,7 @@ pub fn build_query( if let Some(favorite_boost) = boosts.favorite { combined.push(( Occur::Should, - _boosted_term( - Term::from_field_u64(fields.tags, favorite_boost as u64), - 3.0, - ), + _boosted_term(Term::from_field_u64(fields.tags, favorite_boost), 3.0), )); } @@ -153,8 +150,8 @@ pub fn build_document_query( fields: DocFields, urls: &Vec, ids: &Vec, - tags: &Vec, - exclude_tags: &Vec, + tags: &[u64], + exclude_tags: &[u64], ) -> BooleanQuery { let mut term_query: QueryVec = Vec::new(); let mut urls_query: QueryVec = Vec::new(); diff --git a/crates/entities/src/schema.rs b/crates/spyglass-searcher/src/schema.rs similarity index 100% rename from crates/entities/src/schema.rs rename to crates/spyglass-searcher/src/schema.rs diff --git a/crates/spyglass/src/search/similarity.rs b/crates/spyglass-searcher/src/similarity.rs similarity index 95% rename from crates/spyglass/src/search/similarity.rs rename to crates/spyglass-searcher/src/similarity.rs index eed7ac8d6..9d2ffa92e 100644 --- a/crates/spyglass/src/search/similarity.rs +++ b/crates/spyglass-searcher/src/similarity.rs @@ -4,7 +4,7 @@ use shared::response::SimilaritySearchResult; use std::env; use std::time::SystemTime; -use crate::search::{document_to_struct, Searcher}; +use super::{document_to_struct, Searcher}; const EMBEDDING_ENDPOINT: &str = "SIMILARITY_SEARCH_ENDPOINT"; const EMBEDDING_PORT: &str = "SIMILARITY_SEARCH_PORT"; @@ -74,8 +74,8 @@ pub async fn generate_similarity_context( log::info!("Generate Similarity for {} docs", doc_ids.len()); for doc_id in doc_ids { - if let Some(doc) = Searcher::get_by_id(&searcher.reader, doc_id) { - if let Ok(doc) = document_to_struct(&doc) { + if let Some(doc) = searcher.get_by_id(doc_id) { + if let Some(doc) = document_to_struct(&doc) { if let Some(doc_context) = generate_similarity_context_for_doc(&client, query, &doc.content, &doc.url) .await diff --git a/crates/spyglass/src/search/utils.rs b/crates/spyglass-searcher/src/utils.rs similarity index 88% rename from crates/spyglass/src/search/utils.rs rename to crates/spyglass-searcher/src/utils.rs index eab736b20..4b8542937 100644 --- a/crates/spyglass/src/search/utils.rs +++ b/crates/spyglass-searcher/src/utils.rs @@ -1,4 +1,4 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use tantivy::{ fastfield::MultiValuedFastFieldReader, termdict::TermDictionary, tokenizer::TextAnalyzer, DocId, @@ -137,12 +137,25 @@ pub fn generate_highlight_preview(tokenizer: &TextAnalyzer, query: &str, content format!("{}", desc.join(" ")) } +pub fn group_urls_by_scheme(urls: Vec<&str>) -> HashMap<&str, Vec<&str>> { + let mut grouping: HashMap<&str, Vec<&str>> = HashMap::new(); + urls.iter().for_each(|url| { + let part = url.split(':').next(); + if let Some(scheme) = part { + grouping + .entry(scheme) + .and_modify(|list| list.push(url)) + .or_insert_with(|| Vec::from([url.to_owned()])); + } + }); + grouping +} + #[cfg(test)] mod test { - use crate::search::utils::generate_highlight_preview; - use crate::search::{IndexPath, Searcher}; - use entities::schema::DocFields; - use entities::schema::SearchDocument; + use crate::schema::{DocFields, SearchDocument}; + use crate::utils::generate_highlight_preview; + use crate::{IndexPath, Searcher}; #[test] fn test_find_highlights() { diff --git a/crates/spyglass/Cargo.toml b/crates/spyglass/Cargo.toml index cf33a8030..4cbdedc42 100644 --- a/crates/spyglass/Cargo.toml +++ b/crates/spyglass/Cargo.toml @@ -18,7 +18,6 @@ diff-struct = "0.5.1" digest = "0.10" directories = "4.0" docx = { git = "https://github.com/spyglass-search/docx-rs", branch = "master"} -entities = { path = "../entities" } flate2 = "1.0.24" futures = "0.3" glob = "0.3.1" @@ -41,7 +40,6 @@ regex = "1" reqwest = { version = "0.11", features = ["stream", "json"] } ron = "0.8" rubato = "0.12.0" -rusqlite = { version = "*", features = ["bundled"] } sentry = "0.30.0" sentry-tracing = "0.30.0" serde = { version = "1.0", features = ["derive"] } @@ -74,11 +72,13 @@ github = { git = "https://github.com/spyglass-search/third-party-apis", rev = "2 google = { git = "https://github.com/spyglass-search/third-party-apis", rev = "2a2f4532364e931b4ce5cfdeb55383a43d092391" } reddit = { git = "https://github.com/spyglass-search/third-party-apis", rev = "2a2f4532364e931b4ce5cfdeb55383a43d092391" } +entities = { path = "../entities" } migration = { path = "../migrations" } shared = { path = "../shared", features = ["metrics"] } spyglass-netrunner = "0.2.11" spyglass-plugin = { path = "../spyglass-plugin" } spyglass-rpc = { path = "../spyglass-rpc" } +spyglass-searcher = { path = "../spyglass-searcher" } [dev-dependencies] tracing-test = { version = "0.2.4", features = ["no-env-filter"] } diff --git a/crates/spyglass/bin/debug/src/main.rs b/crates/spyglass/bin/debug/src/main.rs index f927d75b6..b1e24a610 100644 --- a/crates/spyglass/bin/debug/src/main.rs +++ b/crates/spyglass/bin/debug/src/main.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use clap::{Parser, Subcommand}; -use entities::models::{self, indexed_document::DocumentIdentifier}; +use entities::models::{self, indexed_document::DocumentIdentifier, tag::check_query_for_tags}; use libspyglass::state::AppState; use ron::ser::PrettyConfig; use shared::config::Config; @@ -10,7 +10,7 @@ use tracing_log::LogTracer; use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter}; use libspyglass::pipeline::cache_pipeline::process_update; -use libspyglass::search::{self, IndexPath, QueryStats, ReadonlySearcher, Searcher}; +use spyglass_searcher::{document_to_struct, IndexPath, QueryBoost, QueryStats, Searcher}; #[cfg(debug_assertions)] const LOG_LEVEL: &str = "spyglassdebug=DEBUG"; @@ -119,28 +119,22 @@ async fn main() -> anyhow::Result { ron::ser::to_string_pretty(&tags, PrettyConfig::new()).unwrap_or_default() ); let index = - ReadonlySearcher::with_index(&IndexPath::LocalPath(config.index_dir())) + Searcher::with_index(&IndexPath::LocalPath(config.index_dir()), true) .expect("Unable to open index."); - let docs = ReadonlySearcher::search_by_query( - &db, - &index, - &DocumentQuery { - urls: Some(vec![doc.url.clone()]), - ..Default::default() - }, - ) - .await; + let docs = index + .search_by_query(Some(vec![doc.url.clone()]), None, &[], &[]) + .await; println!("### Indexed Document ###"); if docs.is_empty() { println!("No indexed document for url {:?}", &doc.url); } else { for (_score, doc_addr) in docs { - if let Ok(Ok(doc)) = index + if let Ok(Some(doc)) = index .reader .searcher() .doc(doc_addr) - .map(|doc| search::document_to_struct(&doc)) + .map(|doc| document_to_struct(&doc)) { println!( "Indexed Document: {}", @@ -171,25 +165,34 @@ async fn main() -> anyhow::Result { } }; - let index = ReadonlySearcher::with_index(&IndexPath::LocalPath(config.index_dir())) + let index = Searcher::with_index(&IndexPath::LocalPath(config.index_dir()), true) .expect("Unable to open index."); - let docs = ReadonlySearcher::search_by_query(&db, &index, &doc_query).await; + let docs = index + .search_by_query(doc_query.urls, doc_query.ids, &[], &[]) + .await; if docs.is_empty() { println!("No indexed document for url {:?}", id_or_url); } else { for (_score, doc_addr) in docs { let mut stats = QueryStats::default(); - let explain = ReadonlySearcher::explain_search_with_lens( - &db, - doc_addr, - &vec![], - &index, - query.as_str(), - &mut stats, - ) - .await; + let boosts = check_query_for_tags(&db, &query) + .await + .iter() + .map(|x| QueryBoost::Tag(*x)) + .collect::>(); + + let explain = index + .explain_search_with_lens( + doc_addr, + &vec![], + query.as_str(), + None, + &boosts, + &mut stats, + ) + .await; match explain { Some(explanation) => { println!( @@ -221,7 +224,7 @@ async fn main() -> anyhow::Result { }; process_update(state.clone(), &lens, archive_path, true).await; - let _ = Searcher::save(&state).await; + let _ = state.index.save().await; } } diff --git a/crates/spyglass/src/api/handler/mod.rs b/crates/spyglass/src/api/handler/mod.rs index a516b40d4..6dd365188 100644 --- a/crates/spyglass/src/api/handler/mod.rs +++ b/crates/spyglass/src/api/handler/mod.rs @@ -17,7 +17,6 @@ use libspyglass::crawler::CrawlResult; use libspyglass::documents::process_crawl_results; use libspyglass::filesystem; use libspyglass::plugin::PluginCommand; -use libspyglass::search::Searcher; use libspyglass::state::AppState; use libspyglass::task::{AppPause, UserSettingsChange}; use num_format::{Locale, ToFormattedString}; @@ -199,11 +198,11 @@ pub async fn app_status(state: AppState) -> Result { /// Remove a doc from the index #[instrument(skip(state))] pub async fn delete_document(state: AppState, id: String) -> Result<(), Error> { - if let Err(e) = Searcher::delete_by_id(&state, &id).await { + if let Err(e) = state.index.delete_by_id(&id).await { log::error!("Unable to delete doc {} due to {}", id, e); return Err(Error::Custom(e.to_string())); } - let _ = Searcher::save(&state).await; + let _ = indexed_document::delete_many_by_doc_id(&state.db, &[id]).await; Ok(()) } @@ -236,10 +235,10 @@ pub async fn delete_domain(state: AppState, domain: String) -> Result<(), Error> if let Ok(indexed) = indexed { log::debug!("removing docs from index"); let indexed_count = indexed.len(); - for result in indexed { - let _ = Searcher::delete_by_id(&state, &result.doc_id).await; - } - let _ = Searcher::save(&state).await; + + let doc_ids: Vec = indexed.iter().map(|x| x.doc_id.to_string()).collect(); + let _ = state.index.delete_many_by_id(&doc_ids).await; + let _ = indexed_document::delete_many_by_doc_id(&state.db, &doc_ids).await; log::debug!("removed {} items from index", indexed_count); } @@ -587,11 +586,14 @@ pub async fn uninstall_lens(state: AppState, config: &Config, name: &str) -> Res if let Ok(ids) = indexed_document::find_by_lens(state.db.clone(), name).await { // - remove from db & index let doc_ids: Vec = ids.iter().map(|x| x.doc_id.to_owned()).collect(); - if let Err(err) = Searcher::delete_many_by_id(&state, &doc_ids, true).await { + let dbids: Vec = ids.iter().map(|x| x.id).collect(); + + // Remove from index + if let Err(err) = state.index.delete_many_by_id(&doc_ids).await { return Err(Error::Custom(err.to_string())); - } else { - let _ = Searcher::save(&state).await; } + // Remove from db + let _ = indexed_document::delete_many_by_id(&state.db, &dbids).await; } // -- remove from crawl queue @@ -654,9 +656,9 @@ mod test { models::{crawl_queue, indexed_document}, test::setup_test_db, }; - use libspyglass::search::{DocumentUpdate, Searcher}; use libspyglass::state::AppState; use shared::config::{Config, LensConfig}; + use spyglass_searcher::DocumentUpdate; #[tokio::test] async fn test_uninstall_lens() { @@ -670,22 +672,19 @@ mod test { ..Default::default() }; - if let Ok(mut writer) = state.index.lock_writer() { - Searcher::upsert_document( - &mut writer, - DocumentUpdate { - doc_id: Some("test_id".into()), - title: "test title", - description: "test desc", - domain: "example.com", - url: "https://example.com/test", - content: "test content", - tags: &[], - }, - ) + state + .index + .upsert_document(DocumentUpdate { + doc_id: Some("test_id".into()), + title: "test title", + description: "test desc", + domain: "example.com", + url: "https://example.com/test", + content: "test content", + tags: &[], + }) .expect("Unable to add doc"); - } - let _ = Searcher::save(&state).await; + let _ = state.index.save().await; let doc = indexed_document::ActiveModel { domain: Set("example.com".into()), diff --git a/crates/spyglass/src/api/handler/search.rs b/crates/spyglass/src/api/handler/search.rs index 706b6c905..de519a9a9 100644 --- a/crates/spyglass/src/api/handler/search.rs +++ b/crates/spyglass/src/api/handler/search.rs @@ -1,16 +1,16 @@ -use entities::models::tag::TagType; +use entities::models::tag::{check_query_for_tags, get_favorite_tag, TagType}; use entities::models::{indexed_document, lens, tag}; -use entities::schema::{DocFields, SearchDocument}; use entities::sea_orm::{ self, prelude::*, sea_query::Expr, FromQueryResult, JoinType, QueryOrder, QuerySelect, }; use jsonrpsee::core::Error; -use libspyglass::search::{document_to_struct, QueryStats, Searcher}; use libspyglass::state::AppState; use libspyglass::task::{CleanupTask, ManagerCommand}; use shared::metrics; use shared::request; use shared::response::{LensResult, SearchLensesResp, SearchMeta, SearchResult, SearchResults}; +use spyglass_searcher::schema::{DocFields, SearchDocument}; +use spyglass_searcher::{document_to_struct, QueryBoost, QueryStats}; use std::collections::HashSet; use std::time::SystemTime; use tracing::instrument; @@ -33,27 +33,33 @@ pub async fn search_docs( let searcher = index.reader.searcher(); let query = search_req.query.clone(); - let tags = tag::Entity::find() + let lens_ids = tag::Entity::find() .filter(tag::Column::Label.eq(tag::TagType::Lens.to_string())) .filter(tag::Column::Value.is_in(search_req.lenses)) .all(&state.db) .await - .unwrap_or_default(); - let tag_ids = tags + .unwrap_or_default() .iter() .map(|model| model.id as u64) .collect::>(); + let mut boosts = Vec::new(); + for tag in check_query_for_tags(&state.db, &query).await { + boosts.push(QueryBoost::Tag(tag)) + } + let favorite_boost = get_favorite_tag(&state.db).await; let mut stats = QueryStats::new(); - let docs = - Searcher::search_with_lens(&state.db, &tag_ids, index, &query, &[], &mut stats).await; + let docs = state + .index + .search_with_lens(&lens_ids, &query, favorite_boost, &boosts, &mut stats, 5) + .await; let mut results: Vec = Vec::new(); let mut missing: Vec<(String, String)> = Vec::new(); for (score, doc_addr) in docs { - if let Ok(Ok(doc)) = searcher.doc(doc_addr).map(|doc| document_to_struct(&doc)) { + if let Ok(Some(doc)) = searcher.doc(doc_addr).map(|doc| document_to_struct(&doc)) { log::debug!("Got id with url {} {}", doc.doc_id, doc.url); let indexed = indexed_document::Entity::find() .filter(indexed_document::Column::DocId.eq(doc.doc_id.clone())) @@ -78,7 +84,7 @@ pub async fn search_docs( .tokenizer_for_field(fields.content) .expect("Unable to get tokenizer for content field"); - let description = libspyglass::search::utils::generate_highlight_preview( + let description = spyglass_searcher::utils::generate_highlight_preview( &tokenizer, &query, &doc.content, diff --git a/crates/spyglass/src/api/mod.rs b/crates/spyglass/src/api/mod.rs index c8e395c35..f0d8e0735 100644 --- a/crates/spyglass/src/api/mod.rs +++ b/crates/spyglass/src/api/mod.rs @@ -1,3 +1,4 @@ +use crate::task::lens::install_lens; use entities::get_library_stats; use entities::models::indexed_document; use entities::sea_orm::{ColumnTrait, Condition, EntityTrait, QueryFilter}; @@ -6,7 +7,6 @@ use jsonrpsee::server::middleware::proxy_get_request::ProxyGetRequestLayer; use jsonrpsee::server::{ServerBuilder, ServerHandle}; use jsonrpsee::types::{SubscriptionEmptyError, SubscriptionResult}; use jsonrpsee::SubscriptionSink; -use libspyglass::search::{self, Searcher}; use libspyglass::state::AppState; use libspyglass::task::{CollectTask, ManagerCommand}; use shared::config::{Config, UserSettings}; @@ -118,7 +118,7 @@ impl RpcServer for SpyglassRpc { } async fn install_lens(&self, lens_name: String) -> Result<(), Error> { - if let Err(error) = search::lens::install_lens(&self.state, &self.config, lens_name).await { + if let Err(error) = install_lens(&self.state, &self.config, lens_name).await { return Err(Error::Custom(error.to_string())); } Ok(()) @@ -164,8 +164,8 @@ impl RpcServer for SpyglassRpc { .map(|m| m.doc_id.clone()) .collect::>(); let _ = connection::revoke_connection(&self.state.db, &api_id, &account).await; - let _ = Searcher::delete_many_by_id(&self.state, &doc_ids, false).await; - let _ = Searcher::save(&self.state).await; + let _ = self.state.index.delete_many_by_id(&doc_ids).await; + let _ = indexed_document::delete_many_by_doc_id(&self.state.db, &doc_ids).await; log::debug!("revoked & deleted {} docs", doc_ids.len()); Ok(()) } diff --git a/crates/spyglass/src/connection/github.rs b/crates/spyglass/src/connection/github.rs index e7ca261cc..7923730a9 100644 --- a/crates/spyglass/src/connection/github.rs +++ b/crates/spyglass/src/connection/github.rs @@ -16,7 +16,6 @@ use super::credentials::connection_secret; use super::{handle_sync_credentials, load_credentials, Connection}; use crate::crawler::{CrawlError, CrawlResult}; use crate::documents::process_crawl_results; -use crate::search::Searcher; use crate::state::AppState; const BUFFER_SYNC_SIZE: usize = 500; @@ -112,7 +111,7 @@ impl GithubConnection { log::error!("Unable to add issue: {}", err); } - if let Err(err) = Searcher::save(state).await { + if let Err(err) = state.index.save().await { log::error!("Unable to save issues: {}", err); } } @@ -156,7 +155,7 @@ impl GithubConnection { log::error!("Unable to add repo: {}", err); } - if let Err(err) = Searcher::save(state).await { + if let Err(err) = state.index.save().await { log::error!("Unable to save repos: {}", err); } } @@ -202,7 +201,7 @@ impl GithubConnection { log::error!("Unable to add repo: {}", err); } - if let Err(err) = Searcher::save(state).await { + if let Err(err) = state.index.save().await { log::error!("Unable to save repos: {}", err); } } @@ -241,7 +240,7 @@ impl GithubConnection { log::debug!("Removing {} unsynced docs", unsynced.len()); let doc_ids: Vec = unsynced.iter().map(|d| d.doc_id.to_owned()).collect(); let dbids: Vec = unsynced.iter().map(|d| d.id).collect(); - let _ = Searcher::delete_many_by_id(state, &doc_ids, false).await; + let _ = state.index.delete_many_by_id(&doc_ids).await; let _ = indexed_document::delete_many_by_id(&state.db, &dbids).await; for doc in unsynced { log::debug!("removed: {}", doc.url) diff --git a/crates/spyglass/src/connection/reddit.rs b/crates/spyglass/src/connection/reddit.rs index 2258a155b..60c0b2275 100644 --- a/crates/spyglass/src/connection/reddit.rs +++ b/crates/spyglass/src/connection/reddit.rs @@ -12,17 +12,15 @@ use libreddit::{ use strum_macros::{Display, EnumString}; use url::Url; +use super::{ + credentials::connection_secret, handle_sync_credentials, load_credentials, Connection, +}; use crate::{ crawler::{CrawlError, CrawlResult}, documents::process_crawl_results, - search::Searcher, state::AppState, }; -use super::{ - credentials::connection_secret, handle_sync_credentials, load_credentials, Connection, -}; - pub struct RedditConnection { client: RedditClient, user: String, @@ -96,7 +94,7 @@ impl RedditConnection { log::warn!("Unable to add posts: {}", err); } - if let Err(err) = Searcher::save(state).await { + if let Err(err) = state.index.save().await { log::warn!("Unable to save posts: {}", err); } } diff --git a/crates/spyglass/src/documents/mod.rs b/crates/spyglass/src/documents/mod.rs index 503bc7e9d..d756a28a1 100644 --- a/crates/spyglass/src/documents/mod.rs +++ b/crates/spyglass/src/documents/mod.rs @@ -16,13 +16,10 @@ use tantivy::DocAddress; use libnetrunner::parser::ParseResult; use url::Url; -use crate::{ - crawler::CrawlResult, - search::{self, DocumentUpdate, RetrievedDocument, Searcher}, - state::AppState, -}; +use crate::{crawler::CrawlResult, state::AppState}; use entities::models::tag::TagType; use entities::sea_orm::{ColumnTrait, EntityTrait, QueryFilter, Set, TransactionTrait}; +use spyglass_searcher::{document_to_struct, DocumentUpdate, RetrievedDocument}; /// Helper method to delete indexed documents, crawl queue items and search /// documents by url @@ -54,14 +51,10 @@ pub async fn delete_documents_by_uri(state: &AppState, uri: Vec) { .map(|x| x.to_owned()) .collect::>(); - if let Err(err) = Searcher::delete_many_by_id(state, &doc_id_list, false).await { + if let Err(err) = state.index.delete_many_by_id(&doc_id_list).await { log::warn!("Unable to delete_many_by_id: {err}") } - if let Err(err) = Searcher::save(state).await { - log::warn!("Unable to save searcher: {err}") - } - // now that the documents are deleted delete from the queue if let Err(error) = indexed_document::delete_many_by_url(&state.db, chunk).await { log::warn!("Error deleting for indexed document store {:?}", error); @@ -121,8 +114,7 @@ pub async fn process_crawl_results( let doc_id_list = id_map.values().cloned().collect::>(); // Delete existing docs - let _ = Searcher::delete_many_by_id(state, &doc_id_list, false).await; - let _ = Searcher::save(state).await; + let _ = state.index.delete_many_by_id(&doc_id_list).await; // Find/create the tags for this crawl. let mut tag_map: HashMap> = HashMap::new(); @@ -147,36 +139,31 @@ pub async fn process_crawl_results( let url = Url::parse(&crawl_result.url)?; let url_host = url.host_str().unwrap_or(""); // Add document to index - if let Ok(mut index_writer) = state.index.lock_writer() { - let doc_id = Searcher::upsert_document( - &mut index_writer, - DocumentUpdate { - doc_id: id_map.get(&crawl_result.url).cloned(), - title: &crawl_result.title.clone().unwrap_or_default(), - description: &crawl_result.description.clone().unwrap_or_default(), - domain: url_host, - url: url.as_str(), - content: &crawl_result.content.clone().unwrap_or_default(), - tags: &tags_for_crawl.clone(), - }, - )?; - - if !id_map.contains_key(&doc_id) { - added_docs.push(url.to_string()); - inserts.push(indexed_document::ActiveModel { - domain: Set(url_host.to_string()), - url: Set(url.to_string()), - open_url: Set(crawl_result.open_url.clone()), - doc_id: Set(doc_id), - updated_at: Set(Utc::now()), - ..Default::default() - }); - } else if let Some(model) = model_map.get(&doc_id) { - // Touch the existing model so we know it's been checked recently. - let mut update: indexed_document::ActiveModel = model.to_owned().into(); - update.updated_at = Set(Utc::now()); - updates.push(update); - } + let doc_id = state.index.upsert_document(DocumentUpdate { + doc_id: id_map.get(&crawl_result.url).cloned(), + title: &crawl_result.title.clone().unwrap_or_default(), + description: &crawl_result.description.clone().unwrap_or_default(), + domain: url_host, + url: url.as_str(), + content: &crawl_result.content.clone().unwrap_or_default(), + tags: &tags_for_crawl.clone(), + })?; + + if !id_map.contains_key(&doc_id) { + added_docs.push(url.to_string()); + inserts.push(indexed_document::ActiveModel { + domain: Set(url_host.to_string()), + url: Set(url.to_string()), + open_url: Set(crawl_result.open_url.clone()), + doc_id: Set(doc_id), + updated_at: Set(Utc::now()), + ..Default::default() + }); + } else if let Some(model) = model_map.get(&doc_id) { + // Touch the existing model so we know it's been checked recently. + let mut update: indexed_document::ActiveModel = model.to_owned().into(); + update.updated_at = Set(Utc::now()); + updates.push(update); } } @@ -187,7 +174,7 @@ pub async fn process_crawl_results( } tx.commit().await?; - let _ = Searcher::save(state).await; + let _ = state.index.save().await; // Find the recently added docs & apply the tags to them. let added_entries: Vec = indexed_document::Entity::find() @@ -254,8 +241,7 @@ pub async fn process_records( .map(|x| x.to_owned()) .collect::>(); - let _ = Searcher::delete_many_by_id(state, &doc_id_list, false).await; - let _ = Searcher::save(state).await; + let _ = state.index.delete_many_by_id(&doc_id_list).await; // Grab tags from the lens. let tags = lens @@ -288,24 +274,17 @@ pub async fn process_records( let url_host = url.host_str().unwrap_or(""); // Add document to index let doc_id: Option = { - if let Ok(mut index_writer) = state.index.lock_writer() { - match Searcher::upsert_document( - &mut index_writer, - DocumentUpdate { - doc_id: id_map.get(&canonical_url_str.clone()).cloned(), - title: &crawl_result.title.clone().unwrap_or_default(), - description: &crawl_result.description.clone(), - domain: url_host, - url: url.as_str(), - content: &crawl_result.content, - tags: &tag_list, - }, - ) { - Ok(new_doc_id) => Some(new_doc_id), - _ => None, - } - } else { - None + match state.index.upsert_document(DocumentUpdate { + doc_id: id_map.get(&canonical_url_str.clone()).cloned(), + title: &crawl_result.title.clone().unwrap_or_default(), + description: &crawl_result.description.clone(), + domain: url_host, + url: url.as_str(), + content: &crawl_result.content, + tags: &tag_list, + }) { + Ok(new_doc_id) => Some(new_doc_id), + _ => None, } }; @@ -385,7 +364,7 @@ pub async fn update_tags( .iter() .filter_map(|addr| { if let Ok(doc) = state.index.reader.searcher().doc(*addr) { - if let Ok(retrieved_doc) = search::document_to_struct(&doc) { + if let Some(retrieved_doc) = document_to_struct(&doc) { return Some(retrieved_doc); } } @@ -455,25 +434,19 @@ pub async fn update_tags( } } - let _ = Searcher::delete_many_by_id(state, document_ids, false).await; - let _ = Searcher::save(state).await; + let _ = state.index.delete_many_by_id(document_ids).await; log::debug!("Tag map generated {}", tag_map.len()); - if let Ok(mut index_writer) = state.index.lock_writer() { - for (_, (doc, ids)) in tag_map.iter() { - let _doc_id = Searcher::upsert_document( - &mut index_writer, - DocumentUpdate { - doc_id: Some(doc.doc_id.clone()), - title: &doc.title, - description: &doc.description, - domain: &doc.domain, - url: &doc.url, - content: &doc.content, - tags: ids, - }, - )?; - } + for (_, (doc, ids)) in tag_map.iter() { + let _doc_id = state.index.upsert_document(DocumentUpdate { + doc_id: Some(doc.doc_id.clone()), + title: &doc.title, + description: &doc.description, + domain: &doc.domain, + url: &doc.url, + content: &doc.content, + tags: ids, + })?; } } diff --git a/crates/spyglass/src/lib.rs b/crates/spyglass/src/lib.rs index e91d0c69e..25e8fb5c3 100644 --- a/crates/spyglass/src/lib.rs +++ b/crates/spyglass/src/lib.rs @@ -6,6 +6,5 @@ pub mod parser; pub mod pipeline; pub mod platform; pub mod plugin; -pub mod search; pub mod state; pub mod task; diff --git a/crates/spyglass/src/pipeline/default_pipeline.rs b/crates/spyglass/src/pipeline/default_pipeline.rs index f73ce76d1..9986df5b6 100644 --- a/crates/spyglass/src/pipeline/default_pipeline.rs +++ b/crates/spyglass/src/pipeline/default_pipeline.rs @@ -1,18 +1,18 @@ use crate::pipeline::collector::DefaultCollector; use crate::pipeline::PipelineContext; -use crate::search::{DocumentUpdate, Searcher}; use crate::state::AppState; use crate::task::CrawlTask; use entities::models::{crawl_queue, indexed_document}; +use entities::sea_orm::prelude::*; +use entities::sea_orm::{ColumnTrait, EntityTrait, QueryFilter, Set}; use shared::config::{Config, LensConfig, PipelineConfiguration}; +use spyglass_searcher::DocumentUpdate; use tokio::sync::mpsc; +use url::Url; use super::parser::DefaultParser; use super::PipelineCommand; -use entities::sea_orm::prelude::*; -use entities::sea_orm::{ColumnTrait, EntityTrait, QueryFilter, Set}; -use url::Url; // General pipeline loop for configured pipelines. This code is responsible for // processing pipeline requests against the provided pipeline configuration @@ -126,30 +126,23 @@ async fn start_crawl( // Delete old document, if any. if let Some(doc) = &existing { - let _ = Searcher::delete_by_id(&state, &doc.doc_id).await; - let _ = Searcher::save(&state).await; + let _ = state.index.delete_by_id(&doc.doc_id).await; + let _ = indexed_document::delete_many_by_id(&state.db, &[doc.id]).await; } // Add document to index let doc_id: Option = { - if let Ok(mut index_writer) = state.index.lock_writer() { - match Searcher::upsert_document( - &mut index_writer, - DocumentUpdate { - doc_id: existing.clone().map(|f| f.doc_id), - title: &crawl_result.title.unwrap_or_default(), - description: &crawl_result.description.unwrap_or_default(), - domain: url_host, - url: url.as_str(), - content: &content, - tags: &[], - }, - ) { - Ok(new_doc_id) => Some(new_doc_id), - _ => None, - } - } else { - None + match state.index.upsert_document(DocumentUpdate { + doc_id: existing.clone().map(|f| f.doc_id), + title: &crawl_result.title.unwrap_or_default(), + description: &crawl_result.description.unwrap_or_default(), + domain: url_host, + url: url.as_str(), + content: &content, + tags: &[], + }) { + Ok(new_doc_id) => Some(new_doc_id), + _ => None, } }; diff --git a/crates/spyglass/src/pipeline/mod.rs b/crates/spyglass/src/pipeline/mod.rs index b5603f16e..d61ede9a6 100644 --- a/crates/spyglass/src/pipeline/mod.rs +++ b/crates/spyglass/src/pipeline/mod.rs @@ -3,9 +3,8 @@ pub mod collector; pub mod default_pipeline; pub mod parser; -use crate::search::lens; use crate::state::AppState; -use crate::task::CrawlTask; +use crate::task::{lens::read_lenses, CrawlTask}; use entities::models::crawl_queue; use shared::config::Config; use shared::config::PipelineConfiguration; @@ -55,7 +54,7 @@ pub async fn initialize_pipelines( let mut shutdown_rx = app_state.shutdown_cmd_tx.lock().await.subscribe(); // Yes probably should do some error handling, but not really needed. No pipelines // just means not tasks to send. - let lens_map = lens::read_lenses(&config).await.unwrap_or_default(); + let lens_map = read_lenses(&config).await.unwrap_or_default(); let _ = read_pipelines(&app_state, &config).await; // Grab all pipelines diff --git a/crates/spyglass/src/plugin/exports.rs b/crates/spyglass/src/plugin/exports.rs index ffc3e9ff7..e7bec942a 100644 --- a/crates/spyglass/src/plugin/exports.rs +++ b/crates/spyglass/src/plugin/exports.rs @@ -28,12 +28,12 @@ use entities::sea_orm::ModelTrait; use entities::sea_orm::QueryFilter; use super::{wasi_read, wasi_read_string, PluginCommand, PluginConfig, PluginEnv, PluginId}; -use crate::search::{self, Searcher}; use crate::state::AppState; use reqwest::header::USER_AGENT; use entities::models::crawl_queue::{enqueue_all, EnqueueSettings}; use spyglass_plugin::{DocumentQuery, PluginCommandRequest}; +use spyglass_searcher::document_to_struct; pub fn register_exports( plugin_id: PluginId, @@ -71,7 +71,14 @@ async fn handle_plugin_cmd_request( match cmd { // Delete document from index PluginCommandRequest::DeleteDoc { url } => { - Searcher::delete_by_url(&env.app_state, url).await? + let docs = indexed_document::Entity::find() + .filter(indexed_document::Column::Url.eq(url)) + .all(&env.app_state.db) + .await + .unwrap_or_default(); + + let doc_ids = docs.iter().map(|x| x.doc_id.to_owned()).collect::>(); + env.app_state.index.delete_many_by_id(&doc_ids).await?; } // Enqueue a list of URLs to be crawled PluginCommandRequest::Enqueue { urls } => handle_plugin_enqueue(env, urls), @@ -149,8 +156,32 @@ async fn handle_plugin_cmd_request( tag_modifications, } => { log::trace!("Received modify tags command {:?}", documents); - let docs = - Searcher::search_by_query(&env.app_state.db, &env.app_state.index, documents).await; + let tag_ids = documents.has_tags.clone().unwrap_or_default(); + let tag_ids = tag::get_tags_by_value(&env.app_state.db, &tag_ids) + .await + .unwrap_or_default() + .iter() + .map(|model| model.id as u64) + .collect::>(); + + let exclude_tags = documents.exclude_tags.clone().unwrap_or_default(); + let exclude_tags = tag::get_tags_by_value(&env.app_state.db, &exclude_tags) + .await + .unwrap_or_default() + .iter() + .map(|model| model.id as u64) + .collect::>(); + + let docs = env + .app_state + .index + .search_by_query( + documents.urls.clone(), + documents.ids.clone(), + &tag_ids, + &exclude_tags, + ) + .await; if !docs.is_empty() { let doc_ids = docs .iter() @@ -280,16 +311,38 @@ async fn query_document_and_send_loop(env: PluginEnv, query: DocumentQuery) { } async fn query_documents_and_send(env: &PluginEnv, query: &DocumentQuery, send_empty: bool) { - let docs = Searcher::search_by_query(&env.app_state.db, &env.app_state.index, query).await; + let tag_ids = query.has_tags.clone().unwrap_or_default(); + let tag_ids = tag::get_tags_by_value(&env.app_state.db, &tag_ids) + .await + .unwrap_or_default() + .iter() + .map(|model| model.id as u64) + .collect::>(); + + let exclude_tags = query.exclude_tags.clone().unwrap_or_default(); + let exclude_tags = tag::get_tags_by_value(&env.app_state.db, &exclude_tags) + .await + .unwrap_or_default() + .iter() + .map(|model| model.id as u64) + .collect::>(); + + let docs = env + .app_state + .index + .search_by_query( + query.urls.clone(), + query.ids.clone(), + &tag_ids, + &exclude_tags, + ) + .await; log::debug!("Found {:?} documents for query", docs.len()); let searcher = &env.app_state.index.reader.searcher(); let mut results = Vec::new(); let db = &env.app_state.db; for (_score, doc_addr) in docs { - if let Ok(Ok(doc)) = searcher - .doc(doc_addr) - .map(|doc| search::document_to_struct(&doc)) - { + if let Ok(Some(doc)) = searcher.doc(doc_addr).map(|doc| document_to_struct(&doc)) { log::trace!("Got id with url {} {}", doc.doc_id, doc.url); let indexed = indexed_document::Entity::find() .filter(indexed_document::Column::DocId.eq(doc.doc_id.clone())) diff --git a/crates/spyglass/src/plugin/mod.rs b/crates/spyglass/src/plugin/mod.rs index b9ff349aa..a570c4089 100644 --- a/crates/spyglass/src/plugin/mod.rs +++ b/crates/spyglass/src/plugin/mod.rs @@ -489,3 +489,103 @@ fn wasi_read(env: &WasiEnv) -> anyhow::Result { fn wasi_write(env: &WasiEnv, obj: &(impl Serialize + ?Sized)) -> anyhow::Result<()> { wasi_write_string(env, &ron::to_string(&obj)?) } + +/// Utility function to map a trigger to the matching lens(es) & convert that into +/// search filters ready to be applied to a search. +pub async fn lens_to_filters(state: AppState, trigger: &str) -> Vec { + // Find the lenses that were triggered + // NOTE: Users can combine lenses together but giving them the same trigger label + let results = lens::Entity::find() + .filter(lens::Column::Trigger.eq(trigger)) + .all(&state.db) + .await + .ok(); + + // Based on the lens type, either use filters defined by the configuration + // or ask the plugin for the search filter. + let mut filters = Vec::new(); + for lens in results.unwrap_or_default() { + match lens.lens_type { + // Load lens configuration from files + lens::LensType::Simple => { + if let Some(lens_config) = state.lenses.get(&lens.name) { + let lens_filters = lens_config.into_regexes(); + filters.extend( + lens_filters + .allowed + .into_iter() + .map(SearchFilter::URLRegexAllow) + .collect::>(), + ); + + filters.extend( + lens_filters + .skipped + .into_iter() + .map(SearchFilter::URLRegexSkip) + .collect::>(), + ); + } + } + // Ask plugin for any filter information + lens::LensType::Plugin => { + let manager = state.plugin_manager.lock().await; + if let Some(plugin) = manager.find_by_name(lens.name) { + filters.extend(plugin.search_filters().await); + } + } + } + } + + if filters.is_empty() { + // lens remove? Plugin disabled? + log::warn!("No filters found for trigger: {}", trigger); + } + + filters +} + +#[cfg(test)] +mod test { + use entities::models::lens; + use entities::sea_orm::EntityTrait; + use entities::test::setup_test_db; + use shared::config::{LensConfig, UserSettings}; + use spyglass_plugin::SearchFilter; + use spyglass_searcher::IndexPath; + + use super::{lens_to_filters, AppState}; + + #[tokio::test] + async fn test_lens_to_filter() { + let db = setup_test_db().await; + let test_lens = LensConfig { + name: "test_lens".to_owned(), + trigger: "test".to_owned(), + urls: vec!["https://oldschool.runescape.wiki/wiki/".to_string()], + ..Default::default() + }; + + if let Err(e) = lens::add_or_enable(&db, &test_lens, lens::LensType::Simple).await { + eprintln!("{e}"); + } + + // Make sure the lens was added + let db_rows = lens::Entity::find().all(&db).await; + assert_eq!(db_rows.unwrap().len(), 1); + + let state = AppState::builder() + .with_db(db) + .with_lenses(&vec![test_lens]) + .with_user_settings(&UserSettings::default()) + .with_index(&IndexPath::Memory, false) + .build(); + + let filters = lens_to_filters(state, "test").await; + assert_eq!(filters.len(), 1); + assert_eq!( + *filters.get(0).unwrap(), + SearchFilter::URLRegexAllow("^https://oldschool.runescape.wiki/wiki/.*".to_owned()) + ); + } +} diff --git a/crates/spyglass/src/search/grouping.rs b/crates/spyglass/src/search/grouping.rs deleted file mode 100644 index 3d8e73551..000000000 --- a/crates/spyglass/src/search/grouping.rs +++ /dev/null @@ -1,15 +0,0 @@ -use std::collections::HashMap; - -pub fn group_urls_by_scheme(urls: Vec<&str>) -> HashMap<&str, Vec<&str>> { - let mut grouping: HashMap<&str, Vec<&str>> = HashMap::new(); - urls.iter().for_each(|url| { - let part = url.split(':').next(); - if let Some(scheme) = part { - grouping - .entry(scheme) - .and_modify(|list| list.push(url)) - .or_insert_with(|| Vec::from([url.to_owned()])); - } - }); - grouping -} diff --git a/crates/spyglass/src/state.rs b/crates/spyglass/src/state.rs index 75f7d6b2f..619e76ac0 100644 --- a/crates/spyglass/src/state.rs +++ b/crates/spyglass/src/state.rs @@ -13,11 +13,11 @@ use crate::task::{AppShutdown, UserSettingsChange}; use crate::{ pipeline::PipelineCommand, plugin::{PluginCommand, PluginManager}, - search::{IndexPath, Searcher}, task::{AppPause, ManagerCommand}, }; use shared::config::{Config, LensConfig, PipelineConfiguration, UserSettings}; use shared::metrics::Metrics; +use spyglass_searcher::{IndexPath, Searcher}; /// Used to track inflight requests and limit things #[derive(Clone, Debug, Hash, PartialEq, Eq)] diff --git a/crates/spyglass/src/task.rs b/crates/spyglass/src/task.rs index 791cced86..d741587e1 100644 --- a/crates/spyglass/src/task.rs +++ b/crates/spyglass/src/task.rs @@ -20,14 +20,14 @@ use crate::connection::{api_id_to_label, load_connection}; use crate::crawler::bootstrap; use crate::filesystem; use crate::filesystem::extensions::AudioExt; -use crate::search::lens::{load_lenses, read_lenses}; -use crate::search::Searcher; use crate::state::AppState; use crate::task::worker::FetchResult; use diff::Diff; +pub mod lens; mod manager; pub mod worker; +use lens::{load_lenses, read_lenses}; #[derive(Debug, Clone)] pub struct CrawlTask { @@ -421,7 +421,7 @@ pub async fn worker_task( log::debug!("committing {} new/updated docs in index", num_updated); updated_docs.store(0, Ordering::Relaxed); tokio::spawn(async move { - let _ = Searcher::save(&state).await; + let _ = state.index.save().await; }); } } diff --git a/crates/spyglass/src/search/lens.rs b/crates/spyglass/src/task/lens.rs similarity index 63% rename from crates/spyglass/src/search/lens.rs rename to crates/spyglass/src/task/lens.rs index 3620de5d8..b052be4e2 100644 --- a/crates/spyglass/src/search/lens.rs +++ b/crates/spyglass/src/task/lens.rs @@ -1,41 +1,17 @@ use dashmap::DashMap; use entities::models::lens; -use entities::sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; use shared::response::InstallableLens; use std::fs; use std::path::PathBuf; -use shared::config::{Config, LensConfig, LensSource}; -use spyglass_plugin::SearchFilter; - -use crate::state::AppState; -use crate::task::{CollectTask, ManagerCommand}; +use crate::{ + state::AppState, + task::{CollectTask, ManagerCommand}, +}; use reqwest::Client; +use shared::config::{Config, LensConfig, LensSource}; use shared::constants; -/// Reads lens directly from disk and provides the map lenses -pub async fn read_lenses(config: &Config) -> anyhow::Result> { - let lens_map = DashMap::new(); - let lense_dir = config.lenses_dir(); - - // Keep track of failures and report to user? - for entry in (fs::read_dir(lense_dir)?).flatten() { - let path = entry.path(); - if path.is_file() && path.extension().unwrap_or_default() == "ron" { - match LensConfig::from_path(path) { - Err(err) => log::warn!("Unable to load lens {:?}: {}", entry.path(), err), - Ok(lens) => { - if lens.is_enabled { - lens_map.insert(lens.name.clone(), lens); - } - } - } - } - } - - Ok(lens_map) -} - /// Loop through lenses in the AppState. Update our internal db & bootstrap anything /// that hasn't been bootstrapped. pub async fn load_lenses(lens_map: &DashMap, state: AppState) { @@ -81,61 +57,6 @@ pub async fn load_lenses(lens_map: &DashMap, state: AppState log::info!("✅ finished lens checks") } -/// Utility function to map a trigger to the matching lens(es) & convert that into -/// search filters ready to be applied to a search. -pub async fn lens_to_filters(state: AppState, trigger: &str) -> Vec { - // Find the lenses that were triggered - // NOTE: Users can combine lenses together but giving them the same trigger label - let results = lens::Entity::find() - .filter(lens::Column::Trigger.eq(trigger)) - .all(&state.db) - .await - .ok(); - - // Based on the lens type, either use filters defined by the configuration - // or ask the plugin for the search filter. - let mut filters = Vec::new(); - for lens in results.unwrap_or_default() { - match lens.lens_type { - // Load lens configuration from files - lens::LensType::Simple => { - if let Some(lens_config) = state.lenses.get(&lens.name) { - let lens_filters = lens_config.into_regexes(); - filters.extend( - lens_filters - .allowed - .into_iter() - .map(SearchFilter::URLRegexAllow) - .collect::>(), - ); - - filters.extend( - lens_filters - .skipped - .into_iter() - .map(SearchFilter::URLRegexSkip) - .collect::>(), - ); - } - } - // Ask plugin for any filter information - lens::LensType::Plugin => { - let manager = state.plugin_manager.lock().await; - if let Some(plugin) = manager.find_by_name(lens.name) { - filters.extend(plugin.search_filters().await); - } - } - } - } - - if filters.is_empty() { - // lens remove? Plugin disabled? - log::warn!("No filters found for trigger: {}", trigger); - } - - filters -} - /// Installs a new lens or updates the current lens. The requested lens will be /// downloaded from the lens store and added to the database. The actually lens /// loading will happen through the normal file system watch mechanism. @@ -241,47 +162,25 @@ async fn install_lens_to_path( Ok(()) } -#[cfg(test)] -mod test { - use crate::search::IndexPath; - use entities::models::lens; - use entities::sea_orm::EntityTrait; - use entities::test::setup_test_db; - use shared::config::{LensConfig, UserSettings}; - use spyglass_plugin::SearchFilter; - - use super::{lens_to_filters, AppState}; - - #[tokio::test] - async fn test_lens_to_filter() { - let db = setup_test_db().await; - let test_lens = LensConfig { - name: "test_lens".to_owned(), - trigger: "test".to_owned(), - urls: vec!["https://oldschool.runescape.wiki/wiki/".to_string()], - ..Default::default() - }; +/// Reads lens directly from disk and provides the map lenses +pub async fn read_lenses(config: &Config) -> anyhow::Result> { + let lens_map = DashMap::new(); + let lense_dir = config.lenses_dir(); - if let Err(e) = lens::add_or_enable(&db, &test_lens, lens::LensType::Simple).await { - eprintln!("{e}"); + // Keep track of failures and report to user? + for entry in (fs::read_dir(lense_dir)?).flatten() { + let path = entry.path(); + if path.is_file() && path.extension().unwrap_or_default() == "ron" { + match LensConfig::from_path(path) { + Err(err) => log::warn!("Unable to load lens {:?}: {}", entry.path(), err), + Ok(lens) => { + if lens.is_enabled { + lens_map.insert(lens.name.clone(), lens); + } + } + } } - - // Make sure the lens was added - let db_rows = lens::Entity::find().all(&db).await; - assert_eq!(db_rows.unwrap().len(), 1); - - let state = AppState::builder() - .with_db(db) - .with_lenses(&vec![test_lens]) - .with_user_settings(&UserSettings::default()) - .with_index(&IndexPath::Memory, false) - .build(); - - let filters = lens_to_filters(state, "test").await; - assert_eq!(filters.len(), 1); - assert_eq!( - *filters.get(0).unwrap(), - SearchFilter::URLRegexAllow("^https://oldschool.runescape.wiki/wiki/.*".to_owned()) - ); } + + Ok(lens_map) } diff --git a/crates/spyglass/src/task/worker.rs b/crates/spyglass/src/task/worker.rs index cf80c311e..d8bd9c23b 100644 --- a/crates/spyglass/src/task/worker.rs +++ b/crates/spyglass/src/task/worker.rs @@ -10,7 +10,7 @@ use shared::config::{Config, LensConfig, LensSource}; use super::{bootstrap, CollectTask, ManagerCommand}; use super::{CleanupTask, CrawlTask}; -use crate::search::Searcher; + use crate::state::AppState; use crate::{ crawler::{CrawlError, CrawlResult, Crawler}, @@ -67,8 +67,7 @@ pub async fn cleanup_database(state: &AppState, cleanup_task: CleanupTask) -> an // Found document for the url, but it has a different doc id. // check if this document exists in the index to see if we // had a duplicate - let indexed_result = - Searcher::get_by_id(&state.index.reader, doc_model.doc_id.as_str()); + let indexed_result = state.index.get_by_id(doc_model.doc_id.as_str()); match indexed_result { Some(_doc) => { log::debug!( @@ -77,7 +76,10 @@ pub async fn cleanup_database(state: &AppState, cleanup_task: CleanupTask) -> an doc_id ); // Found indexed document, so we must have had duplicates, remove dup - let _ = Searcher::delete_by_id(state, doc_id.as_str()).await; + let _ = state.index.delete_by_id(doc_id.as_str()).await; + let _ = + indexed_document::delete_many_by_doc_id(&state.db, &[doc_id]) + .await; changed = true; } None => { @@ -96,7 +98,8 @@ pub async fn cleanup_database(state: &AppState, cleanup_task: CleanupTask) -> an Ok(None) => { log::debug!("Could not find document for url {}, removing", url); // can't find the url at all must be an old doc that was removed - let _ = Searcher::delete_by_id(state, doc_id.as_str()).await; + let _ = state.index.delete_by_id(doc_id.as_str()).await; + let _ = indexed_document::delete_many_by_doc_id(&state.db, &[doc_id]).await; changed = true; } Err(error) => { @@ -108,7 +111,7 @@ pub async fn cleanup_database(state: &AppState, cleanup_task: CleanupTask) -> an } if changed { - let _ = Searcher::save(state).await; + let _ = state.index.save().await; } Ok(()) @@ -299,9 +302,8 @@ pub async fn handle_deletion(state: AppState, task_id: i64) -> anyhow::Result<() .collect::>(); // Remove doc references from DB & from index - for doc_id in doc_ids { - let _ = Searcher::delete_by_id(&state, &doc_id).await; - } + let _ = state.index.delete_many_by_id(&doc_ids).await; + let _ = indexed_document::delete_many_by_doc_id(&state.db, &doc_ids).await; // Finally delete this crawl task as well. task.delete(&state.db).await?; @@ -313,13 +315,13 @@ pub async fn handle_deletion(state: AppState, task_id: i64) -> anyhow::Result<() #[cfg(test)] mod test { use crate::crawler::CrawlResult; - use crate::search::IndexPath; use entities::models::crawl_queue::{self, CrawlStatus, CrawlType}; use entities::models::tag::{self, TagType}; use entities::models::{bootstrap_queue, indexed_document}; use entities::sea_orm::{ActiveModelTrait, EntityTrait, ModelTrait, Set}; use entities::test::setup_test_db; use shared::config::{LensConfig, UserSettings}; + use spyglass_searcher::IndexPath; use super::{handle_cdx_collection, process_crawl, AppState, FetchResult}; From ed0c4c3fb5e039b42b097f30dab22e796eaa89ce Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Wed, 26 Apr 2023 15:43:41 -0700 Subject: [PATCH 12/30] updating searcher to directly return Document struct instead of just the internal address (#440) --- crates/spyglass-searcher/src/lib.rs | 77 ++++++++++-------- crates/spyglass/bin/debug/src/main.rs | 23 ++---- crates/spyglass/src/api/handler/search.rs | 96 +++++++++++------------ crates/spyglass/src/documents/mod.rs | 19 +---- crates/spyglass/src/plugin/exports.rs | 71 +++++++++-------- 5 files changed, 136 insertions(+), 150 deletions(-) diff --git a/crates/spyglass-searcher/src/lib.rs b/crates/spyglass-searcher/src/lib.rs index d8dfc49c9..fe10b4679 100644 --- a/crates/spyglass-searcher/src/lib.rs +++ b/crates/spyglass-searcher/src/lib.rs @@ -7,7 +7,7 @@ use std::time::Instant; use tantivy::collector::TopDocs; use tantivy::query::{BooleanQuery, Occur, Query, TermQuery}; -use tantivy::{schema::*, DocAddress}; +use tantivy::schema::*; use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy}; use thiserror::Error; use uuid::Uuid; @@ -23,7 +23,6 @@ pub use query::QueryStats; use query::{build_document_query, build_query, QueryBoosts}; type Score = f32; -type SearchResult = (Score, DocAddress); pub enum IndexPath { // Directory @@ -215,7 +214,7 @@ impl Searcher { ids: Option>, has_tags: &[u64], exclude_tags: &[u64], - ) -> Vec { + ) -> Vec<(Score, RetrievedDocument)> { let urls = urls.unwrap_or_default(); let ids = ids.unwrap_or_default(); @@ -231,7 +230,16 @@ impl Searcher { .search(&query, &collector) .expect("Unable to execute query"); - docs.into_iter().map(|addr| (1.0, addr)).collect() + docs.into_iter() + .map(|addr| (1.0, addr)) + .flat_map(|(score, addr)| { + if let Ok(Some(doc)) = index_search.doc(addr).map(|x| document_to_struct(&x)) { + Some((score, doc)) + } else { + None + } + }) + .collect() } pub async fn search_with_lens( @@ -242,7 +250,7 @@ impl Searcher { boosts: &[QueryBoost], stats: &mut QueryStats, num_results: usize, - ) -> Vec { + ) -> Vec<(Score, RetrievedDocument)> { let start_timer = Instant::now(); let index = &self.index; let reader = &self.reader; @@ -294,16 +302,24 @@ impl Searcher { Instant::now().duration_since(start_timer).as_millis() ); + let doc_reader = self.reader.searcher(); top_docs .into_iter() // Filter out negative scores .filter(|(score, _)| *score > 0.0) + .flat_map(|(score, addr)| { + if let Ok(Some(doc)) = doc_reader.doc(addr).map(|x| document_to_struct(&x)) { + Some((score, doc)) + } else { + None + } + }) .collect() } pub async fn explain_search_with_lens( &self, - doc_address: DocAddress, + doc: RetrievedDocument, applied_lenses: &Vec, query_string: &str, favorite_id: Option, @@ -347,21 +363,14 @@ impl Searcher { ); let mut combined: Vec<(Occur, Box)> = vec![(Occur::Should, Box::new(query))]; - if let Ok(Some(doc)) = self - .reader - .searcher() - .doc(doc_address) - .map(|doc| document_to_struct(&doc)) - { - combined.push(( - Occur::Must, - Box::new(TermQuery::new( - Term::from_field_text(fields.id, &doc.doc_id), - // Needs WithFreqs otherwise scoring is wonky. - IndexRecordOption::WithFreqs, - )), - )); - } + combined.push(( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_text(fields.id, &doc.doc_id), + // Needs WithFreqs otherwise scoring is wonky. + IndexRecordOption::WithFreqs, + )), + )); let content_terms = query::terms_for_field(&index.schema(), &tokenizers, query_string, fields.content); @@ -374,24 +383,26 @@ impl Searcher { .search(&final_query, &collector) .expect("Unable to execute query"); for (score, addr) in docs { - if addr == doc_address { - for t in content_terms { - let info = tantivy_searcher - .segment_reader(addr.segment_ord) - .inverted_index(fields.content) - .unwrap() - .get_term_info(&t.1); - log::info!("Term {:?} Info {:?} ", t, info); + if let Ok(Some(result)) = tantivy_searcher.doc(addr).map(|x| document_to_struct(&x)) { + if result.doc_id == doc.doc_id { + for t in content_terms { + let info = tantivy_searcher + .segment_reader(addr.segment_ord) + .inverted_index(fields.content) + .unwrap() + .get_term_info(&t.1); + log::info!("Term {:?} Info {:?} ", t, info); + } + + return Some(score); } - - return Some(score); } } None } } -#[derive(Serialize)] +#[derive(Clone, Serialize)] pub struct RetrievedDocument { pub doc_id: String, pub domain: String, @@ -537,7 +548,7 @@ mod test { let mut stats = QueryStats::new(); let query = "salinas"; - let results: Vec<(f32, tantivy::DocAddress)> = searcher + let results = searcher .search_with_lens(&vec![2_u64], query, None, &[], &mut stats, 5) .await; diff --git a/crates/spyglass/bin/debug/src/main.rs b/crates/spyglass/bin/debug/src/main.rs index b1e24a610..3e6ec2f67 100644 --- a/crates/spyglass/bin/debug/src/main.rs +++ b/crates/spyglass/bin/debug/src/main.rs @@ -10,7 +10,7 @@ use tracing_log::LogTracer; use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter}; use libspyglass::pipeline::cache_pipeline::process_update; -use spyglass_searcher::{document_to_struct, IndexPath, QueryBoost, QueryStats, Searcher}; +use spyglass_searcher::{IndexPath, QueryBoost, QueryStats, Searcher}; #[cfg(debug_assertions)] const LOG_LEVEL: &str = "spyglassdebug=DEBUG"; @@ -129,21 +129,12 @@ async fn main() -> anyhow::Result { if docs.is_empty() { println!("No indexed document for url {:?}", &doc.url); } else { - for (_score, doc_addr) in docs { - if let Ok(Some(doc)) = index - .reader - .searcher() - .doc(doc_addr) - .map(|doc| document_to_struct(&doc)) - { - println!( - "Indexed Document: {}", - ron::ser::to_string_pretty(&doc, PrettyConfig::new()) - .unwrap_or_default() - ); - } else { - println!("Error accessing Doc at address {:?}", doc_addr); - } + for (_score, doc) in docs { + println!( + "Indexed Document: {}", + ron::ser::to_string_pretty(&doc, PrettyConfig::new()) + .unwrap_or_default() + ); } } } diff --git a/crates/spyglass/src/api/handler/search.rs b/crates/spyglass/src/api/handler/search.rs index de519a9a9..f22965eda 100644 --- a/crates/spyglass/src/api/handler/search.rs +++ b/crates/spyglass/src/api/handler/search.rs @@ -10,7 +10,7 @@ use shared::metrics; use shared::request; use shared::response::{LensResult, SearchLensesResp, SearchMeta, SearchResult, SearchResults}; use spyglass_searcher::schema::{DocFields, SearchDocument}; -use spyglass_searcher::{document_to_struct, QueryBoost, QueryStats}; +use spyglass_searcher::{QueryBoost, QueryStats}; use std::collections::HashSet; use std::time::SystemTime; use tracing::instrument; @@ -58,54 +58,52 @@ pub async fn search_docs( let mut results: Vec = Vec::new(); let mut missing: Vec<(String, String)> = Vec::new(); - for (score, doc_addr) in docs { - if let Ok(Some(doc)) = searcher.doc(doc_addr).map(|doc| document_to_struct(&doc)) { - log::debug!("Got id with url {} {}", doc.doc_id, doc.url); - let indexed = indexed_document::Entity::find() - .filter(indexed_document::Column::DocId.eq(doc.doc_id.clone())) - .one(&state.db) - .await; - - let crawl_uri = doc.url; - match indexed { - Ok(Some(indexed)) => { - let tags = indexed - .find_related(tag::Entity) - .all(&state.db) - .await - .unwrap_or_default() - .iter() - .map(|tag| (tag.label.to_string(), tag.value.clone())) - .collect::>(); - - let fields = DocFields::as_fields(); - let tokenizer = index - .index - .tokenizer_for_field(fields.content) - .expect("Unable to get tokenizer for content field"); - - let description = spyglass_searcher::utils::generate_highlight_preview( - &tokenizer, - &query, - &doc.content, - ); - - let result = SearchResult { - doc_id: doc.doc_id.clone(), - domain: doc.domain, - title: doc.title, - crawl_uri: crawl_uri.clone(), - description, - url: indexed.open_url.unwrap_or(crawl_uri), - tags, - score, - }; - - results.push(result); - } - _ => { - missing.push((doc.doc_id.to_owned(), crawl_uri.to_owned())); - } + for (score, doc) in docs { + log::debug!("Got id with url {} {}", doc.doc_id, doc.url); + let indexed = indexed_document::Entity::find() + .filter(indexed_document::Column::DocId.eq(doc.doc_id.clone())) + .one(&state.db) + .await; + + let crawl_uri = doc.url; + match indexed { + Ok(Some(indexed)) => { + let tags = indexed + .find_related(tag::Entity) + .all(&state.db) + .await + .unwrap_or_default() + .iter() + .map(|tag| (tag.label.to_string(), tag.value.clone())) + .collect::>(); + + let fields = DocFields::as_fields(); + let tokenizer = index + .index + .tokenizer_for_field(fields.content) + .expect("Unable to get tokenizer for content field"); + + let description = spyglass_searcher::utils::generate_highlight_preview( + &tokenizer, + &query, + &doc.content, + ); + + let result = SearchResult { + doc_id: doc.doc_id.clone(), + domain: doc.domain, + title: doc.title, + crawl_uri: crawl_uri.clone(), + description, + url: indexed.open_url.unwrap_or(crawl_uri), + tags, + score, + }; + + results.push(result); + } + _ => { + missing.push((doc.doc_id.to_owned(), crawl_uri.to_owned())); } } } diff --git a/crates/spyglass/src/documents/mod.rs b/crates/spyglass/src/documents/mod.rs index d756a28a1..65e549290 100644 --- a/crates/spyglass/src/documents/mod.rs +++ b/crates/spyglass/src/documents/mod.rs @@ -11,7 +11,6 @@ use entities::{ use shared::config::LensConfig; use spyglass_plugin::TagModification; use std::{collections::HashMap, str::FromStr, time::Instant}; -use tantivy::DocAddress; use libnetrunner::parser::ParseResult; use url::Url; @@ -19,7 +18,7 @@ use url::Url; use crate::{crawler::CrawlResult, state::AppState}; use entities::models::tag::TagType; use entities::sea_orm::{ColumnTrait, EntityTrait, QueryFilter, Set, TransactionTrait}; -use spyglass_searcher::{document_to_struct, DocumentUpdate, RetrievedDocument}; +use spyglass_searcher::{DocumentUpdate, RetrievedDocument}; /// Helper method to delete indexed documents, crawl queue items and search /// documents by url @@ -346,7 +345,7 @@ pub async fn process_records( /// 5. Updates the indexed document with the new tags (index) pub async fn update_tags( state: &AppState, - doc_ids: &[DocAddress], + documents: &[RetrievedDocument], tag_modifications: &TagModification, ) -> anyhow::Result<()> { let mut tag_cache: HashMap = HashMap::new(); @@ -360,18 +359,6 @@ pub async fn update_tags( None => Vec::new(), }; - let documents = doc_ids - .iter() - .filter_map(|addr| { - if let Ok(doc) = state.index.reader.searcher().doc(*addr) { - if let Some(retrieved_doc) = document_to_struct(&doc) { - return Some(retrieved_doc); - } - } - None - }) - .collect::>(); - let document_ids = &documents .iter() .map(|doc| doc.doc_id.clone()) @@ -420,7 +407,7 @@ pub async fn update_tags( tag_map.insert( doc.doc_id.clone(), ( - doc, + doc.clone(), ids.iter().map(|tag_id| tag_id.id).collect::>(), ), ); diff --git a/crates/spyglass/src/plugin/exports.rs b/crates/spyglass/src/plugin/exports.rs index e7bec942a..fe00d96b3 100644 --- a/crates/spyglass/src/plugin/exports.rs +++ b/crates/spyglass/src/plugin/exports.rs @@ -14,9 +14,9 @@ use spyglass_plugin::Authentication; use spyglass_plugin::DocumentUpdate; use spyglass_plugin::HttpMethod; use spyglass_plugin::{DocumentResult, PluginEvent}; +use spyglass_searcher::RetrievedDocument; use std::path::Path; use std::str::FromStr; -use tantivy::DocAddress; use tokio::sync::mpsc::Sender; use url::Url; use wasmer::{Exports, Function, Store}; @@ -33,7 +33,6 @@ use reqwest::header::USER_AGENT; use entities::models::crawl_queue::{enqueue_all, EnqueueSettings}; use spyglass_plugin::{DocumentQuery, PluginCommandRequest}; -use spyglass_searcher::document_to_struct; pub fn register_exports( plugin_id: PluginId, @@ -182,13 +181,16 @@ async fn handle_plugin_cmd_request( &exclude_tags, ) .await; + if !docs.is_empty() { - let doc_ids = docs + let docs = docs .iter() - .map(|(_, addr)| *addr) - .collect::>(); + .map(|(_, doc)| doc) + .cloned() + .collect::>(); + if let Err(error) = - documents::update_tags(&env.app_state, &doc_ids, tag_modifications).await + documents::update_tags(&env.app_state, &docs, tag_modifications).await { log::error!("Error updating document tags {:?}", error); } @@ -338,39 +340,36 @@ async fn query_documents_and_send(env: &PluginEnv, query: &DocumentQuery, send_e ) .await; log::debug!("Found {:?} documents for query", docs.len()); - let searcher = &env.app_state.index.reader.searcher(); let mut results = Vec::new(); let db = &env.app_state.db; - for (_score, doc_addr) in docs { - if let Ok(Some(doc)) = searcher.doc(doc_addr).map(|doc| document_to_struct(&doc)) { - log::trace!("Got id with url {} {}", doc.doc_id, doc.url); - let indexed = indexed_document::Entity::find() - .filter(indexed_document::Column::DocId.eq(doc.doc_id.clone())) - .one(db) - .await; + for (_score, doc) in docs { + log::trace!("Got id with url {} {}", doc.doc_id, doc.url); + let indexed = indexed_document::Entity::find() + .filter(indexed_document::Column::DocId.eq(doc.doc_id.clone())) + .one(db) + .await; - let crawl_uri = doc.url; - if let Ok(Some(indexed)) = indexed { - let tags = indexed - .find_related(tag::Entity) - .all(db) - .await - .unwrap_or_default() - .iter() - .map(|tag| (tag.label.to_string(), tag.value.clone())) - .collect::>(); - - let result = DocumentResult { - doc_id: doc.doc_id.clone(), - domain: doc.domain, - title: doc.title, - description: doc.description, - url: indexed.open_url.unwrap_or(crawl_uri), - tags, - }; - - results.push(result); - } + let crawl_uri = doc.url; + if let Ok(Some(indexed)) = indexed { + let tags = indexed + .find_related(tag::Entity) + .all(db) + .await + .unwrap_or_default() + .iter() + .map(|tag| (tag.label.to_string(), tag.value.clone())) + .collect::>(); + + let result = DocumentResult { + doc_id: doc.doc_id.clone(), + domain: doc.domain, + title: doc.title, + description: doc.description, + url: indexed.open_url.unwrap_or(crawl_uri), + tags, + }; + + results.push(result); } } From edfb84da406b3ed0a10bd1725df1923e2c53babf Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Wed, 26 Apr 2023 20:35:14 -0700 Subject: [PATCH 13/30] update endpoints for web client --- apps/web/Makefile | 2 +- apps/web/src/constants.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/web/Makefile b/apps/web/Makefile index 8b2b9f59d..970c24dad 100644 --- a/apps/web/Makefile +++ b/apps/web/Makefile @@ -3,4 +3,4 @@ deploy: npm run build trunk build --release - aws s3 cp --recursive dist s3://app.spyglass.fyi + aws s3 cp --recursive dist s3://search.spyglass.fyi diff --git a/apps/web/src/constants.rs b/apps/web/src/constants.rs index a34bb1ee9..43df4c34d 100644 --- a/apps/web/src/constants.rs +++ b/apps/web/src/constants.rs @@ -1,5 +1,5 @@ // todo: pull these from environment variables? config? #[cfg(not(debug_assertions))] -pub const HTTP_ENDPOINT: &str = "https://search.spyglass.fyi"; +pub const HTTP_ENDPOINT: &str = "https://api.spyglass.fyi"; #[cfg(debug_assertions)] pub const HTTP_ENDPOINT: &str = "http://127.0.0.1:8757"; From 63cb36e8daacee6aeaf4d7d7c6397dd5cfd4920e Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Thu, 27 Apr 2023 17:26:32 -0700 Subject: [PATCH 14/30] tweak: make boosts more configurable & add read/write search traits (#441) * wip: adding traits to handle different search clients * renaming IndexPath -> IndexBackend to be more generic * creating a WriteTrait * add upsert/upsert_mant to WriteTrait * make boosts more configurable & simplify search function * remove unused struct * fix test --- Cargo.lock | 4 + crates/spyglass-searcher/Cargo.toml | 5 +- crates/spyglass-searcher/src/client/local.rs | 313 +++++++++++ crates/spyglass-searcher/src/client/mod.rs | 2 + crates/spyglass-searcher/src/lib.rs | 486 ++++-------------- crates/spyglass-searcher/src/query.rs | 193 +++---- crates/spyglass-searcher/src/schema.rs | 14 + crates/spyglass-searcher/src/similarity.rs | 16 +- crates/spyglass-searcher/src/utils.rs | 5 +- crates/spyglass/bin/debug/src/main.rs | 20 +- crates/spyglass/src/api/handler/mod.rs | 12 +- crates/spyglass/src/api/handler/search.rs | 35 +- crates/spyglass/src/api/mod.rs | 1 + crates/spyglass/src/connection/github.rs | 1 + crates/spyglass/src/documents/mod.rs | 69 ++- .../spyglass/src/pipeline/default_pipeline.rs | 29 +- crates/spyglass/src/plugin/exports.rs | 2 +- crates/spyglass/src/plugin/mod.rs | 4 +- crates/spyglass/src/state.rs | 10 +- crates/spyglass/src/task/worker.rs | 19 +- 20 files changed, 669 insertions(+), 571 deletions(-) create mode 100644 crates/spyglass-searcher/src/client/local.rs create mode 100644 crates/spyglass-searcher/src/client/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 77c559ae0..79a8ff80a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6774,6 +6774,8 @@ name = "spyglass-searcher" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", + "chrono", "log", "reqwest", "ron", @@ -6786,6 +6788,7 @@ dependencies = [ "tracing", "tracing-log", "tracing-subscriber", + "url", "uuid 1.3.0", ] @@ -8483,6 +8486,7 @@ dependencies = [ "getrandom 0.2.8", "rand 0.8.5", "serde", + "sha1_smol", "wasm-bindgen", ] diff --git a/crates/spyglass-searcher/Cargo.toml b/crates/spyglass-searcher/Cargo.toml index 16e09d525..94f00bcc2 100644 --- a/crates/spyglass-searcher/Cargo.toml +++ b/crates/spyglass-searcher/Cargo.toml @@ -7,6 +7,8 @@ edition = "2021" [dependencies] anyhow = "1.0" +async-trait = "0.1.68" +chrono = { version = "0.4.23", features = ["serde"] } log = "0.4" serde = "1.0" serde_json = "1.0" @@ -18,10 +20,11 @@ tracing = "0.1" tracing-log = "0.1.3" tracing-subscriber = { version = "0.3", features = ["env-filter", "std"]} tokio = { version = "1", features = ["full"] } +url = "2.3.1" # Internal spyglass libs shared = { path = "../shared" } -uuid = { version = "1.0.0", features = ["serde", "v4"], default-features = false } +uuid = { version = "1.0.0", features = ["serde", "v5"], default-features = false } [lib] path = "src/lib.rs" diff --git a/crates/spyglass-searcher/src/client/local.rs b/crates/spyglass-searcher/src/client/local.rs new file mode 100644 index 000000000..767d7ed8a --- /dev/null +++ b/crates/spyglass-searcher/src/client/local.rs @@ -0,0 +1,313 @@ +use std::fmt::{Debug, Error, Formatter}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Instant; + +use tantivy::collector::TopDocs; +use tantivy::query::TermQuery; +use tantivy::schema::*; +use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy}; +use uuid::Uuid; + +use crate::query::{build_document_query, build_query, terms_for_field, QueryOptions}; +use crate::schema::{self, DocFields, SearchDocument}; +use crate::{ + document_to_struct, Boost, DocumentUpdate, IndexBackend, QueryBoost, RetrievedDocument, Score, + SearchError, SearchQueryResult, SearchTrait, SearcherResult, WriteTrait, +}; + +const SPYGLASS_NS: Uuid = uuid::uuid!("5fdfe40a-de2c-11ed-bfa7-00155deae876"); + +/// Tantivy searcher client +#[derive(Clone)] +pub struct Searcher { + pub index: Index, + pub reader: IndexReader, + pub writer: Option>>, +} + +impl Debug for Searcher { + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { + f.debug_struct("Searcher") + .field("index", &self.index) + .finish() + } +} + +#[async_trait::async_trait] +impl WriteTrait for Searcher { + async fn delete_many_by_id(&self, doc_ids: &[String]) -> SearcherResult { + { + let writer = self.lock_writer()?; + let fields = DocFields::as_fields(); + for doc_id in doc_ids { + writer.delete_term(Term::from_field_text(fields.id, doc_id)); + } + } + + self.save().await?; + Ok(doc_ids.len()) + } + + async fn upsert_many(&self, updates: &[DocumentUpdate]) -> SearcherResult> { + let mut upserted = Vec::new(); + for doc_update in updates { + let fields = DocFields::as_fields(); + + let doc_id = doc_update.doc_id.clone().map_or_else( + || { + Uuid::new_v5(&SPYGLASS_NS, doc_update.url.as_bytes()) + .as_hyphenated() + .to_string() + }, + |s| s, + ); + + let mut doc = Document::default(); + doc.add_text(fields.content, doc_update.content); + doc.add_text(fields.domain, doc_update.domain); + doc.add_text(fields.id, &doc_id); + doc.add_text(fields.title, doc_update.title); + doc.add_text(fields.url, doc_update.url); + for t in doc_update.tags { + doc.add_u64(fields.tags, *t as u64); + } + + let writer = self.lock_writer()?; + writer.add_document(doc)?; + + upserted.push(doc_id.clone()); + } + + Ok(upserted) + } +} + +#[async_trait::async_trait] +impl SearchTrait for Searcher { + /// Get a single document by id + async fn get(&self, doc_id: &str) -> Option { + let fields = DocFields::as_fields(); + let searcher = self.reader.searcher(); + + let query = TermQuery::new( + Term::from_field_text(fields.id, doc_id), + IndexRecordOption::Basic, + ); + + let res = searcher + .search(&query, &TopDocs::with_limit(1)) + .map_or(Vec::new(), |x| x); + + if res.is_empty() { + return None; + } + + if let Some((_, doc_address)) = res.first() { + if let Ok(doc) = searcher.doc(*doc_address) { + return document_to_struct(&doc); + } + } + + None + } + + /// Runs a search against the index + async fn search( + &self, + query_string: &str, + filters: &[QueryBoost], + boosts: &[QueryBoost], + num_results: usize, + ) -> SearchQueryResult { + let start_timer = Instant::now(); + + let index = &self.index; + let reader = &self.reader; + let searcher = reader.searcher(); + + let (term_counts, query) = build_query( + index, + query_string, + filters, + boosts, + QueryOptions::default(), + ); + + let collector = TopDocs::with_limit(num_results); + + let top_docs = searcher + .search(&query, &collector) + .expect("Unable to execute query"); + + log::debug!( + "query `{}` returned {} results from {} docs in {} ms", + query_string, + top_docs.len(), + searcher.num_docs(), + Instant::now().duration_since(start_timer).as_millis() + ); + + let doc_reader = self.reader.searcher(); + let docs = top_docs + .into_iter() + // Filter out negative scores + .filter(|(score, _)| *score > 0.0) + .flat_map(|(score, addr)| { + if let Ok(Some(doc)) = doc_reader.doc(addr).map(|x| document_to_struct(&x)) { + Some((score, doc)) + } else { + None + } + }) + .collect(); + + SearchQueryResult { + wall_time_ms: Instant::now().duration_since(start_timer).as_millis(), + num_docs: searcher.num_docs(), + term_counts, + documents: docs, + } + } +} + +impl Searcher { + pub fn is_readonly(&self) -> bool { + self.writer.is_none() + } + + pub fn lock_writer(&self) -> SearcherResult> { + if let Some(index) = &self.writer { + match index.lock() { + Ok(lock) => Ok(lock), + Err(_) => Err(SearchError::WriterLocked), + } + } else { + Err(SearchError::ReadOnly) + } + } + + pub async fn save(&self) -> SearcherResult<()> { + let mut writer = self.lock_writer()?; + writer.commit()?; + Ok(()) + } + + /// Constructs a new Searcher object w/ the index @ `index_path` + pub fn with_index(index_path: &IndexBackend, readonly: bool) -> SearcherResult { + let index = match index_path { + IndexBackend::LocalPath(path) => schema::initialize_index(path)?, + IndexBackend::Memory => schema::initialize_in_memory_index(), + IndexBackend::Http(_) => unimplemented!(""), + }; + + // Should only be one writer at a time. This single IndexWriter is already + // multithreaded. + let writer = if readonly { + None + } else { + Some(Arc::new(Mutex::new( + index + .writer(50_000_000) + .expect("Unable to create index_writer"), + ))) + }; + + // For a search server you will typically create on reader for the entire + // lifetime of your program. + let reader = index + .reader_builder() + .reload_policy(ReloadPolicy::OnCommit) + .try_into() + .expect("Unable to create reader"); + + Ok(Searcher { + index, + reader, + writer, + }) + } + + /// Helper method to execute a search based on the provided document query + pub async fn search_by_query( + &self, + urls: Option>, + ids: Option>, + has_tags: &[u64], + exclude_tags: &[u64], + ) -> Vec<(Score, RetrievedDocument)> { + let urls = urls.unwrap_or_default(); + let ids = ids.unwrap_or_default(); + + let fields = DocFields::as_fields(); + let query = build_document_query(fields, &urls, &ids, has_tags, exclude_tags); + + let collector = tantivy::collector::DocSetCollector; + + let reader = &self.reader; + let index_search = reader.searcher(); + + let docs = index_search + .search(&query, &collector) + .expect("Unable to execute query"); + + docs.into_iter() + .map(|addr| (1.0, addr)) + .flat_map(|(score, addr)| { + if let Ok(Some(doc)) = index_search.doc(addr).map(|x| document_to_struct(&x)) { + Some((score, doc)) + } else { + None + } + }) + .collect() + } + + pub async fn explain_search_with_lens( + &self, + doc_id: String, + query_string: &str, + boosts: &[QueryBoost], + ) -> Option { + let index = &self.index; + let reader = &self.reader; + let fields = DocFields::as_fields(); + + let tantivy_searcher = reader.searcher(); + let filters = vec![QueryBoost::new(Boost::DocId(doc_id.clone()))]; + let (_, final_query) = build_query( + &self.index, + query_string, + &filters, + boosts, + QueryOptions::default(), + ); + + let tokenizers = index.tokenizers(); + let content_terms = + terms_for_field(&index.schema(), tokenizers, query_string, fields.content); + log::info!("Content Tokens {:?}", content_terms); + + let collector = tantivy::collector::TopDocs::with_limit(1); + let docs = tantivy_searcher + .search(&final_query, &collector) + .expect("Unable to execute query"); + + for (score, addr) in docs { + if let Ok(Some(result)) = tantivy_searcher.doc(addr).map(|x| document_to_struct(&x)) { + if result.doc_id == doc_id { + for t in content_terms { + let info = tantivy_searcher + .segment_reader(addr.segment_ord) + .inverted_index(fields.content) + .unwrap() + .get_term_info(&t.1); + log::info!("Term {:?} Info {:?} ", t, info); + } + + return Some(score); + } + } + } + None + } +} diff --git a/crates/spyglass-searcher/src/client/mod.rs b/crates/spyglass-searcher/src/client/mod.rs new file mode 100644 index 000000000..19dcded8a --- /dev/null +++ b/crates/spyglass-searcher/src/client/mod.rs @@ -0,0 +1,2 @@ +mod local; +pub use self::local::*; diff --git a/crates/spyglass-searcher/src/lib.rs b/crates/spyglass-searcher/src/lib.rs index fe10b4679..734b46e0b 100644 --- a/crates/spyglass-searcher/src/lib.rs +++ b/crates/spyglass-searcher/src/lib.rs @@ -1,30 +1,23 @@ use serde::Serialize; -use std::collections::HashSet; -use std::fmt::{Debug, Error, Formatter}; +use std::fmt::Debug; use std::path::PathBuf; -use std::sync::{Arc, Mutex, MutexGuard}; -use std::time::Instant; - -use tantivy::collector::TopDocs; -use tantivy::query::{BooleanQuery, Occur, Query, TermQuery}; use tantivy::schema::*; -use tantivy::{Index, IndexReader, IndexWriter, ReloadPolicy}; use thiserror::Error; -use uuid::Uuid; +use url::Url; +pub mod client; pub mod schema; -use schema::{DocFields, SearchDocument}; +use schema::{DocFields, DocumentUpdate, SearchDocument}; mod query; pub mod similarity; pub mod utils; -pub use query::QueryStats; -use query::{build_document_query, build_query, QueryBoosts}; - type Score = f32; -pub enum IndexPath { +pub enum IndexBackend { + // Elasticsearch compatible REST API (such as Quickwit for example) + Http(Url), // Directory LocalPath(PathBuf), // In memory index for testing purposes. @@ -32,23 +25,47 @@ pub enum IndexPath { } #[derive(Clone)] -pub struct DocumentUpdate<'a> { - pub doc_id: Option, - pub title: &'a str, - pub description: &'a str, - pub domain: &'a str, - pub url: &'a str, - pub content: &'a str, - pub tags: &'a [i64], +pub struct QueryBoost { + /// What to boost + field: Boost, + /// The boost value (negative to lessen impact of something) + value: f32, +} + +impl QueryBoost { + pub fn new(boost: Boost) -> Self { + let value = &match boost { + Boost::DocId(_) => 3.0, + Boost::Favorite { .. } => 3.0, + Boost::Tag(_) => 1.5, + Boost::Url(_) => 3.0, + }; + + QueryBoost { + field: boost, + value: *value, + } + } } #[derive(Clone)] -pub enum QueryBoost { +pub enum Boost { + // If required is set to true, _only_ favorites will be searched. + Favorite { id: u64, required: bool }, Url(String), DocId(String), Tag(u64), } +/// Contains stats & results for a search request +#[derive(Clone)] +pub struct SearchQueryResult { + pub wall_time_ms: u128, + pub num_docs: u64, + pub term_counts: usize, + pub documents: Vec<(Score, RetrievedDocument)>, +} + #[allow(clippy::enum_variant_names)] #[derive(Error, Debug)] pub enum SearchError { @@ -62,346 +79,42 @@ pub enum SearchError { Other(#[from] anyhow::Error), } -type SearcherResult = Result; -#[derive(Clone)] -pub struct Searcher { - pub index: Index, - pub reader: IndexReader, - pub writer: Option>>, -} - -impl Debug for Searcher { - fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error> { - f.debug_struct("Searcher") - .field("index", &self.index) - .finish() - } -} - -impl Searcher { - pub fn is_readonly(&self) -> bool { - self.writer.is_none() - } - - pub fn lock_writer(&self) -> SearcherResult> { - if let Some(index) = &self.writer { - match index.lock() { - Ok(lock) => Ok(lock), - Err(_) => Err(SearchError::WriterLocked), - } - } else { - Err(SearchError::ReadOnly) - } - } - - pub async fn save(&self) -> SearcherResult<()> { - let mut writer = self.lock_writer()?; - writer.commit()?; - Ok(()) - } - - /// Deletes a single entry from the database & index - pub async fn delete_by_id(&self, doc_id: &str) -> SearcherResult<()> { - self.delete_many_by_id(&[doc_id.into()]).await?; - Ok(()) - } - - /// Deletes multiple ids from the searcher at one time. The caller can decide if the - /// documents should also be removed from the database by setting the remove_documents - /// flag. - pub async fn delete_many_by_id(&self, doc_ids: &[String]) -> SearcherResult<()> { - { - let writer = self.lock_writer()?; - let fields = DocFields::as_fields(); - for doc_id in doc_ids { - writer.delete_term(Term::from_field_text(fields.id, doc_id)); - } - } - - self.save().await?; - Ok(()) - } - - /// Get document with `doc_id` from index. - pub fn get_by_id(&self, doc_id: &str) -> Option { - let fields = DocFields::as_fields(); - let searcher = self.reader.searcher(); - - let query = TermQuery::new( - Term::from_field_text(fields.id, doc_id), - IndexRecordOption::Basic, - ); - - let res = searcher - .search(&query, &TopDocs::with_limit(1)) - .map_or(Vec::new(), |x| x); - - if res.is_empty() { - return None; - } - - if let Some((_, doc_address)) = res.first() { - if let Ok(doc) = searcher.doc(*doc_address) { - return Some(doc); - } - } - - None - } - - /// Constructs a new Searcher object w/ the index @ `index_path` - pub fn with_index(index_path: &IndexPath, readonly: bool) -> SearcherResult { - let index = match index_path { - IndexPath::LocalPath(path) => schema::initialize_index(path)?, - IndexPath::Memory => schema::initialize_in_memory_index(), - }; - - // Should only be one writer at a time. This single IndexWriter is already - // multithreaded. - let writer = if readonly { - None - } else { - Some(Arc::new(Mutex::new( - index - .writer(50_000_000) - .expect("Unable to create index_writer"), - ))) - }; - - // For a search server you will typically create on reader for the entire - // lifetime of your program. - let reader = index - .reader_builder() - .reload_policy(ReloadPolicy::OnCommit) - .try_into() - .expect("Unable to create reader"); - - Ok(Searcher { - index, - reader, - writer, - }) - } - - pub fn upsert_document(&self, doc_update: DocumentUpdate) -> SearcherResult { - let fields = DocFields::as_fields(); - - let doc_id = doc_update - .doc_id - .map_or_else(|| Uuid::new_v4().as_hyphenated().to_string(), |s| s); - - let mut doc = Document::default(); - doc.add_text(fields.content, doc_update.content); - doc.add_text(fields.description, doc_update.description); - doc.add_text(fields.domain, doc_update.domain); - doc.add_text(fields.id, &doc_id); - doc.add_text(fields.title, doc_update.title); - doc.add_text(fields.url, doc_update.url); - for t in doc_update.tags { - doc.add_u64(fields.tags, *t as u64); - } - - let writer = self.lock_writer()?; - writer.add_document(doc)?; - - Ok(doc_id) - } - - /// Helper method to execute a search based on the provided document query - pub async fn search_by_query( - &self, - urls: Option>, - ids: Option>, - has_tags: &[u64], - exclude_tags: &[u64], - ) -> Vec<(Score, RetrievedDocument)> { - let urls = urls.unwrap_or_default(); - let ids = ids.unwrap_or_default(); - - let fields = DocFields::as_fields(); - let query = build_document_query(fields, &urls, &ids, has_tags, exclude_tags); - - let collector = tantivy::collector::DocSetCollector; - - let reader = &self.reader; - let index_search = reader.searcher(); - - let docs = index_search - .search(&query, &collector) - .expect("Unable to execute query"); - - docs.into_iter() - .map(|addr| (1.0, addr)) - .flat_map(|(score, addr)| { - if let Ok(Some(doc)) = index_search.doc(addr).map(|x| document_to_struct(&x)) { - Some((score, doc)) - } else { - None - } - }) - .collect() - } - - pub async fn search_with_lens( +/// Generic API for an index that can perform queries & get specific documents. +#[async_trait::async_trait] +pub trait SearchTrait { + /// Get a single document by id + async fn get(&self, doc_id: &str) -> Option; + /// Runs a search against the index + async fn search( &self, - applied_lenses: &Vec, - query_string: &str, - favorite_id: Option, + query: &str, + filters: &[QueryBoost], boosts: &[QueryBoost], - stats: &mut QueryStats, num_results: usize, - ) -> Vec<(Score, RetrievedDocument)> { - let start_timer = Instant::now(); - let index = &self.index; - let reader = &self.reader; - let fields = DocFields::as_fields(); - let searcher = reader.searcher(); - let tokenizers = index.tokenizers().clone(); - - let mut tag_boosts = HashSet::new(); - let mut docid_boosts = Vec::new(); - let mut url_boosts = Vec::new(); - for boost in boosts { - match boost { - QueryBoost::DocId(doc_id) => docid_boosts.push(doc_id.clone()), - QueryBoost::Url(url) => url_boosts.push(url.clone()), - QueryBoost::Tag(tag_id) => { - tag_boosts.insert(*tag_id); - } - } - } - - let boosts = QueryBoosts { - tags: tag_boosts.into_iter().collect(), - favorite: favorite_id, - urls: url_boosts, - doc_ids: docid_boosts, - }; + ) -> SearchQueryResult; +} - let query = build_query( - index.schema(), - tokenizers, - fields, - query_string, - applied_lenses, - stats, - &boosts, - ); - - let collector = TopDocs::with_limit(num_results); - - let top_docs = searcher - .search(&query, &collector) - .expect("Unable to execute query"); - - log::debug!( - "query `{}` returned {} results from {} docs in {} ms", - query_string, - top_docs.len(), - searcher.num_docs(), - Instant::now().duration_since(start_timer).as_millis() - ); - - let doc_reader = self.reader.searcher(); - top_docs - .into_iter() - // Filter out negative scores - .filter(|(score, _)| *score > 0.0) - .flat_map(|(score, addr)| { - if let Ok(Some(doc)) = doc_reader.doc(addr).map(|x| document_to_struct(&x)) { - Some((score, doc)) - } else { - None - } - }) - .collect() +#[async_trait::async_trait] +pub trait WriteTrait { + /// Delete a single document. + async fn delete(&self, doc_id: &str) -> SearcherResult<()> { + self.delete_many_by_id(&[doc_id.to_owned()]).await?; + Ok(()) } + /// Delete documents from the index by id, returning the number of docs deleted. + async fn delete_many_by_id(&self, doc_ids: &[String]) -> SearcherResult; - pub async fn explain_search_with_lens( - &self, - doc: RetrievedDocument, - applied_lenses: &Vec, - query_string: &str, - favorite_id: Option, - boosts: &[QueryBoost], - stats: &mut QueryStats, - ) -> Option { - let mut tag_boosts = HashSet::new(); - let mut docid_boosts = Vec::new(); - let mut url_boosts = Vec::new(); - for boost in boosts { - match boost { - QueryBoost::DocId(doc_id) => docid_boosts.push(doc_id.clone()), - QueryBoost::Url(url) => url_boosts.push(url.clone()), - QueryBoost::Tag(tag_id) => { - tag_boosts.insert(*tag_id); - } - } - } - - let index = &self.index; - let reader = &self.reader; - let fields = DocFields::as_fields(); - - let tantivy_searcher = reader.searcher(); - let tokenizers = index.tokenizers().clone(); - let boosts = QueryBoosts { - tags: tag_boosts.into_iter().collect(), - favorite: favorite_id, - urls: url_boosts, - doc_ids: docid_boosts, - }; - - let query = build_query( - index.schema(), - tokenizers.clone(), - fields.clone(), - query_string, - applied_lenses, - stats, - &boosts, - ); - - let mut combined: Vec<(Occur, Box)> = vec![(Occur::Should, Box::new(query))]; - combined.push(( - Occur::Must, - Box::new(TermQuery::new( - Term::from_field_text(fields.id, &doc.doc_id), - // Needs WithFreqs otherwise scoring is wonky. - IndexRecordOption::WithFreqs, - )), - )); - - let content_terms = - query::terms_for_field(&index.schema(), &tokenizers, query_string, fields.content); - log::info!("Content Tokens {:?}", content_terms); - - let final_query = BooleanQuery::new(combined); - let collector = tantivy::collector::TopDocs::with_limit(1); - - let docs = tantivy_searcher - .search(&final_query, &collector) - .expect("Unable to execute query"); - for (score, addr) in docs { - if let Ok(Some(result)) = tantivy_searcher.doc(addr).map(|x| document_to_struct(&x)) { - if result.doc_id == doc.doc_id { - for t in content_terms { - let info = tantivy_searcher - .segment_reader(addr.segment_ord) - .inverted_index(fields.content) - .unwrap() - .get_term_info(&t.1); - log::info!("Term {:?} Info {:?} ", t, info); - } - - return Some(score); - } - } - } - None + async fn upsert(&self, single_doc: &DocumentUpdate) -> SearcherResult { + let upserted = self.upsert_many(&[single_doc.clone()]).await?; + Ok(upserted.get(0).expect("Expected a single doc").to_owned()) } + + /// Insert/update documents in the index, returning the list of document ids + async fn upsert_many(&self, updates: &[DocumentUpdate]) -> SearcherResult>; } +type SearcherResult = Result; + #[derive(Clone, Serialize)] pub struct RetrievedDocument { pub doc_id: String, @@ -454,14 +167,14 @@ pub fn document_to_struct(doc: &Document) -> Option { #[cfg(test)] mod test { - use crate::{DocumentUpdate, IndexPath, QueryStats, Searcher}; + use crate::client::Searcher; + use crate::{Boost, DocumentUpdate, IndexBackend, QueryBoost, SearchTrait, WriteTrait}; async fn _build_test_index(searcher: &mut Searcher) { searcher - .upsert_document(DocumentUpdate { + .upsert(&DocumentUpdate { doc_id: None, title: "Of Mice and Men", - description: "Of Mice and Men passage", domain: "example.com", url: "https://example.com/mice_and_men", content: @@ -474,14 +187,16 @@ mod test { debris of the winter’s flooding; and sycamores with mottled, white, recumbent limbs and branches that arch over the pool", tags: &vec![1_i64], + published_at: None, + last_modified: None, }) + .await .expect("Unable to add doc"); searcher - .upsert_document(DocumentUpdate { + .upsert(&DocumentUpdate { doc_id: None, title: "Of Mice and Men", - description: "Of Mice and Men passage", domain: "en.wikipedia.org", url: "https://en.wikipedia.org/mice_and_men", content: @@ -494,14 +209,16 @@ mod test { debris of the winter’s flooding; and sycamores with mottled, white, recumbent limbs and branches that arch over the pool", tags: &vec![2_i64], + published_at: None, + last_modified: None, }) + .await .expect("Unable to add doc"); searcher - .upsert_document(DocumentUpdate { + .upsert(&DocumentUpdate { doc_id: None, title: "Of Cheese and Crackers", - description: "Of Cheese and Crackers Passage", domain: "en.wikipedia.org", url: "https://en.wikipedia.org/cheese_and_crackers", content: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla @@ -512,14 +229,16 @@ mod test { ac volutpat massa. Vivamus sed imperdiet est, id pretium ex. Praesent suscipit mattis ipsum, a lacinia nunc semper vitae.", tags: &vec![2_i64], + published_at: None, + last_modified: None, }) + .await .expect("Unable to add doc"); - searcher.upsert_document( - DocumentUpdate { + searcher.upsert( + &DocumentUpdate { doc_id: None, title:"Frankenstein: The Modern Prometheus", - description: "A passage from Frankenstein", domain:"monster.com", url:"https://example.com/frankenstein", content:"You will rejoice to hear that no disaster has accompanied the commencement of an @@ -527,8 +246,9 @@ mod test { yesterday, and my first task is to assure my dear sister of my welfare and increasing confidence in the success of my undertaking.", tags: &vec![1_i64], - } - ) + published_at: None, + last_modified: None + }).await .expect("Unable to add doc"); let res = searcher.save().await; @@ -543,44 +263,36 @@ mod test { #[tokio::test] pub async fn test_basic_lense_search() { let mut searcher = - Searcher::with_index(&IndexPath::Memory, false).expect("Unable to open index"); + Searcher::with_index(&IndexBackend::Memory, false).expect("Unable to open index"); _build_test_index(&mut searcher).await; - let mut stats = QueryStats::new(); let query = "salinas"; - let results = searcher - .search_with_lens(&vec![2_u64], query, None, &[], &mut stats, 5) - .await; - - assert_eq!(results.len(), 1); + let filters = vec![QueryBoost::new(Boost::Tag(2_u64))]; + let results = searcher.search(query, &filters, &[], 5).await; + assert_eq!(results.documents.len(), 1); } #[tokio::test] pub async fn test_url_lens_search() { let mut searcher = - Searcher::with_index(&IndexPath::Memory, false).expect("Unable to open index"); - - let mut stats = QueryStats::new(); + Searcher::with_index(&IndexBackend::Memory, false).expect("Unable to open index"); _build_test_index(&mut searcher).await; - let query = "salinas"; - let results = searcher - .search_with_lens(&vec![2_u64], query, None, &[], &mut stats, 5) - .await; - assert_eq!(results.len(), 1); + let query = "salinas"; + let filters = vec![QueryBoost::new(Boost::Tag(2_u64))]; + let results = searcher.search(query, &filters, &[], 5).await; + assert_eq!(results.documents.len(), 1); } #[tokio::test] pub async fn test_singular_url_lens_search() { let mut searcher = - Searcher::with_index(&IndexPath::Memory, false).expect("Unable to open index"); + Searcher::with_index(&IndexBackend::Memory, false).expect("Unable to open index"); _build_test_index(&mut searcher).await; - let mut stats = QueryStats::new(); let query = "salinasd"; - let results = searcher - .search_with_lens(&vec![2_u64], query, None, &[], &mut stats, 5) - .await; - assert_eq!(results.len(), 0); + let filters = vec![QueryBoost::new(Boost::Tag(2_u64))]; + let results = searcher.search(query, &filters, &[], 5).await; + assert_eq!(results.documents.len(), 0); } } diff --git a/crates/spyglass-searcher/src/query.rs b/crates/spyglass-searcher/src/query.rs index 2d98e1bc5..b834e57f3 100644 --- a/crates/spyglass-searcher/src/query.rs +++ b/crates/spyglass-searcher/src/query.rs @@ -1,29 +1,15 @@ use tantivy::query::{BooleanQuery, BoostQuery, Occur, PhraseQuery, Query, TermQuery}; -use tantivy::schema::*; use tantivy::tokenizer::*; use tantivy::Score; +use tantivy::{schema::*, Index}; + +use crate::schema::SearchDocument; +use crate::{Boost, QueryBoost}; use super::DocFields; type QueryVec = Vec<(Occur, Box)>; -#[derive(Clone, Debug)] -pub struct QueryStats { - pub term_count: i32, -} - -impl Default for QueryStats { - fn default() -> Self { - Self::new() - } -} - -impl QueryStats { - pub fn new() -> Self { - QueryStats { term_count: -1 } - } -} - fn _boosted_term(term: Term, boost: Score) -> Box { Box::new(BoostQuery::new( Box::new(TermQuery::new( @@ -46,33 +32,47 @@ fn _boosted_phrase(terms: Vec<(usize, Term)>, boost: Score) -> Box { )) } -#[derive(Clone, Default)] -pub struct QueryBoosts { - /// Boosts based on implicit/explicit tag detection - pub tags: Vec, - /// Id of favorited boost - pub favorite: Option, - /// Urls to boost - pub urls: Vec, - /// Specific doc ids to boost - pub doc_ids: Vec, +pub struct QueryOptions { + /// single term matches in the content + content_boost: f32, + /// full phrase matches in the content + content_phrase_boost: f32, + /// single term matches in the title + title_boost: f32, + /// full phrase matches in the title + title_phrase_boost: f32, +} + +impl Default for QueryOptions { + fn default() -> Self { + QueryOptions { + content_boost: 1.0, + content_phrase_boost: 1.5, + // weight title matches a little more + title_boost: 2.0, + title_phrase_boost: 2.5, + } + } } -#[allow(clippy::too_many_arguments)] pub fn build_query( - schema: Schema, - tokenizers: TokenizerManager, - fields: DocFields, + index: &Index, query_string: &str, // Applied filters - applied_lenses: &Vec, - stats: &mut QueryStats, - boosts: &QueryBoosts, -) -> BooleanQuery { - let content_terms = terms_for_field(&schema, &tokenizers, query_string, fields.content); - let title_terms = terms_for_field(&schema, &tokenizers, query_string, fields.title); + filters: &[QueryBoost], + // Applied boosts, + boosts: &[QueryBoost], + // title/content boost options + opts: QueryOptions, +) -> (usize, BooleanQuery) { + let schema = index.schema(); + let tokenizers = index.tokenizers(); + let fields = DocFields::as_fields(); - stats.term_count = content_terms.len() as i32; + let content_terms = terms_for_field(&schema, tokenizers, query_string, fields.content); + let title_terms = terms_for_field(&schema, tokenizers, query_string, fields.title); + + let term_count = content_terms.len(); let mut term_query: QueryVec = Vec::new(); @@ -80,7 +80,7 @@ pub fn build_query( if content_terms.len() > 1 { // boosting phrases relative to the number of segments in a // continuous phrase - let boost = 2.0 * content_terms.len() as f32; + let boost = opts.content_phrase_boost * content_terms.len() as f32; term_query.push((Occur::Should, _boosted_phrase(content_terms.clone(), boost))); } @@ -89,60 +89,81 @@ pub fn build_query( // boosting phrases relative to the number of segments in a // continuous phrase, base score higher for title // than content - let boost = 2.5 * title_terms.len() as f32; + let boost = opts.title_phrase_boost * title_terms.len() as f32; term_query.push((Occur::Should, _boosted_phrase(title_terms.clone(), boost))); } for (_position, term) in content_terms { - term_query.push((Occur::Should, _boosted_term(term, 1.0))); + term_query.push((Occur::Should, _boosted_term(term, opts.content_boost))); } for (_position, term) in title_terms { - term_query.push((Occur::Should, _boosted_term(term, 2.0))); - } - - // Tags that might be represented by search terms (e.g. "repository" or "file") - for tag_id in &boosts.tags { - term_query.push(( - Occur::Should, - _boosted_term(Term::from_field_u64(fields.tags, *tag_id), 1.5), - )) - } - - // Greatly boost selected urls - // todo: handle regex/prefixes? - for url in &boosts.urls { - term_query.push(( - Occur::Should, - _boosted_term(Term::from_field_text(fields.url, url), 3.0), - )); - } - - // Greatly boost selected docs - for doc_id in &boosts.doc_ids { - term_query.push(( - Occur::Should, - _boosted_term(Term::from_field_text(fields.id, doc_id), 3.0), - )); - } - + term_query.push((Occur::Should, _boosted_term(term, opts.title_boost))); + } + + // Boost fields that happen to have a value, such as + // - Tags that might be represented by search terms (e.g. "repository" or "file") + // - Certain URLs or documents we want to focus on + for boost in boosts { + let term = match &boost.field { + Boost::DocId(doc_id) => { + // Originally boosted to 3.0 + _boosted_term(Term::from_field_text(fields.id, doc_id), boost.value) + } + // Only considered in filters + Boost::Favorite { .. } => continue, + Boost::Tag(tag_id) => { + // Defaults to 1.5 + _boosted_term(Term::from_field_u64(fields.tags, *tag_id), boost.value) + } + // todo: handle regex/prefixes? + Boost::Url(url) => { + // Originally boosted to 3.0 + _boosted_term(Term::from_field_text(fields.url, url), boost.value) + } + }; + + term_query.push((Occur::Should, term)); + } + + // Must hit at least one of the terms let mut combined: QueryVec = vec![(Occur::Must, Box::new(BooleanQuery::new(term_query)))]; - for id in applied_lenses { - combined.push(( - Occur::Must, - _boosted_term(Term::from_field_u64(fields.tags, *id), 0.0), - )); - } - - // Greatly boost content that have our terms + a favorite. - if let Some(favorite_boost) = boosts.favorite { - combined.push(( - Occur::Should, - _boosted_term(Term::from_field_u64(fields.tags, favorite_boost), 3.0), - )); - } - - BooleanQuery::new(combined) + // Must have one of these, will filter out stuff that doesn't + for filter in filters { + let term = match &filter.field { + Boost::DocId(doc_id) => { + // Originally boosted to 3.0 + _boosted_term(Term::from_field_text(fields.id, doc_id), 0.0) + } + Boost::Favorite { id, required } => { + let occur = if *required { + Occur::Must + } else { + Occur::Should + }; + + combined.push(( + occur, + _boosted_term(Term::from_field_u64(fields.tags, *id), 3.0), + )); + + continue; + } + Boost::Tag(tag_id) => { + // Defaults to 1.5 + _boosted_term(Term::from_field_u64(fields.tags, *tag_id), 0.0) + } + // todo: handle regex/prefixes? + Boost::Url(url) => { + // Originally boosted to 3.0 + _boosted_term(Term::from_field_text(fields.url, url), 0.0) + } + }; + + combined.push((Occur::Must, term)); + } + + (term_count, BooleanQuery::new(combined)) } /// Helper method used to build a document query based on urls, ids or tags. diff --git a/crates/spyglass-searcher/src/schema.rs b/crates/spyglass-searcher/src/schema.rs index 1a88cf0e2..fa71250d0 100644 --- a/crates/spyglass-searcher/src/schema.rs +++ b/crates/spyglass-searcher/src/schema.rs @@ -1,3 +1,4 @@ +use chrono::Utc; use std::path::PathBuf; use tantivy::{directory::MmapDirectory, schema::*, tokenizer::*, Index}; @@ -77,6 +78,19 @@ pub fn register_tokenizer(index: &Index) { .register(TOKENIZER_NAME, full_content_tokenizer_en); } +/// A new document to be added +#[derive(Clone)] +pub struct DocumentUpdate<'a> { + pub doc_id: Option, + pub title: &'a str, + pub domain: &'a str, + pub url: &'a str, + pub content: &'a str, + pub tags: &'a [i64], + pub published_at: Option>, + pub last_modified: Option>, +} + #[derive(Clone)] pub struct DocFields { pub id: Field, diff --git a/crates/spyglass-searcher/src/similarity.rs b/crates/spyglass-searcher/src/similarity.rs index 9d2ffa92e..290e5dcf2 100644 --- a/crates/spyglass-searcher/src/similarity.rs +++ b/crates/spyglass-searcher/src/similarity.rs @@ -4,7 +4,8 @@ use shared::response::SimilaritySearchResult; use std::env; use std::time::SystemTime; -use super::{document_to_struct, Searcher}; +use super::client::Searcher; +use crate::SearchTrait; const EMBEDDING_ENDPOINT: &str = "SIMILARITY_SEARCH_ENDPOINT"; const EMBEDDING_PORT: &str = "SIMILARITY_SEARCH_PORT"; @@ -74,14 +75,11 @@ pub async fn generate_similarity_context( log::info!("Generate Similarity for {} docs", doc_ids.len()); for doc_id in doc_ids { - if let Some(doc) = searcher.get_by_id(doc_id) { - if let Some(doc) = document_to_struct(&doc) { - if let Some(doc_context) = - generate_similarity_context_for_doc(&client, query, &doc.content, &doc.url) - .await - { - context.push(doc_context); - } + if let Some(doc) = searcher.get(doc_id).await { + if let Some(doc_context) = + generate_similarity_context_for_doc(&client, query, &doc.content, &doc.url).await + { + context.push(doc_context); } } } diff --git a/crates/spyglass-searcher/src/utils.rs b/crates/spyglass-searcher/src/utils.rs index 4b8542937..51d30457a 100644 --- a/crates/spyglass-searcher/src/utils.rs +++ b/crates/spyglass-searcher/src/utils.rs @@ -153,14 +153,15 @@ pub fn group_urls_by_scheme(urls: Vec<&str>) -> HashMap<&str, Vec<&str>> { #[cfg(test)] mod test { + use crate::client::Searcher; use crate::schema::{DocFields, SearchDocument}; use crate::utils::generate_highlight_preview; - use crate::{IndexPath, Searcher}; + use crate::IndexBackend; #[test] fn test_find_highlights() { let searcher = - Searcher::with_index(&IndexPath::Memory, false).expect("Unable to open index"); + Searcher::with_index(&IndexBackend::Memory, false).expect("Unable to open index"); let blurb = r#"Rust rust is a multi-paradigm, high-level, general-purpose programming"#; let fields = DocFields::as_fields(); diff --git a/crates/spyglass/bin/debug/src/main.rs b/crates/spyglass/bin/debug/src/main.rs index 3e6ec2f67..56b06a66d 100644 --- a/crates/spyglass/bin/debug/src/main.rs +++ b/crates/spyglass/bin/debug/src/main.rs @@ -10,7 +10,7 @@ use tracing_log::LogTracer; use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter}; use libspyglass::pipeline::cache_pipeline::process_update; -use spyglass_searcher::{IndexPath, QueryBoost, QueryStats, Searcher}; +use spyglass_searcher::{client::Searcher, Boost, IndexBackend, QueryBoost}; #[cfg(debug_assertions)] const LOG_LEVEL: &str = "spyglassdebug=DEBUG"; @@ -119,7 +119,7 @@ async fn main() -> anyhow::Result { ron::ser::to_string_pretty(&tags, PrettyConfig::new()).unwrap_or_default() ); let index = - Searcher::with_index(&IndexPath::LocalPath(config.index_dir()), true) + Searcher::with_index(&IndexBackend::LocalPath(config.index_dir()), true) .expect("Unable to open index."); let docs = index @@ -156,7 +156,7 @@ async fn main() -> anyhow::Result { } }; - let index = Searcher::with_index(&IndexPath::LocalPath(config.index_dir()), true) + let index = Searcher::with_index(&IndexBackend::LocalPath(config.index_dir()), true) .expect("Unable to open index."); let docs = index @@ -166,23 +166,15 @@ async fn main() -> anyhow::Result { if docs.is_empty() { println!("No indexed document for url {:?}", id_or_url); } else { - for (_score, doc_addr) in docs { - let mut stats = QueryStats::default(); + for (_score, doc) in docs { let boosts = check_query_for_tags(&db, &query) .await .iter() - .map(|x| QueryBoost::Tag(*x)) + .map(|x| QueryBoost::new(Boost::Tag(*x))) .collect::>(); let explain = index - .explain_search_with_lens( - doc_addr, - &vec![], - query.as_str(), - None, - &boosts, - &mut stats, - ) + .explain_search_with_lens(doc.doc_id, query.as_str(), &boosts) .await; match explain { Some(explanation) => { diff --git a/crates/spyglass/src/api/handler/mod.rs b/crates/spyglass/src/api/handler/mod.rs index 6dd365188..b781daaf4 100644 --- a/crates/spyglass/src/api/handler/mod.rs +++ b/crates/spyglass/src/api/handler/mod.rs @@ -28,6 +28,7 @@ use shared::response::{ PluginResult, SupportedConnection, UserConnection, }; use spyglass_rpc::{RpcEvent, RpcEventType}; +use spyglass_searcher::WriteTrait; use std::collections::HashMap; use std::path::PathBuf; use std::str::FromStr; @@ -198,7 +199,7 @@ pub async fn app_status(state: AppState) -> Result { /// Remove a doc from the index #[instrument(skip(state))] pub async fn delete_document(state: AppState, id: String) -> Result<(), Error> { - if let Err(e) = state.index.delete_by_id(&id).await { + if let Err(e) = state.index.delete(&id).await { log::error!("Unable to delete doc {} due to {}", id, e); return Err(Error::Custom(e.to_string())); } @@ -658,7 +659,8 @@ mod test { }; use libspyglass::state::AppState; use shared::config::{Config, LensConfig}; - use spyglass_searcher::DocumentUpdate; + use spyglass_searcher::schema::DocumentUpdate; + use spyglass_searcher::WriteTrait; #[tokio::test] async fn test_uninstall_lens() { @@ -674,15 +676,17 @@ mod test { state .index - .upsert_document(DocumentUpdate { + .upsert(&DocumentUpdate { doc_id: Some("test_id".into()), title: "test title", - description: "test desc", domain: "example.com", url: "https://example.com/test", content: "test content", tags: &[], + published_at: None, + last_modified: None, }) + .await .expect("Unable to add doc"); let _ = state.index.save().await; diff --git a/crates/spyglass/src/api/handler/search.rs b/crates/spyglass/src/api/handler/search.rs index f22965eda..22633dba2 100644 --- a/crates/spyglass/src/api/handler/search.rs +++ b/crates/spyglass/src/api/handler/search.rs @@ -10,7 +10,7 @@ use shared::metrics; use shared::request; use shared::response::{LensResult, SearchLensesResp, SearchMeta, SearchResult, SearchResults}; use spyglass_searcher::schema::{DocFields, SearchDocument}; -use spyglass_searcher::{QueryBoost, QueryStats}; +use spyglass_searcher::{Boost, QueryBoost, SearchTrait}; use std::collections::HashSet; use std::time::SystemTime; use tracing::instrument; @@ -45,20 +45,33 @@ pub async fn search_docs( let mut boosts = Vec::new(); for tag in check_query_for_tags(&state.db, &query).await { - boosts.push(QueryBoost::Tag(tag)) + boosts.push(QueryBoost::new(Boost::Tag(tag))) } - let favorite_boost = get_favorite_tag(&state.db).await; - let mut stats = QueryStats::new(); - let docs = state - .index - .search_with_lens(&lens_ids, &query, favorite_boost, &boosts, &mut stats, 5) - .await; + let mut filters = Vec::new(); + for lens in lens_ids { + filters.push(QueryBoost::new(Boost::Tag(lens))); + } + + if let Some(tag_id) = get_favorite_tag(&state.db).await { + filters.push(QueryBoost::new(Boost::Favorite { + id: tag_id, + required: false, + })); + } + + let search_result = state.index.search(&query, &filters, &boosts, 5).await; + log::debug!( + "query {}: {} results from {} docs in {}ms", + query, + search_result.documents.len(), + search_result.num_docs, + search_result.wall_time_ms + ); let mut results: Vec = Vec::new(); let mut missing: Vec<(String, String)> = Vec::new(); - - for (score, doc) in docs { + for (score, doc) in search_result.documents { log::debug!("Got id with url {} {}", doc.doc_id, doc.url); let indexed = indexed_document::Entity::find() .filter(indexed_document::Column::DocId.eq(doc.doc_id.clone())) @@ -125,7 +138,7 @@ pub async fn search_docs( .track(metrics::Event::SearchResult { num_results: results.len(), num_docs, - term_count: stats.term_count, + term_count: search_result.term_counts as i32, domains: domains.iter().cloned().collect(), wall_time_ms, }) diff --git a/crates/spyglass/src/api/mod.rs b/crates/spyglass/src/api/mod.rs index f0d8e0735..b5c45cd7a 100644 --- a/crates/spyglass/src/api/mod.rs +++ b/crates/spyglass/src/api/mod.rs @@ -13,6 +13,7 @@ use shared::config::{Config, UserSettings}; use shared::request::{BatchDocumentRequest, RawDocumentRequest, SearchLensesParam, SearchParam}; use shared::response::{self as resp, DefaultIndices, LibraryStats}; use spyglass_rpc::{RpcEventType, RpcServer}; +use spyglass_searcher::WriteTrait; use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; diff --git a/crates/spyglass/src/connection/github.rs b/crates/spyglass/src/connection/github.rs index 7923730a9..a270bceb6 100644 --- a/crates/spyglass/src/connection/github.rs +++ b/crates/spyglass/src/connection/github.rs @@ -9,6 +9,7 @@ use entities::{ use jsonrpsee::core::async_trait; use libgithub::types::{Issue, Repo}; use libgithub::GithubClient; +use spyglass_searcher::WriteTrait; use strum_macros::{Display, EnumString}; use url::Url; diff --git a/crates/spyglass/src/documents/mod.rs b/crates/spyglass/src/documents/mod.rs index 65e549290..18820f014 100644 --- a/crates/spyglass/src/documents/mod.rs +++ b/crates/spyglass/src/documents/mod.rs @@ -18,7 +18,7 @@ use url::Url; use crate::{crawler::CrawlResult, state::AppState}; use entities::models::tag::TagType; use entities::sea_orm::{ColumnTrait, EntityTrait, QueryFilter, Set, TransactionTrait}; -use spyglass_searcher::{DocumentUpdate, RetrievedDocument}; +use spyglass_searcher::{schema::DocumentUpdate, RetrievedDocument, WriteTrait}; /// Helper method to delete indexed documents, crawl queue items and search /// documents by url @@ -138,15 +138,19 @@ pub async fn process_crawl_results( let url = Url::parse(&crawl_result.url)?; let url_host = url.host_str().unwrap_or(""); // Add document to index - let doc_id = state.index.upsert_document(DocumentUpdate { - doc_id: id_map.get(&crawl_result.url).cloned(), - title: &crawl_result.title.clone().unwrap_or_default(), - description: &crawl_result.description.clone().unwrap_or_default(), - domain: url_host, - url: url.as_str(), - content: &crawl_result.content.clone().unwrap_or_default(), - tags: &tags_for_crawl.clone(), - })?; + let doc_id = state + .index + .upsert(&DocumentUpdate { + doc_id: id_map.get(&crawl_result.url).cloned(), + title: &crawl_result.title.clone().unwrap_or_default(), + domain: url_host, + url: url.as_str(), + content: &crawl_result.content.clone().unwrap_or_default(), + tags: &tags_for_crawl.clone(), + published_at: None, + last_modified: None, + }) + .await?; if !id_map.contains_key(&doc_id) { added_docs.push(url.to_string()); @@ -273,15 +277,20 @@ pub async fn process_records( let url_host = url.host_str().unwrap_or(""); // Add document to index let doc_id: Option = { - match state.index.upsert_document(DocumentUpdate { - doc_id: id_map.get(&canonical_url_str.clone()).cloned(), - title: &crawl_result.title.clone().unwrap_or_default(), - description: &crawl_result.description.clone(), - domain: url_host, - url: url.as_str(), - content: &crawl_result.content, - tags: &tag_list, - }) { + match state + .index + .upsert(&DocumentUpdate { + doc_id: id_map.get(&canonical_url_str.clone()).cloned(), + title: &crawl_result.title.clone().unwrap_or_default(), + domain: url_host, + url: url.as_str(), + content: &crawl_result.content, + tags: &tag_list, + published_at: None, + last_modified: None, + }) + .await + { Ok(new_doc_id) => Some(new_doc_id), _ => None, } @@ -425,15 +434,19 @@ pub async fn update_tags( log::debug!("Tag map generated {}", tag_map.len()); for (_, (doc, ids)) in tag_map.iter() { - let _doc_id = state.index.upsert_document(DocumentUpdate { - doc_id: Some(doc.doc_id.clone()), - title: &doc.title, - description: &doc.description, - domain: &doc.domain, - url: &doc.url, - content: &doc.content, - tags: ids, - })?; + let _doc_id = state + .index + .upsert(&DocumentUpdate { + doc_id: Some(doc.doc_id.clone()), + title: &doc.title, + domain: &doc.domain, + url: &doc.url, + content: &doc.content, + tags: ids, + published_at: None, + last_modified: None, + }) + .await?; } } diff --git a/crates/spyglass/src/pipeline/default_pipeline.rs b/crates/spyglass/src/pipeline/default_pipeline.rs index 9986df5b6..14d4f7e83 100644 --- a/crates/spyglass/src/pipeline/default_pipeline.rs +++ b/crates/spyglass/src/pipeline/default_pipeline.rs @@ -2,12 +2,12 @@ use crate::pipeline::collector::DefaultCollector; use crate::pipeline::PipelineContext; use crate::state::AppState; use crate::task::CrawlTask; - use entities::models::{crawl_queue, indexed_document}; use entities::sea_orm::prelude::*; use entities::sea_orm::{ColumnTrait, EntityTrait, QueryFilter, Set}; use shared::config::{Config, LensConfig, PipelineConfiguration}; -use spyglass_searcher::DocumentUpdate; +use spyglass_searcher::schema::DocumentUpdate; +use spyglass_searcher::WriteTrait; use tokio::sync::mpsc; use url::Url; @@ -126,21 +126,26 @@ async fn start_crawl( // Delete old document, if any. if let Some(doc) = &existing { - let _ = state.index.delete_by_id(&doc.doc_id).await; + let _ = state.index.delete(&doc.doc_id).await; let _ = indexed_document::delete_many_by_id(&state.db, &[doc.id]).await; } // Add document to index let doc_id: Option = { - match state.index.upsert_document(DocumentUpdate { - doc_id: existing.clone().map(|f| f.doc_id), - title: &crawl_result.title.unwrap_or_default(), - description: &crawl_result.description.unwrap_or_default(), - domain: url_host, - url: url.as_str(), - content: &content, - tags: &[], - }) { + match state + .index + .upsert(&DocumentUpdate { + doc_id: existing.clone().map(|f| f.doc_id), + title: &crawl_result.title.unwrap_or_default(), + domain: url_host, + url: url.as_str(), + content: &content, + tags: &[], + published_at: None, + last_modified: None, + }) + .await + { Ok(new_doc_id) => Some(new_doc_id), _ => None, } diff --git a/crates/spyglass/src/plugin/exports.rs b/crates/spyglass/src/plugin/exports.rs index fe00d96b3..d77969cb2 100644 --- a/crates/spyglass/src/plugin/exports.rs +++ b/crates/spyglass/src/plugin/exports.rs @@ -14,7 +14,7 @@ use spyglass_plugin::Authentication; use spyglass_plugin::DocumentUpdate; use spyglass_plugin::HttpMethod; use spyglass_plugin::{DocumentResult, PluginEvent}; -use spyglass_searcher::RetrievedDocument; +use spyglass_searcher::{RetrievedDocument, WriteTrait}; use std::path::Path; use std::str::FromStr; use tokio::sync::mpsc::Sender; diff --git a/crates/spyglass/src/plugin/mod.rs b/crates/spyglass/src/plugin/mod.rs index a570c4089..6a5143ae4 100644 --- a/crates/spyglass/src/plugin/mod.rs +++ b/crates/spyglass/src/plugin/mod.rs @@ -552,7 +552,7 @@ mod test { use entities::test::setup_test_db; use shared::config::{LensConfig, UserSettings}; use spyglass_plugin::SearchFilter; - use spyglass_searcher::IndexPath; + use spyglass_searcher::IndexBackend; use super::{lens_to_filters, AppState}; @@ -578,7 +578,7 @@ mod test { .with_db(db) .with_lenses(&vec![test_lens]) .with_user_settings(&UserSettings::default()) - .with_index(&IndexPath::Memory, false) + .with_index(&IndexBackend::Memory, false) .build(); let filters = lens_to_filters(state, "test").await; diff --git a/crates/spyglass/src/state.rs b/crates/spyglass/src/state.rs index 619e76ac0..e4fae93d3 100644 --- a/crates/spyglass/src/state.rs +++ b/crates/spyglass/src/state.rs @@ -17,7 +17,7 @@ use crate::{ }; use shared::config::{Config, LensConfig, PipelineConfiguration, UserSettings}; use shared::metrics::Metrics; -use spyglass_searcher::{IndexPath, Searcher}; +use spyglass_searcher::{client::Searcher, IndexBackend}; /// Used to track inflight requests and limit things #[derive(Clone, Debug, Hash, PartialEq, Eq)] @@ -88,7 +88,7 @@ impl AppState { AppStateBuilder::new() .with_db(db) - .with_index(&IndexPath::LocalPath(config.index_dir()), readonly_mode) + .with_index(&IndexBackend::LocalPath(config.index_dir()), readonly_mode) .with_lenses(&config.lenses.values().cloned().collect()) .with_pipelines( &config @@ -166,7 +166,7 @@ impl AppStateBuilder { let index = if let Some(index) = &self.index { index.to_owned() } else { - Searcher::with_index(&IndexPath::Memory, false).expect("Unable to open search index") + Searcher::with_index(&IndexBackend::Memory, false).expect("Unable to open search index") }; let user_settings = if let Some(settings) = &self.user_settings { @@ -229,8 +229,8 @@ impl AppStateBuilder { self } - pub fn with_index(&mut self, index: &IndexPath, readonly: bool) -> &mut Self { - if let IndexPath::LocalPath(path) = &index { + pub fn with_index(&mut self, index: &IndexBackend, readonly: bool) -> &mut Self { + if let IndexBackend::LocalPath(path) = &index { if !path.exists() { let _ = std::fs::create_dir_all(path); } diff --git a/crates/spyglass/src/task/worker.rs b/crates/spyglass/src/task/worker.rs index d8bd9c23b..0abc3b341 100644 --- a/crates/spyglass/src/task/worker.rs +++ b/crates/spyglass/src/task/worker.rs @@ -7,6 +7,7 @@ use entities::models::{ use entities::sea_orm::prelude::*; use entities::sea_orm::{ColumnTrait, EntityTrait, QueryFilter, Set}; use shared::config::{Config, LensConfig, LensSource}; +use spyglass_searcher::{SearchTrait, WriteTrait}; use super::{bootstrap, CollectTask, ManagerCommand}; use super::{CleanupTask, CrawlTask}; @@ -67,7 +68,7 @@ pub async fn cleanup_database(state: &AppState, cleanup_task: CleanupTask) -> an // Found document for the url, but it has a different doc id. // check if this document exists in the index to see if we // had a duplicate - let indexed_result = state.index.get_by_id(doc_model.doc_id.as_str()); + let indexed_result = state.index.get(doc_model.doc_id.as_str()).await; match indexed_result { Some(_doc) => { log::debug!( @@ -76,7 +77,7 @@ pub async fn cleanup_database(state: &AppState, cleanup_task: CleanupTask) -> an doc_id ); // Found indexed document, so we must have had duplicates, remove dup - let _ = state.index.delete_by_id(doc_id.as_str()).await; + let _ = state.index.delete(doc_id.as_str()).await; let _ = indexed_document::delete_many_by_doc_id(&state.db, &[doc_id]) .await; @@ -98,7 +99,7 @@ pub async fn cleanup_database(state: &AppState, cleanup_task: CleanupTask) -> an Ok(None) => { log::debug!("Could not find document for url {}, removing", url); // can't find the url at all must be an old doc that was removed - let _ = state.index.delete_by_id(doc_id.as_str()).await; + let _ = state.index.delete(doc_id.as_str()).await; let _ = indexed_document::delete_many_by_doc_id(&state.db, &[doc_id]).await; changed = true; } @@ -321,7 +322,7 @@ mod test { use entities::sea_orm::{ActiveModelTrait, EntityTrait, ModelTrait, Set}; use entities::test::setup_test_db; use shared::config::{LensConfig, UserSettings}; - use spyglass_searcher::IndexPath; + use spyglass_searcher::IndexBackend; use super::{handle_cdx_collection, process_crawl, AppState, FetchResult}; @@ -334,7 +335,7 @@ mod test { let state = AppState::builder() .with_db(db) .with_user_settings(&UserSettings::default()) - .with_index(&IndexPath::Memory, false) + .with_index(&IndexBackend::Memory, false) .build(); // Should skip this lens since it's been bootstrapped already. @@ -352,7 +353,7 @@ mod test { let state = AppState::builder() .with_db(db.clone()) .with_user_settings(&UserSettings::default()) - .with_index(&IndexPath::Memory, false) + .with_index(&IndexBackend::Memory, false) .build(); let model = crawl_queue::ActiveModel { @@ -401,7 +402,7 @@ mod test { let state = AppState::builder() .with_db(db.clone()) .with_user_settings(&UserSettings::default()) - .with_index(&IndexPath::Memory, false) + .with_index(&IndexBackend::Memory, false) .build(); let task = crawl_queue::ActiveModel { @@ -452,7 +453,7 @@ mod test { let state = AppState::builder() .with_db(db.clone()) .with_user_settings(&UserSettings::default()) - .with_index(&IndexPath::Memory, false) + .with_index(&IndexBackend::Memory, false) .build(); let model = crawl_queue::ActiveModel { @@ -517,7 +518,7 @@ mod test { let state = AppState::builder() .with_db(db.clone()) .with_user_settings(&UserSettings::default()) - .with_index(&IndexPath::Memory, false) + .with_index(&IndexBackend::Memory, false) .build(); let task = crawl_queue::ActiveModel { From 8b80df1484a79edb1714fe69872d22061abf8416 Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Fri, 28 Apr 2023 11:30:27 -0700 Subject: [PATCH 15/30] make QueryBoost/Boost serializable --- crates/spyglass-searcher/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/spyglass-searcher/src/lib.rs b/crates/spyglass-searcher/src/lib.rs index 734b46e0b..95160d546 100644 --- a/crates/spyglass-searcher/src/lib.rs +++ b/crates/spyglass-searcher/src/lib.rs @@ -1,4 +1,4 @@ -use serde::Serialize; +use serde::{Serialize, Deserialize}; use std::fmt::Debug; use std::path::PathBuf; use tantivy::schema::*; @@ -24,7 +24,7 @@ pub enum IndexBackend { Memory, } -#[derive(Clone)] +#[derive(Clone, Serialize, Deserialize)] pub struct QueryBoost { /// What to boost field: Boost, @@ -48,7 +48,7 @@ impl QueryBoost { } } -#[derive(Clone)] +#[derive(Clone, Serialize, Deserialize)] pub enum Boost { // If required is set to true, _only_ favorites will be searched. Favorite { id: u64, required: bool }, From aab1a2758ece573428e9a38b2510a2893e0d1cb6 Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Fri, 28 Apr 2023 12:31:37 -0700 Subject: [PATCH 16/30] add ability to use a custom schema w/ the Searcher client (#442) --- .../m20230315_000001_migrate_search_schema.rs | 3 ++- crates/spyglass-searcher/src/client/local.rs | 10 ++++++--- crates/spyglass-searcher/src/lib.rs | 12 ++++++---- crates/spyglass-searcher/src/schema.rs | 6 ++--- crates/spyglass-searcher/src/utils.rs | 4 ++-- crates/spyglass/bin/debug/src/main.rs | 19 +++++++++++----- crates/spyglass/src/plugin/mod.rs | 5 +++-- crates/spyglass/src/state.rs | 22 +++++++++++++++---- crates/spyglass/src/task/worker.rs | 12 +++++----- 9 files changed, 62 insertions(+), 31 deletions(-) diff --git a/crates/migrations/src/m20230315_000001_migrate_search_schema.rs b/crates/migrations/src/m20230315_000001_migrate_search_schema.rs index 66555c574..0a2271ab6 100644 --- a/crates/migrations/src/m20230315_000001_migrate_search_schema.rs +++ b/crates/migrations/src/m20230315_000001_migrate_search_schema.rs @@ -22,7 +22,8 @@ impl Migration { } pub fn after_writer(&self, path: &PathBuf) -> IndexWriter { - let index = schema::initialize_index(path).expect("Unable to open search index"); + let index = schema::initialize_index(DocFields::as_schema(), path) + .expect("Unable to open search index"); index.writer(50_000_000).expect("Unable to create writer") } diff --git a/crates/spyglass-searcher/src/client/local.rs b/crates/spyglass-searcher/src/client/local.rs index 767d7ed8a..7831dd166 100644 --- a/crates/spyglass-searcher/src/client/local.rs +++ b/crates/spyglass-searcher/src/client/local.rs @@ -193,10 +193,14 @@ impl Searcher { } /// Constructs a new Searcher object w/ the index @ `index_path` - pub fn with_index(index_path: &IndexBackend, readonly: bool) -> SearcherResult { + pub fn with_index( + index_path: &IndexBackend, + schema: Schema, + readonly: bool, + ) -> SearcherResult { let index = match index_path { - IndexBackend::LocalPath(path) => schema::initialize_index(path)?, - IndexBackend::Memory => schema::initialize_in_memory_index(), + IndexBackend::LocalPath(path) => schema::initialize_index(schema, path)?, + IndexBackend::Memory => schema::initialize_in_memory_index(schema), IndexBackend::Http(_) => unimplemented!(""), }; diff --git a/crates/spyglass-searcher/src/lib.rs b/crates/spyglass-searcher/src/lib.rs index 95160d546..01a11d7c8 100644 --- a/crates/spyglass-searcher/src/lib.rs +++ b/crates/spyglass-searcher/src/lib.rs @@ -1,4 +1,4 @@ -use serde::{Serialize, Deserialize}; +use serde::{Deserialize, Serialize}; use std::fmt::Debug; use std::path::PathBuf; use tantivy::schema::*; @@ -168,6 +168,7 @@ pub fn document_to_struct(doc: &Document) -> Option { #[cfg(test)] mod test { use crate::client::Searcher; + use crate::schema::{DocFields, SearchDocument}; use crate::{Boost, DocumentUpdate, IndexBackend, QueryBoost, SearchTrait, WriteTrait}; async fn _build_test_index(searcher: &mut Searcher) { @@ -263,7 +264,8 @@ mod test { #[tokio::test] pub async fn test_basic_lense_search() { let mut searcher = - Searcher::with_index(&IndexBackend::Memory, false).expect("Unable to open index"); + Searcher::with_index(&IndexBackend::Memory, DocFields::as_schema(), false) + .expect("Unable to open index"); _build_test_index(&mut searcher).await; let query = "salinas"; @@ -275,7 +277,8 @@ mod test { #[tokio::test] pub async fn test_url_lens_search() { let mut searcher = - Searcher::with_index(&IndexBackend::Memory, false).expect("Unable to open index"); + Searcher::with_index(&IndexBackend::Memory, DocFields::as_schema(), false) + .expect("Unable to open index"); _build_test_index(&mut searcher).await; let query = "salinas"; @@ -287,7 +290,8 @@ mod test { #[tokio::test] pub async fn test_singular_url_lens_search() { let mut searcher = - Searcher::with_index(&IndexBackend::Memory, false).expect("Unable to open index"); + Searcher::with_index(&IndexBackend::Memory, DocFields::as_schema(), false) + .expect("Unable to open index"); _build_test_index(&mut searcher).await; let query = "salinasd"; diff --git a/crates/spyglass-searcher/src/schema.rs b/crates/spyglass-searcher/src/schema.rs index fa71250d0..e0173a2ff 100644 --- a/crates/spyglass-searcher/src/schema.rs +++ b/crates/spyglass-searcher/src/schema.rs @@ -46,8 +46,7 @@ pub fn mapping_to_schema(mapping: &SchemaMapping) -> Schema { } /// Helper used to create and configure an index from a path -pub fn initialize_index(index_path: &PathBuf) -> anyhow::Result { - let schema = DocFields::as_schema(); +pub fn initialize_index(schema: Schema, index_path: &PathBuf) -> anyhow::Result { let dir = MmapDirectory::open(index_path)?; let index = Index::open_or_create(dir, schema)?; register_tokenizer(&index); @@ -56,8 +55,7 @@ pub fn initialize_index(index_path: &PathBuf) -> anyhow::Result { } /// Helper used to create and configure an in memory index -pub fn initialize_in_memory_index() -> Index { - let schema = DocFields::as_schema(); +pub fn initialize_in_memory_index(schema: Schema) -> Index { let index = Index::create_in_ram(schema); register_tokenizer(&index); diff --git a/crates/spyglass-searcher/src/utils.rs b/crates/spyglass-searcher/src/utils.rs index 51d30457a..3ebd5bfee 100644 --- a/crates/spyglass-searcher/src/utils.rs +++ b/crates/spyglass-searcher/src/utils.rs @@ -160,8 +160,8 @@ mod test { #[test] fn test_find_highlights() { - let searcher = - Searcher::with_index(&IndexBackend::Memory, false).expect("Unable to open index"); + let searcher = Searcher::with_index(&IndexBackend::Memory, DocFields::as_schema(), false) + .expect("Unable to open index"); let blurb = r#"Rust rust is a multi-paradigm, high-level, general-purpose programming"#; let fields = DocFields::as_fields(); diff --git a/crates/spyglass/bin/debug/src/main.rs b/crates/spyglass/bin/debug/src/main.rs index 56b06a66d..b3dc94b9c 100644 --- a/crates/spyglass/bin/debug/src/main.rs +++ b/crates/spyglass/bin/debug/src/main.rs @@ -10,7 +10,8 @@ use tracing_log::LogTracer; use tracing_subscriber::{fmt, layer::SubscriberExt, EnvFilter}; use libspyglass::pipeline::cache_pipeline::process_update; -use spyglass_searcher::{client::Searcher, Boost, IndexBackend, QueryBoost}; +use spyglass_searcher::schema::SearchDocument; +use spyglass_searcher::{client::Searcher, schema::DocFields, Boost, IndexBackend, QueryBoost}; #[cfg(debug_assertions)] const LOG_LEVEL: &str = "spyglassdebug=DEBUG"; @@ -107,6 +108,7 @@ async fn main() -> anyhow::Result { let doc_details = models::indexed_document::get_document_details(&db, identifier).await?; + let schema = DocFields::as_schema(); println!("## Document Details ##"); match doc_details { Some((doc, tags)) => { @@ -118,9 +120,12 @@ async fn main() -> anyhow::Result { "Tags: {}", ron::ser::to_string_pretty(&tags, PrettyConfig::new()).unwrap_or_default() ); - let index = - Searcher::with_index(&IndexBackend::LocalPath(config.index_dir()), true) - .expect("Unable to open index."); + let index = Searcher::with_index( + &IndexBackend::LocalPath(config.index_dir()), + schema, + true, + ) + .expect("Unable to open index."); let docs = index .search_by_query(Some(vec![doc.url.clone()]), None, &[], &[]) @@ -156,8 +161,10 @@ async fn main() -> anyhow::Result { } }; - let index = Searcher::with_index(&IndexBackend::LocalPath(config.index_dir()), true) - .expect("Unable to open index."); + let schema = DocFields::as_schema(); + let index = + Searcher::with_index(&IndexBackend::LocalPath(config.index_dir()), schema, true) + .expect("Unable to open index."); let docs = index .search_by_query(doc_query.urls, doc_query.ids, &[], &[]) diff --git a/crates/spyglass/src/plugin/mod.rs b/crates/spyglass/src/plugin/mod.rs index 6a5143ae4..288d32953 100644 --- a/crates/spyglass/src/plugin/mod.rs +++ b/crates/spyglass/src/plugin/mod.rs @@ -552,7 +552,8 @@ mod test { use entities::test::setup_test_db; use shared::config::{LensConfig, UserSettings}; use spyglass_plugin::SearchFilter; - use spyglass_searcher::IndexBackend; + use spyglass_searcher::schema::SearchDocument; + use spyglass_searcher::{schema::DocFields, IndexBackend}; use super::{lens_to_filters, AppState}; @@ -578,7 +579,7 @@ mod test { .with_db(db) .with_lenses(&vec![test_lens]) .with_user_settings(&UserSettings::default()) - .with_index(&IndexBackend::Memory, false) + .with_index(&IndexBackend::Memory, DocFields::as_schema(), false) .build(); let filters = lens_to_filters(state, "test").await; diff --git a/crates/spyglass/src/state.rs b/crates/spyglass/src/state.rs index e4fae93d3..8ac16053a 100644 --- a/crates/spyglass/src/state.rs +++ b/crates/spyglass/src/state.rs @@ -3,7 +3,10 @@ use dashmap::DashMap; use entities::models::create_connection; use entities::sea_orm::DatabaseConnection; use spyglass_rpc::RpcEvent; +use spyglass_searcher::schema::DocFields; +use spyglass_searcher::schema::SearchDocument; use std::sync::Arc; +use tantivy::schema::Schema; use tokio::sync::mpsc::error::SendError; use tokio::sync::Mutex; use tokio::sync::{broadcast, mpsc}; @@ -88,7 +91,11 @@ impl AppState { AppStateBuilder::new() .with_db(db) - .with_index(&IndexBackend::LocalPath(config.index_dir()), readonly_mode) + .with_index( + &IndexBackend::LocalPath(config.index_dir()), + DocFields::as_schema(), + readonly_mode, + ) .with_lenses(&config.lenses.values().cloned().collect()) .with_pipelines( &config @@ -166,7 +173,8 @@ impl AppStateBuilder { let index = if let Some(index) = &self.index { index.to_owned() } else { - Searcher::with_index(&IndexBackend::Memory, false).expect("Unable to open search index") + Searcher::with_index(&IndexBackend::Memory, DocFields::as_schema(), false) + .expect("Unable to open search index") }; let user_settings = if let Some(settings) = &self.user_settings { @@ -229,14 +237,20 @@ impl AppStateBuilder { self } - pub fn with_index(&mut self, index: &IndexBackend, readonly: bool) -> &mut Self { + pub fn with_index( + &mut self, + index: &IndexBackend, + schema: Schema, + readonly: bool, + ) -> &mut Self { if let IndexBackend::LocalPath(path) = &index { if !path.exists() { let _ = std::fs::create_dir_all(path); } } - self.index = Some(Searcher::with_index(index, readonly).expect("Unable to open index")); + self.index = + Some(Searcher::with_index(index, schema, readonly).expect("Unable to open index")); self } } diff --git a/crates/spyglass/src/task/worker.rs b/crates/spyglass/src/task/worker.rs index 0abc3b341..155615edb 100644 --- a/crates/spyglass/src/task/worker.rs +++ b/crates/spyglass/src/task/worker.rs @@ -322,6 +322,8 @@ mod test { use entities::sea_orm::{ActiveModelTrait, EntityTrait, ModelTrait, Set}; use entities::test::setup_test_db; use shared::config::{LensConfig, UserSettings}; + use spyglass_searcher::schema::DocFields; + use spyglass_searcher::schema::SearchDocument; use spyglass_searcher::IndexBackend; use super::{handle_cdx_collection, process_crawl, AppState, FetchResult}; @@ -335,7 +337,7 @@ mod test { let state = AppState::builder() .with_db(db) .with_user_settings(&UserSettings::default()) - .with_index(&IndexBackend::Memory, false) + .with_index(&IndexBackend::Memory, DocFields::as_schema(), false) .build(); // Should skip this lens since it's been bootstrapped already. @@ -353,7 +355,7 @@ mod test { let state = AppState::builder() .with_db(db.clone()) .with_user_settings(&UserSettings::default()) - .with_index(&IndexBackend::Memory, false) + .with_index(&IndexBackend::Memory, DocFields::as_schema(), false) .build(); let model = crawl_queue::ActiveModel { @@ -402,7 +404,7 @@ mod test { let state = AppState::builder() .with_db(db.clone()) .with_user_settings(&UserSettings::default()) - .with_index(&IndexBackend::Memory, false) + .with_index(&IndexBackend::Memory, DocFields::as_schema(), false) .build(); let task = crawl_queue::ActiveModel { @@ -453,7 +455,7 @@ mod test { let state = AppState::builder() .with_db(db.clone()) .with_user_settings(&UserSettings::default()) - .with_index(&IndexBackend::Memory, false) + .with_index(&IndexBackend::Memory, DocFields::as_schema(), false) .build(); let model = crawl_queue::ActiveModel { @@ -518,7 +520,7 @@ mod test { let state = AppState::builder() .with_db(db.clone()) .with_user_settings(&UserSettings::default()) - .with_index(&IndexBackend::Memory, false) + .with_index(&IndexBackend::Memory, DocFields::as_schema(), false) .build(); let task = crawl_queue::ActiveModel { From 9ea3db004ffeacda6495c8841a21702ecfe3f32a Mon Sep 17 00:00:00 2001 From: travolin Date: Fri, 28 Apr 2023 15:02:42 -0700 Subject: [PATCH 17/30] Update stop word filter (#443) * Update stop word filter --------- Co-authored-by: Joel Bredeson --- crates/spyglass-searcher/Cargo.toml | 1 + crates/spyglass-searcher/src/lib.rs | 1 + crates/spyglass-searcher/src/schema.rs | 14 +++- .../spyglass-searcher/src/stop_word_filter.rs | 83 +++++++++++++++++++ 4 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 crates/spyglass-searcher/src/stop_word_filter.rs diff --git a/crates/spyglass-searcher/Cargo.toml b/crates/spyglass-searcher/Cargo.toml index 94f00bcc2..010cc87c3 100644 --- a/crates/spyglass-searcher/Cargo.toml +++ b/crates/spyglass-searcher/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" anyhow = "1.0" async-trait = "0.1.68" chrono = { version = "0.4.23", features = ["serde"] } +fnv = "1.0.7" log = "0.4" serde = "1.0" serde_json = "1.0" diff --git a/crates/spyglass-searcher/src/lib.rs b/crates/spyglass-searcher/src/lib.rs index 01a11d7c8..d46cfeedb 100644 --- a/crates/spyglass-searcher/src/lib.rs +++ b/crates/spyglass-searcher/src/lib.rs @@ -7,6 +7,7 @@ use url::Url; pub mod client; pub mod schema; +pub mod stop_word_filter; use schema::{DocFields, DocumentUpdate, SearchDocument}; mod query; diff --git a/crates/spyglass-searcher/src/schema.rs b/crates/spyglass-searcher/src/schema.rs index e0173a2ff..4da405c4c 100644 --- a/crates/spyglass-searcher/src/schema.rs +++ b/crates/spyglass-searcher/src/schema.rs @@ -1,7 +1,15 @@ +use super::stop_word_filter::StopWordFilter; use chrono::Utc; use std::path::PathBuf; -use tantivy::{directory::MmapDirectory, schema::*, tokenizer::*, Index}; - +use tantivy::{ + directory::MmapDirectory, + schema::*, + tokenizer::{ + AsciiFoldingFilter, Language, LowerCaser, RemoveLongFilter, SimpleTokenizer, Stemmer, + TextAnalyzer, + }, + Index, +}; pub type FieldName = String; pub const TOKENIZER_NAME: &str = "spyglass_tokenizer_en"; @@ -68,7 +76,7 @@ pub fn register_tokenizer(index: &Index) { .filter(RemoveLongFilter::limit(40)) .filter(LowerCaser) .filter(AsciiFoldingFilter) - .filter(StopWordFilter::new(Language::English).unwrap()) + .filter(StopWordFilter::default()) .filter(Stemmer::new(Language::English)); index diff --git a/crates/spyglass-searcher/src/stop_word_filter.rs b/crates/spyglass-searcher/src/stop_word_filter.rs new file mode 100644 index 000000000..fcf4f88b4 --- /dev/null +++ b/crates/spyglass-searcher/src/stop_word_filter.rs @@ -0,0 +1,83 @@ +use fnv::FnvHasher; +use std::collections::HashSet; +use std::hash::BuildHasherDefault; +use tantivy::tokenizer::{BoxTokenStream, Token, TokenFilter, TokenStream}; + +// configure our hashers for SPEED +type StopWordHasher = BuildHasherDefault; +type StopWordHashSet = HashSet; + +/// `TokenFilter` that removes stop words from a token stream +#[derive(Clone)] +pub struct StopWordFilter { + words: StopWordHashSet, +} + +impl StopWordFilter { + /// Creates a `StopWordFilter` given a list of words to remove + pub fn remove(words: Vec) -> StopWordFilter { + let mut set = StopWordHashSet::default(); + + for word in words { + set.insert(word); + } + + StopWordFilter { words: set } + } + + fn english() -> StopWordFilter { + let words: [&'static str; 44] = [ + "a", "about", "an", "and", "are", "as", "at", "be", "but", "by", "com", "for", "from", + "how", "if", "I", "in", "into", "is", "it", "no", "not", "of", "on", "or", "such", + "that", "the", "their", "then", "there", "these", "they", "this", "to", "was", "what", + "when", "where", "who", "will", "with", "the", "www", + ]; + + StopWordFilter::remove(words.iter().map(|&s| s.to_string()).collect()) + } +} + +pub struct StopWordFilterStream<'a> { + words: StopWordHashSet, + tail: BoxTokenStream<'a>, +} + +impl TokenFilter for StopWordFilter { + fn transform<'a>(&self, token_stream: BoxTokenStream<'a>) -> BoxTokenStream<'a> { + BoxTokenStream::from(StopWordFilterStream { + words: self.words.clone(), + tail: token_stream, + }) + } +} + +impl<'a> StopWordFilterStream<'a> { + fn predicate(&self, token: &Token) -> bool { + !self.words.contains(&token.text) + } +} + +impl<'a> TokenStream for StopWordFilterStream<'a> { + fn advance(&mut self) -> bool { + while self.tail.advance() { + if self.predicate(self.tail.token()) { + return true; + } + } + false + } + + fn token(&self) -> &Token { + self.tail.token() + } + + fn token_mut(&mut self) -> &mut Token { + self.tail.token_mut() + } +} + +impl Default for StopWordFilter { + fn default() -> StopWordFilter { + StopWordFilter::english() + } +} From b12d9ae9e6d42773f3b642b1c6c7faab3a08ab60 Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Sat, 29 Apr 2023 11:46:01 -0700 Subject: [PATCH 18/30] tweak: update WriteTrait to be more generic & add ToDocument trait (#444) * update WriteTrait to be more generic & add ToDocument trait to convert to tantivy Documents before upserts * updating tests * cargo fmt * add support for custom boosts/filters --- Cargo.lock | 1 + crates/spyglass-searcher/src/client/local.rs | 32 ++------ crates/spyglass-searcher/src/lib.rs | 41 +++++----- crates/spyglass-searcher/src/query.rs | 14 ++++ crates/spyglass-searcher/src/schema.rs | 35 +++++++++ crates/spyglass/src/api/handler/mod.rs | 25 ++++--- crates/spyglass/src/documents/mod.rs | 74 +++++++++++-------- .../spyglass/src/pipeline/default_pipeline.rs | 25 ++++--- 8 files changed, 151 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 79a8ff80a..2f7b9e28f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6776,6 +6776,7 @@ dependencies = [ "anyhow", "async-trait", "chrono", + "fnv", "log", "reqwest", "ron", diff --git a/crates/spyglass-searcher/src/client/local.rs b/crates/spyglass-searcher/src/client/local.rs index 7831dd166..9abfb2138 100644 --- a/crates/spyglass-searcher/src/client/local.rs +++ b/crates/spyglass-searcher/src/client/local.rs @@ -11,11 +11,11 @@ use uuid::Uuid; use crate::query::{build_document_query, build_query, terms_for_field, QueryOptions}; use crate::schema::{self, DocFields, SearchDocument}; use crate::{ - document_to_struct, Boost, DocumentUpdate, IndexBackend, QueryBoost, RetrievedDocument, Score, + document_to_struct, field_to_string, Boost, IndexBackend, QueryBoost, RetrievedDocument, Score, SearchError, SearchQueryResult, SearchTrait, SearcherResult, WriteTrait, }; -const SPYGLASS_NS: Uuid = uuid::uuid!("5fdfe40a-de2c-11ed-bfa7-00155deae876"); +pub const SPYGLASS_NS: Uuid = uuid::uuid!("5fdfe40a-de2c-11ed-bfa7-00155deae876"); /// Tantivy searcher client #[derive(Clone)] @@ -48,33 +48,15 @@ impl WriteTrait for Searcher { Ok(doc_ids.len()) } - async fn upsert_many(&self, updates: &[DocumentUpdate]) -> SearcherResult> { + async fn upsert_many(&self, updates: &[Document]) -> SearcherResult> { let mut upserted = Vec::new(); - for doc_update in updates { - let fields = DocFields::as_fields(); - - let doc_id = doc_update.doc_id.clone().map_or_else( - || { - Uuid::new_v5(&SPYGLASS_NS, doc_update.url.as_bytes()) - .as_hyphenated() - .to_string() - }, - |s| s, - ); - - let mut doc = Document::default(); - doc.add_text(fields.content, doc_update.content); - doc.add_text(fields.domain, doc_update.domain); - doc.add_text(fields.id, &doc_id); - doc.add_text(fields.title, doc_update.title); - doc.add_text(fields.url, doc_update.url); - for t in doc_update.tags { - doc.add_u64(fields.tags, *t as u64); - } + let fields = DocFields::as_fields(); + for doc_update in updates { let writer = self.lock_writer()?; - writer.add_document(doc)?; + writer.add_document(doc_update.clone())?; + let doc_id = field_to_string(doc_update, fields.id); upserted.push(doc_id.clone()); } diff --git a/crates/spyglass-searcher/src/lib.rs b/crates/spyglass-searcher/src/lib.rs index d46cfeedb..2404b948b 100644 --- a/crates/spyglass-searcher/src/lib.rs +++ b/crates/spyglass-searcher/src/lib.rs @@ -8,7 +8,7 @@ use url::Url; pub mod client; pub mod schema; pub mod stop_word_filter; -use schema::{DocFields, DocumentUpdate, SearchDocument}; +use schema::{DocFields, SearchDocument}; mod query; pub mod similarity; @@ -40,6 +40,7 @@ impl QueryBoost { Boost::Favorite { .. } => 3.0, Boost::Tag(_) => 1.5, Boost::Url(_) => 3.0, + Boost::CustomField { .. } => 0.0, }; QueryBoost { @@ -56,6 +57,7 @@ pub enum Boost { Url(String), DocId(String), Tag(u64), + CustomField { field_name: String, value: u64 }, } /// Contains stats & results for a search request @@ -105,13 +107,13 @@ pub trait WriteTrait { /// Delete documents from the index by id, returning the number of docs deleted. async fn delete_many_by_id(&self, doc_ids: &[String]) -> SearcherResult; - async fn upsert(&self, single_doc: &DocumentUpdate) -> SearcherResult { + async fn upsert(&self, single_doc: &Document) -> SearcherResult { let upserted = self.upsert_many(&[single_doc.clone()]).await?; Ok(upserted.get(0).expect("Expected a single doc").to_owned()) } /// Insert/update documents in the index, returning the list of document ids - async fn upsert_many(&self, updates: &[DocumentUpdate]) -> SearcherResult>; + async fn upsert_many(&self, updates: &[Document]) -> SearcherResult>; } type SearcherResult = Result; @@ -169,8 +171,8 @@ pub fn document_to_struct(doc: &Document) -> Option { #[cfg(test)] mod test { use crate::client::Searcher; - use crate::schema::{DocFields, SearchDocument}; - use crate::{Boost, DocumentUpdate, IndexBackend, QueryBoost, SearchTrait, WriteTrait}; + use crate::schema::{DocFields, DocumentUpdate, SearchDocument, ToDocument}; + use crate::{Boost, IndexBackend, QueryBoost, SearchTrait, WriteTrait}; async fn _build_test_index(searcher: &mut Searcher) { searcher @@ -191,7 +193,7 @@ mod test { tags: &vec![1_i64], published_at: None, last_modified: None, - }) + }.to_document()) .await .expect("Unable to add doc"); @@ -213,27 +215,30 @@ mod test { tags: &vec![2_i64], published_at: None, last_modified: None, - }) + }.to_document()) .await .expect("Unable to add doc"); searcher - .upsert(&DocumentUpdate { - doc_id: None, - title: "Of Cheese and Crackers", - domain: "en.wikipedia.org", - url: "https://en.wikipedia.org/cheese_and_crackers", - content: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla + .upsert( + &DocumentUpdate { + doc_id: None, + title: "Of Cheese and Crackers", + domain: "en.wikipedia.org", + url: "https://en.wikipedia.org/cheese_and_crackers", + content: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla tellus tortor, varius sit amet fermentum a, finibus porttitor erat. Proin suscipit, dui ac posuere vulputate, justo est faucibus est, a bibendum nulla nulla sed elit. Vivamus et libero a tortor ultricies feugiat in vel eros. Donec rhoncus mauris libero, et imperdiet neque sagittis sed. Nulla ac volutpat massa. Vivamus sed imperdiet est, id pretium ex. Praesent suscipit mattis ipsum, a lacinia nunc semper vitae.", - tags: &vec![2_i64], - published_at: None, - last_modified: None, - }) + tags: &vec![2_i64], + published_at: None, + last_modified: None, + } + .to_document(), + ) .await .expect("Unable to add doc"); @@ -250,7 +255,7 @@ mod test { tags: &vec![1_i64], published_at: None, last_modified: None - }).await + }.to_document()).await .expect("Unable to add doc"); let res = searcher.save().await; diff --git a/crates/spyglass-searcher/src/query.rs b/crates/spyglass-searcher/src/query.rs index b834e57f3..6668e7d10 100644 --- a/crates/spyglass-searcher/src/query.rs +++ b/crates/spyglass-searcher/src/query.rs @@ -121,6 +121,13 @@ pub fn build_query( // Originally boosted to 3.0 _boosted_term(Term::from_field_text(fields.url, url), boost.value) } + Boost::CustomField { field_name, value } => { + if let Some((field, _)) = schema.find_field(field_name) { + _boosted_term(Term::from_field_u64(field, *value), boost.value) + } else { + continue; + } + } }; term_query.push((Occur::Should, term)); @@ -158,6 +165,13 @@ pub fn build_query( // Originally boosted to 3.0 _boosted_term(Term::from_field_text(fields.url, url), 0.0) } + Boost::CustomField { field_name, value } => { + if let Some((field, _)) = schema.find_field(field_name) { + _boosted_term(Term::from_field_u64(field, *value), 0.0) + } else { + continue; + } + } }; combined.push((Occur::Must, term)); diff --git a/crates/spyglass-searcher/src/schema.rs b/crates/spyglass-searcher/src/schema.rs index 4da405c4c..9a938ba35 100644 --- a/crates/spyglass-searcher/src/schema.rs +++ b/crates/spyglass-searcher/src/schema.rs @@ -10,6 +10,10 @@ use tantivy::{ }, Index, }; +use uuid::Uuid; + +use crate::client::SPYGLASS_NS; + pub type FieldName = String; pub const TOKENIZER_NAME: &str = "spyglass_tokenizer_en"; @@ -84,6 +88,10 @@ pub fn register_tokenizer(index: &Index) { .register(TOKENIZER_NAME, full_content_tokenizer_en); } +pub trait ToDocument { + fn to_document(&self) -> Document; +} + /// A new document to be added #[derive(Clone)] pub struct DocumentUpdate<'a> { @@ -97,6 +105,33 @@ pub struct DocumentUpdate<'a> { pub last_modified: Option>, } +impl<'a> ToDocument for DocumentUpdate<'a> { + fn to_document(&self) -> Document { + let fields = DocFields::as_fields(); + + let doc_id = self.doc_id.clone().map_or_else( + || { + Uuid::new_v5(&SPYGLASS_NS, self.url.as_bytes()) + .as_hyphenated() + .to_string() + }, + |s| s, + ); + + let mut doc = Document::default(); + doc.add_text(fields.content, self.content); + doc.add_text(fields.domain, self.domain); + doc.add_text(fields.id, &doc_id); + doc.add_text(fields.title, self.title); + doc.add_text(fields.url, self.url); + for t in self.tags { + doc.add_u64(fields.tags, *t as u64); + } + + doc + } +} + #[derive(Clone)] pub struct DocFields { pub id: Field, diff --git a/crates/spyglass/src/api/handler/mod.rs b/crates/spyglass/src/api/handler/mod.rs index b781daaf4..74cdd79a0 100644 --- a/crates/spyglass/src/api/handler/mod.rs +++ b/crates/spyglass/src/api/handler/mod.rs @@ -659,7 +659,7 @@ mod test { }; use libspyglass::state::AppState; use shared::config::{Config, LensConfig}; - use spyglass_searcher::schema::DocumentUpdate; + use spyglass_searcher::schema::{DocumentUpdate, ToDocument}; use spyglass_searcher::WriteTrait; #[tokio::test] @@ -676,16 +676,19 @@ mod test { state .index - .upsert(&DocumentUpdate { - doc_id: Some("test_id".into()), - title: "test title", - domain: "example.com", - url: "https://example.com/test", - content: "test content", - tags: &[], - published_at: None, - last_modified: None, - }) + .upsert( + &DocumentUpdate { + doc_id: Some("test_id".into()), + title: "test title", + domain: "example.com", + url: "https://example.com/test", + content: "test content", + tags: &[], + published_at: None, + last_modified: None, + } + .to_document(), + ) .await .expect("Unable to add doc"); let _ = state.index.save().await; diff --git a/crates/spyglass/src/documents/mod.rs b/crates/spyglass/src/documents/mod.rs index 18820f014..cf8ac3ff4 100644 --- a/crates/spyglass/src/documents/mod.rs +++ b/crates/spyglass/src/documents/mod.rs @@ -18,7 +18,10 @@ use url::Url; use crate::{crawler::CrawlResult, state::AppState}; use entities::models::tag::TagType; use entities::sea_orm::{ColumnTrait, EntityTrait, QueryFilter, Set, TransactionTrait}; -use spyglass_searcher::{schema::DocumentUpdate, RetrievedDocument, WriteTrait}; +use spyglass_searcher::{ + schema::{DocumentUpdate, ToDocument}, + RetrievedDocument, WriteTrait, +}; /// Helper method to delete indexed documents, crawl queue items and search /// documents by url @@ -140,16 +143,19 @@ pub async fn process_crawl_results( // Add document to index let doc_id = state .index - .upsert(&DocumentUpdate { - doc_id: id_map.get(&crawl_result.url).cloned(), - title: &crawl_result.title.clone().unwrap_or_default(), - domain: url_host, - url: url.as_str(), - content: &crawl_result.content.clone().unwrap_or_default(), - tags: &tags_for_crawl.clone(), - published_at: None, - last_modified: None, - }) + .upsert( + &DocumentUpdate { + doc_id: id_map.get(&crawl_result.url).cloned(), + title: &crawl_result.title.clone().unwrap_or_default(), + domain: url_host, + url: url.as_str(), + content: &crawl_result.content.clone().unwrap_or_default(), + tags: &tags_for_crawl.clone(), + published_at: None, + last_modified: None, + } + .to_document(), + ) .await?; if !id_map.contains_key(&doc_id) { @@ -279,16 +285,19 @@ pub async fn process_records( let doc_id: Option = { match state .index - .upsert(&DocumentUpdate { - doc_id: id_map.get(&canonical_url_str.clone()).cloned(), - title: &crawl_result.title.clone().unwrap_or_default(), - domain: url_host, - url: url.as_str(), - content: &crawl_result.content, - tags: &tag_list, - published_at: None, - last_modified: None, - }) + .upsert( + &DocumentUpdate { + doc_id: id_map.get(&canonical_url_str.clone()).cloned(), + title: &crawl_result.title.clone().unwrap_or_default(), + domain: url_host, + url: url.as_str(), + content: &crawl_result.content, + tags: &tag_list, + published_at: None, + last_modified: None, + } + .to_document(), + ) .await { Ok(new_doc_id) => Some(new_doc_id), @@ -436,16 +445,19 @@ pub async fn update_tags( for (_, (doc, ids)) in tag_map.iter() { let _doc_id = state .index - .upsert(&DocumentUpdate { - doc_id: Some(doc.doc_id.clone()), - title: &doc.title, - domain: &doc.domain, - url: &doc.url, - content: &doc.content, - tags: ids, - published_at: None, - last_modified: None, - }) + .upsert( + &DocumentUpdate { + doc_id: Some(doc.doc_id.clone()), + title: &doc.title, + domain: &doc.domain, + url: &doc.url, + content: &doc.content, + tags: ids, + published_at: None, + last_modified: None, + } + .to_document(), + ) .await?; } } diff --git a/crates/spyglass/src/pipeline/default_pipeline.rs b/crates/spyglass/src/pipeline/default_pipeline.rs index 14d4f7e83..49ad98c11 100644 --- a/crates/spyglass/src/pipeline/default_pipeline.rs +++ b/crates/spyglass/src/pipeline/default_pipeline.rs @@ -6,7 +6,7 @@ use entities::models::{crawl_queue, indexed_document}; use entities::sea_orm::prelude::*; use entities::sea_orm::{ColumnTrait, EntityTrait, QueryFilter, Set}; use shared::config::{Config, LensConfig, PipelineConfiguration}; -use spyglass_searcher::schema::DocumentUpdate; +use spyglass_searcher::schema::{DocumentUpdate, ToDocument}; use spyglass_searcher::WriteTrait; use tokio::sync::mpsc; use url::Url; @@ -134,16 +134,19 @@ async fn start_crawl( let doc_id: Option = { match state .index - .upsert(&DocumentUpdate { - doc_id: existing.clone().map(|f| f.doc_id), - title: &crawl_result.title.unwrap_or_default(), - domain: url_host, - url: url.as_str(), - content: &content, - tags: &[], - published_at: None, - last_modified: None, - }) + .upsert( + &DocumentUpdate { + doc_id: existing.clone().map(|f| f.doc_id), + title: &crawl_result.title.unwrap_or_default(), + domain: url_host, + url: url.as_str(), + content: &content, + tags: &[], + published_at: None, + last_modified: None, + } + .to_document(), + ) .await { Ok(new_doc_id) => Some(new_doc_id), From d9ad2f11af7d801b15bc820a079e531c577fec95 Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Mon, 1 May 2023 20:54:35 -0700 Subject: [PATCH 19/30] make field/value in QueryBoost public --- crates/spyglass-searcher/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/spyglass-searcher/src/lib.rs b/crates/spyglass-searcher/src/lib.rs index 2404b948b..f29ddfd76 100644 --- a/crates/spyglass-searcher/src/lib.rs +++ b/crates/spyglass-searcher/src/lib.rs @@ -28,9 +28,9 @@ pub enum IndexBackend { #[derive(Clone, Serialize, Deserialize)] pub struct QueryBoost { /// What to boost - field: Boost, + pub field: Boost, /// The boost value (negative to lessen impact of something) - value: f32, + pub value: f32, } impl QueryBoost { From 0363abda8e7b95695d9c1c80b08aff3a2c66743f Mon Sep 17 00:00:00 2001 From: Andrew Huynh Date: Thu, 4 May 2023 14:16:54 -0700 Subject: [PATCH 20/30] feature/auth0 integration (#446) * handling Auth0 integration * adding in external auth helper functions & load secrets into web app via env vars * basic styling for login/logout buttons & profile name * pull auth0 id from user_profile * also grab auth token for API requests when we authenticate against Auth0 * add user data from backend to context * pulling out HTTP_ENDPOINT into an env var * removing debug info --- .env.template | 7 +++ .github/workflows/rust.yml | 2 + .gitignore | 1 + Cargo.lock | 32 ++++++++++++ apps/web/Cargo.toml | 5 +- apps/web/index.html | 1 + apps/web/public/auth.js | 43 +++++++++++++++++ apps/web/src/client.rs | 50 ++++++++++++++++--- apps/web/src/constants.rs | 5 -- apps/web/src/main.rs | 99 +++++++++++++++++++++++++++++++++++--- apps/web/src/pages/mod.rs | 83 +++++++++++++++++++++++++++++--- 11 files changed, 300 insertions(+), 28 deletions(-) create mode 100644 .env.template create mode 100644 apps/web/public/auth.js delete mode 100644 apps/web/src/constants.rs diff --git a/.env.template b/.env.template new file mode 100644 index 000000000..d278266f7 --- /dev/null +++ b/.env.template @@ -0,0 +1,7 @@ +SPYGLASS_BACKEND_DEV=http://127.0.0.1:8757 +SPYGLASS_BACKEND_PROD= + +AUTH0_DOMAIN= +AUTH0_CLIENT_ID= +AUTH0_REDIRECT_URI= +AUTH0_AUDIENCE= \ No newline at end of file diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 0621b1c66..2a6878d0a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -30,6 +30,8 @@ jobs: echo "target_arch=$(rustc -Vv | grep host | awk '{print $2 " "}')" >> $GITHUB_ENV; echo "target_ext=" >> $GITHUB_ENV; echo "target_os_name=linux" >> $GITHUB_ENV; + - name: Setup env file + run: cp .env.template .env # Setup rust toolchain - name: Setup rust toolchain uses: actions-rs/toolchain@v1 diff --git a/.gitignore b/.gitignore index e7485a492..20d7a9614 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ data _notebooks *.wasm Makefile.dev +.env # Added by cargo target diff --git a/Cargo.lock b/Cargo.lock index 2f7b9e28f..94c952c7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1633,6 +1633,35 @@ dependencies = [ "zip", ] +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + +[[package]] +name = "dotenv_codegen" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56966279c10e4f8ee8c22123a15ed74e7c8150b658b26c619c53f4a56eb4a8aa" +dependencies = [ + "dotenv_codegen_implementation", + "proc-macro-hack", +] + +[[package]] +name = "dotenv_codegen_implementation" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e737a3522cd45f6adc19b644ce43ef53e1e9045f2d2de425c1f468abd4cf33" +dependencies = [ + "dotenv", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -9027,12 +9056,15 @@ name = "web" version = "0.1.0" dependencies = [ "console_log", + "dotenv_codegen", "futures", "gloo", + "js-sys", "log", "markdown", "reqwest", "serde", + "serde-wasm-bindgen 0.5.0", "serde_json", "shared", "strum 0.24.1", diff --git a/apps/web/Cargo.toml b/apps/web/Cargo.toml index 0ebedd851..9f8dea7aa 100644 --- a/apps/web/Cargo.toml +++ b/apps/web/Cargo.toml @@ -4,10 +4,12 @@ version = "0.1.0" edition = "2021" [dependencies] +dotenv_codegen = "0.15.0" "shared" = { path = "../../crates/shared" } console_log = "1.0" futures = "0.3" gloo = "0.8.0" +js-sys = "0.3" log = "0.4" markdown = "1.0.0-alpha.7" reqwest = { version = "0.11", features = ["json", "stream"] } @@ -19,6 +21,7 @@ thiserror = "1.0" ui-components = { path = "../../crates/ui-components" } wasm-bindgen = "0.2.83" wasm-bindgen-futures = "0.4.33" -web-sys = { version = "0.3.60", features = ["Navigator", "VisibilityState"] } +serde-wasm-bindgen = "0.5" +web-sys = { version = "0.3.60", features = ["History", "Navigator", "VisibilityState"] } yew = { version = "0.20.0", features = ["csr"] } yew-router = "0.17" \ No newline at end of file diff --git a/apps/web/index.html b/apps/web/index.html index 5599bd9b8..bea0d15f2 100644 --- a/apps/web/index.html +++ b/apps/web/index.html @@ -4,6 +4,7 @@ + diff --git a/apps/web/public/auth.js b/apps/web/public/auth.js new file mode 100644 index 000000000..e92ef0d43 --- /dev/null +++ b/apps/web/public/auth.js @@ -0,0 +1,43 @@ +export function init_env(domain, client_id, redirect_uri, audience) { + window.AUTH0 = { + domain, + client_id, + redirect_uri, + audience + }; +} + +async function get_client() { + let client = await auth0 + .createAuth0Client({ + domain: window.AUTH0.domain, + clientId: window.AUTH0.client_id, + authorizationParams: { + audience: window.AUTH0.audience, + redirect_uri: window.AUTH0.redirect_uri, + }, + }); + + return client; +} + +export async function auth0_login() { + await get_client().then(client => client.loginWithRedirect()); +} + +export async function auth0_logout() { + await get_client().then(client => client.logout()); +} + +export async function handle_login_callback() { + return await get_client() + .then(async client => { + await client.handleRedirectCallback(); + + const isAuthenticated = await client.isAuthenticated(); + const userProfile = await client.getUser(); + const token = await client.getTokenSilently(); + return { isAuthenticated, userProfile, token }; + }) + .catch(err => console.log(err)); +} \ No newline at end of file diff --git a/apps/web/src/client.rs b/apps/web/src/client.rs index ef36fa3bb..c7c003fb1 100644 --- a/apps/web/src/client.rs +++ b/apps/web/src/client.rs @@ -1,20 +1,19 @@ use std::str::Utf8Error; use std::time::Duration; +use dotenv_codegen::dotenv; use futures::io::BufReader; use futures::{AsyncBufReadExt, TryStreamExt}; use gloo::timers::future::sleep; use reqwest::Client; +use serde::{Deserialize, Serialize}; use shared::request::{AskClippyRequest, ClippyContext}; use shared::response::{ChatErrorType, ChatUpdate, SearchResult}; use thiserror::Error; use yew::html::Scope; use crate::pages::search::{HistoryItem, HistorySource}; -use crate::{ - constants, - pages::search::{Msg, SearchPage}, -}; +use crate::pages::search::{Msg, SearchPage}; #[allow(clippy::enum_variant_names)] #[derive(Error, Debug)] @@ -30,12 +29,23 @@ pub enum ClientError { pub struct SpyglassClient { client: Client, lens: String, + endpoint: String, } impl SpyglassClient { pub fn new(lens: String) -> Self { let client = Client::new(); - Self { client, lens } + + #[cfg(debug_assertions)] + let endpoint = dotenv!("SPYGLASS_BACKEND_DEV"); + #[cfg(not(debug_assertions))] + let endpoint = dotenv!("SPYGLASS_BACKEND_PROD"); + + Self { + client, + lens, + endpoint: endpoint.to_string(), + } } pub async fn followup( @@ -84,7 +94,7 @@ impl SpyglassClient { body: &AskClippyRequest, link: Scope, ) -> Result<(), ClientError> { - let url = format!("{}/chat", constants::HTTP_ENDPOINT); + let url = format!("{}/chat", self.endpoint); let res = self .client @@ -153,3 +163,31 @@ impl SpyglassClient { Ok(()) } } + +#[derive(Clone, Serialize, Deserialize, PartialEq)] +pub struct Lens { + pub id: i64, + pub name: String, +} + +#[derive(Clone, Serialize, Deserialize, PartialEq)] +pub struct UserData { + pub display_name: String, + pub lenses: Vec, +} + +pub async fn get_user_data(auth_token: &str) -> Result { + #[cfg(debug_assertions)] + let endpoint = dotenv!("SPYGLASS_BACKEND_DEV"); + #[cfg(not(debug_assertions))] + let endpoint = dotenv!("SPYGLASS_BACKEND_PROD"); + + let client = reqwest::Client::new(); + client + .get(format!("{}/user", endpoint)) + .bearer_auth(auth_token) + .send() + .await? + .json::() + .await +} diff --git a/apps/web/src/constants.rs b/apps/web/src/constants.rs deleted file mode 100644 index 43df4c34d..000000000 --- a/apps/web/src/constants.rs +++ /dev/null @@ -1,5 +0,0 @@ -// todo: pull these from environment variables? config? -#[cfg(not(debug_assertions))] -pub const HTTP_ENDPOINT: &str = "https://api.spyglass.fyi"; -#[cfg(debug_assertions)] -pub const HTTP_ENDPOINT: &str = "http://127.0.0.1:8757"; diff --git a/apps/web/src/main.rs b/apps/web/src/main.rs index a8903f9e4..271c289bf 100644 --- a/apps/web/src/main.rs +++ b/apps/web/src/main.rs @@ -1,20 +1,57 @@ -use wasm_bindgen::{prelude::Closure, JsValue}; +use client::UserData; +use dotenv_codegen::dotenv; +use gloo::utils::{history, window}; +use serde::{Deserialize, Serialize}; +use wasm_bindgen::{prelude::*, JsValue}; +use wasm_bindgen_futures::spawn_local; use yew::prelude::*; use yew_router::prelude::*; mod client; -mod constants; mod pages; use pages::AppPage; +#[derive(Clone, Serialize, Deserialize, PartialEq)] +pub struct Auth0User { + pub name: String, + pub email: String, + pub picture: String, + pub sub: String, +} + +#[derive(Clone, Serialize, Deserialize, PartialEq)] +pub struct AuthStatus { + #[serde(rename(deserialize = "isAuthenticated"))] + pub is_authenticated: bool, + #[serde(rename(deserialize = "userProfile"))] + pub user_profile: Option, + pub token: Option, + // Only used internall + #[serde(skip)] + pub user_data: Option, +} + +#[wasm_bindgen(module = "/public/auth.js")] +extern "C" { + #[wasm_bindgen] + pub fn init_env(domain: &str, client_id: &str, redirect_uri: &str, audience: &str); + + #[wasm_bindgen(catch)] + pub async fn auth0_login() -> Result<(), JsValue>; + + #[wasm_bindgen(catch)] + pub async fn auth0_logout() -> Result<(), JsValue>; + + #[wasm_bindgen(catch)] + pub async fn handle_login_callback() -> Result; +} + #[derive(Clone, Routable, PartialEq)] pub enum Route { #[at("/")] Start, #[at("/lens/:lens")] Search { lens: String }, - // #[at("/library")] - // MyLibrary, #[not_found] #[at("/404")] NotFound, @@ -22,7 +59,7 @@ pub enum Route { fn switch(routes: Route) -> Html { match &routes { - Route::Start => html! { to={Route::Search { lens: "yc".into() }} /> }, + Route::Start => html! { }, Route::Search { lens } => html! { }, Route::NotFound => html! {
    {"Not Found!"}
    }, } @@ -34,10 +71,56 @@ pub async fn listen(_event_name: &str, _cb: &Closure) -> Result #[function_component] fn App() -> Html { + // Initialize JS env vars + init_env( + dotenv!("AUTH0_DOMAIN"), + dotenv!("AUTH0_CLIENT_ID"), + dotenv!("AUTH0_REDIRECT_URI"), + dotenv!("AUTH0_AUDIENCE"), + ); + + let auth_status: UseStateHandle = use_state_eq(|| AuthStatus { + is_authenticated: false, + user_profile: None, + token: None, + user_data: None, + }); + let search = window().location().search().unwrap_or_default(); + + if search.contains("state=") { + log::info!("handling auth callback"); + let auth_status_handle = auth_status.clone(); + spawn_local(async move { + if let Ok(details) = handle_login_callback().await { + let _ = + history().replace_state_with_url(&JsValue::NULL, "Spyglass Search", Some("/")); + match serde_wasm_bindgen::from_value::(details) { + Ok(mut value) => { + let token = value + .token + .as_ref() + .map(|x| x.to_string()) + .unwrap_or_default(); + + if let Ok(user_data) = client::get_user_data(&token).await { + value.user_data = Some(user_data); + } + + auth_status_handle.set(value) + } + Err(err) => log::error!("Unable to parse user profile: {}", err.to_string()), + } + } + }); + } + html! { - - render={switch} /> - + context={(*auth_status).clone()}> + + render={switch} /> + + > + } } diff --git a/apps/web/src/pages/mod.rs b/apps/web/src/pages/mod.rs index 97bdf883c..7d5346046 100644 --- a/apps/web/src/pages/mod.rs +++ b/apps/web/src/pages/mod.rs @@ -1,8 +1,8 @@ use ui_components::icons; -use yew::prelude::*; -use yew_router::prelude::Link; +use yew::{platform::spawn_local, prelude::*}; +use yew_router::prelude::{use_navigator, Link}; -use crate::Route; +use crate::{auth0_login, auth0_logout, AuthStatus, Route}; pub mod search; #[derive(PartialEq, Properties)] @@ -39,18 +39,85 @@ pub struct AppPageProps { #[function_component] pub fn AppPage(props: &AppPageProps) -> Html { + let navigator = use_navigator().unwrap(); + let auth_status = use_context::().expect("Ctxt not set up"); + + let user_data = auth_status.user_data.clone(); + + let auth_login = Callback::from(|e: MouseEvent| { + e.prevent_default(); + spawn_local(async { + let _ = auth0_login().await; + }); + }); + + let auth_logout = Callback::from(|e: MouseEvent| { + e.prevent_default(); + spawn_local(async { + let _ = auth0_logout().await; + }); + }); + + let mut lenses = Vec::new(); + if let Some(user_data) = user_data { + for lens in user_data.lenses { + let navi = navigator.clone(); + let lens_name = lens.name.clone(); + let onclick = Callback::from(move |_| { + navi.push(&Route::Search { + lens: lens_name.clone(), + }) + }); + lenses.push(html! { +
  • + + + {lens.name.clone()} + +
    • + }); + } + } + html! {
    -
    +
    {"Spyglass"}
    + {if auth_status.is_authenticated { + if let Some(profile) = auth_status.user_profile { + html! { +
    +
    + +
    {profile.name}
    +
    + +
    + } + } else { + html !{} + } + } else { + html! { +
    + +
    + } + }} +
    +
    +
    + {"My Collections"} +
      -
    • - - {props.lens.clone()} -
      • + {lenses}